tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-05 12:23:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into litellm_org_usage

Browse Source
This commit is contained in:
yuneng-jiang
2025-11-14 18:05:50 -08:00
parent cfc44ea279 6063a75155
commit 63dcd00b8b
114 changed files with 5755 additions and 3005 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+24 -16
View File
@@ -563,8 +563,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.13
command: |
@@ -1770,8 +1771,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@@ -1908,8 +1910,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@@ -2050,8 +2053,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@@ -2234,8 +2238,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@@ -2342,8 +2347,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@@ -2475,8 +2481,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
sudo systemctl restart docker
- run:
name: Install Python 3.9
@@ -2684,8 +2691,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
+184
View File
@@ -0,0 +1,184 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Adding a New Guardrail Integration
You're going to create a class that checks text before it goes to the LLM or after it comes back. If it violates your rules, you block it.
## How It Works
Request with guardrail:
```bash
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "How do I hack a system?"}],
"guardrails": ["my-guardrail"]
}'
```
Your guardrail checks input, then output. If something's wrong, raise an exception.
## Build Your Guardrail
### Create Your Directory
```bash
mkdir -p litellm/proxy/guardrails/guardrail_hooks/my_guardrail
cd litellm/proxy/guardrails/guardrail_hooks/my_guardrail
```
Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization).
### Write the Main Class
`my_guardrail.py`:
```python
import os
from typing import Optional, List
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import PiiEntityType
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
class MyGuardrail(CustomGuardrail):
def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs):
self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY")
self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com")
super().__init__(default_on=True)
async def apply_guardrail(
self,
text: str,
language: Optional[str] = None,
entities: Optional[List[PiiEntityType]] = None,
request_data: Optional[dict] = None,
) -> str:
result = await self._check_with_api(text, request_data)
if result.get("action") == "BLOCK":
raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}")
return text
async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict:
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
response = await async_client.post(
f"{self.api_base}/check",
headers=headers,
json={"text": text},
timeout=5,
)
response.raise_for_status()
return response.json()
```
### Create the Init File
`__init__.py`:
```python
from typing import TYPE_CHECKING
from litellm.types.guardrails import SupportedGuardrailIntegrations
from .my_guardrail import MyGuardrail
if TYPE_CHECKING:
from litellm.types.guardrails import Guardrail, LitellmParams
def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"):
import litellm
_my_guardrail_callback = MyGuardrail(
api_base=litellm_params.api_base,
api_key=litellm_params.api_key,
guardrail_name=guardrail.get("guardrail_name", ""),
event_hook=litellm_params.mode,
default_on=litellm_params.default_on,
)
litellm.logging_callback_manager.add_litellm_callback(_my_guardrail_callback)
return _my_guardrail_callback
guardrail_initializer_registry = {
SupportedGuardrailIntegrations.MY_GUARDRAIL.value: initialize_guardrail,
}
guardrail_class_registry = {
SupportedGuardrailIntegrations.MY_GUARDRAIL.value: MyGuardrail,
}
```
### Register Your Guardrail Type
Add to `litellm/types/guardrails.py`:
```python
class SupportedGuardrailIntegrations(str, Enum):
LAKERA = "lakera_prompt_injection"
APORIA = "aporia"
BEDROCK = "bedrock_guardrails"
PRESIDIO = "presidio"
ZSCALER_AI_GUARD = "zscaler_ai_guard"
MY_GUARDRAIL = "my_guardrail"
```
## Usage
### Config File
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
guardrails:
- guardrail_name: my_guardrail
litellm_params:
guardrail: my_guardrail
mode: during_call
api_key: os.environ/MY_GUARDRAIL_API_KEY
api_base: https://api.myguardrail.com
```
### Per-Request
```bash
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Test message"}],
"guardrails": ["my_guardrail"]
}'
```
## Testing
Add unit tests inside `test_litellm/` folder.
docs/my-website/docs/providers/bedrock_batches.md
+31
View File
@@ -40,6 +40,8 @@ model_list:
s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
# Optional: Custom KMS encryption key for S3 output
# s3_encryption_key_id: arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012
model_info:
mode: batch # 👈 SPECIFY MODE AS BATCH, to tell user this is a batch model
```
@@ -55,6 +57,12 @@ model_list:
| `aws_batch_role_arn` | IAM role ARN for Bedrock batch operations. Bedrock Batch APIs require an IAM role ARN to be set. |
| `mode: batch` | Indicates to LiteLLM this is a batch model |
**Optional Parameters:**
| Parameter | Description |
|-----------|-------------|
| `s3_encryption_key_id` | Custom KMS encryption key ID for S3 output data. If not specified, Bedrock uses AWS managed encryption keys. |
### 2. Create Virtual Key
```bash showLineNumbers title="create_virtual_key.sh"
@@ -174,6 +182,29 @@ When a `target_model_names` is specified, the file is written to the S3 bucket c
LiteLLM only supports Bedrock Anthropic Models for Batch API. If you want other bedrock models file an issue [here](https://github.com/BerriAI/litellm/issues/new/choose).
### How do I use a custom KMS encryption key?
If your S3 bucket requires a custom KMS encryption key, you can specify it in your configuration using `s3_encryption_key_id`. This is useful for enterprise customers with specific encryption requirements.
You can set the encryption key in 2 ways:
1. **In config.yaml** (recommended):
```yaml
model_list:
- model_name: "bedrock-batch-claude"
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
s3_encryption_key_id: arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012
# ... other params
```
2. **As an environment variable**:
```bash
export AWS_S3_ENCRYPTION_KEY_ID=arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012
```
## Further Reading
- [AWS Bedrock Batch Inference Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)
docs/my-website/docs/providers/fal_ai.md
+1
View File
@@ -31,6 +31,7 @@ Get your API key from [fal.ai](https://fal.ai/).
| Model Name | Description | Documentation |
|------------|-------------|---------------|
| `fal_ai/flux/schnell` | Flux Schnell - Low-latency generation with `image_size` support | [Docs ↗](https://fal.ai/models/fal-ai/flux/schnell) |
| `fal_ai/fal-ai/flux-pro/v1.1-ultra` | FLUX Pro v1.1 Ultra - High-quality image generation | [Docs ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra) |
| `fal_ai/fal-ai/imagen4/preview` | Google's Imagen 4 - Highest quality model | [Docs ↗](https://fal.ai/models/fal-ai/imagen4/preview) |
| `fal_ai/fal-ai/recraft/v3/text-to-image` | Recraft v3 - Multiple style options | [Docs ↗](https://fal.ai/models/fal-ai/recraft/v3/text-to-image) |
docs/my-website/docs/providers/openai.md
+8 -3
View File
@@ -412,7 +412,7 @@ Expected Response:
### Advanced: Using `reasoning_effort` with `summary` field
By default, `reasoning_effort` accepts a string value (`"low"`, `"medium"`, `"high"`, `"minimal"`) and only sets the effort level without including a reasoning summary.
By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`) and only sets the effort level without including a reasoning summary.
To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI.
@@ -472,12 +472,17 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
| Model | Default (when not set) | Supported Values |
|-------|----------------------|------------------|
| `gpt-5.1` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `none`, `minimal`, `low`, `medium`, `high` |
| `gpt-5-nano` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5-pro` | `high` | `high` only |
**Note:** `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error. When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column.
**Note:**
- GPT-5.1 introduced a new `reasoning_effort="none"` setting for faster, lower-latency responses. This replaces the `"minimal"` setting from GPT-5.
- `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error.
- When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column.
See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements.
docs/my-website/docs/providers/vertex.md
+509 -47
View File
@@ -1604,53 +1604,6 @@ litellm.vertex_location = "us-central1 # Your Location
| gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` |
| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
## Private Service Connect (PSC) Endpoints
LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments.
### Usage
```python
from litellm import completion
# Use PSC endpoint with custom api_base
response = completion(
model="vertex_ai/1234567890", # Numeric endpoint ID
messages=[{"role": "user", "content": "Hello!"}],
api_base="http://10.96.32.8", # Your PSC endpoint
vertex_project="my-project-id",
vertex_location="us-central1"
)
```
**Key Features:**
- Supports both numeric endpoint IDs and custom model names
- Works with both completion and embedding endpoints
- Automatically constructs full PSC URL: `{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}`
- Compatible with streaming requests
### Configuration
Add PSC endpoints to your `config.yaml`:
```yaml
model_list:
- model_name: psc-gemini
litellm_params:
model: vertex_ai/1234567890 # Numeric endpoint ID
api_base: "http://10.96.32.8" # Your PSC endpoint
vertex_project: "my-project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
- model_name: psc-embedding
litellm_params:
model: vertex_ai/text-embedding-004
api_base: "http://10.96.32.8" # Your PSC endpoint
vertex_project: "my-project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
```
## Fine-tuned Models
You can call fine-tuned Vertex AI Gemini models through LiteLLM
@@ -2089,6 +2042,515 @@ curl http://0.0.0.0:4000/v1/chat/completions \
| code-gecko@latest| `completion('code-gecko@latest', messages)` |
## **Embedding Models**
#### Usage - Embedding
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
from litellm import embedding
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = embedding(
model="vertex_ai/textembedding-gecko",
input=["good morning from litellm"],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: snowflake-arctic-embed-m-long-1731622468876
litellm_params:
model: vertex_ai/<your-model-id>
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK, Langchain Python SDK
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="snowflake-arctic-embed-m-long-1731622468876",
input = ["good morning from litellm", "this is another item"],
)
print(response)
```
</TabItem>
</Tabs>
#### Supported Embedding Models
All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` |
| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` |
| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` |
| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` |
| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` |
| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` |
| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` |
| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` |
| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` |
### Supported OpenAI (Unified) Params
| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) |
|-------|-------------|--------------------|
| `input` | **string or List[string]** | `instances` |
| `dimensions` | **int** | `output_dimensionality` |
| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` |
#### Usage with OpenAI (Unified) Params
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
input_type = "RETRIEVAL_DOCUMENT",
dimensions=1,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"input_type": "RETRIEVAL_QUERY",
}
)
print(response)
```
</TabItem>
</Tabs>
### Supported Vertex Specific Params
| param | type |
|-------|-------------|
| `auto_truncate` | **bool** |
| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** |
| `title` | **str** |
#### Usage with Vertex Specific Params (Use `task_type` and `title`)
You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this:
[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
task_type = "RETRIEVAL_DOCUMENT",
title = "test",
dimensions=1,
auto_truncate=True,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"task_type": "RETRIEVAL_QUERY",
"auto_truncate": True,
"title": "test",
}
)
print(response)
```
</TabItem>
</Tabs>
## **Multi-Modal Embeddings**
Known Limitations:
- Only supports 1 image / video / image per request
- Only supports GCS or base64 encoded images / videos
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
Using GCS Images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image
)
```
Using base 64 encoded images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
<Tabs>
<TabItem value="OpenAI SDK" label="OpenAI SDK">
Requests with GCS Image / Video URI
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
)
print(response)
```
Requests with base64 encoded images
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "data:image/jpeg;base64,...",
)
print(response)
```
</TabItem>
<TabItem value="langchain" label="Langchain">
Requests with GCS Image / Video URI
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
print(query_result)
```
Requests with base64 encoded images
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"data:image/jpeg;base64,..."
)
print(query_result)
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="proxy-vtx" label="LiteLLM PROXY (Vertex SDK)">
1. Add model to config.yaml
```yaml
default_vertex_config:
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK
```python
import vertexai
from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video
from vertexai.vision_models import VideoSegmentConfig
from google.auth.credentials import Credentials
LITELLM_PROXY_API_KEY = "sk-1234"
LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai"
import datetime
class CredentialsWrapper(Credentials):
def __init__(self, token=None):
super().__init__()
self.token = token
self.expiry = None # or set to a future date if needed
def refresh(self, request):
pass
def apply(self, headers, token=None):
headers['Authorization'] = f'Bearer {self.token}'
@property
def expired(self):
return False # Always consider the token as non-expired
@property
def valid(self):
return True # Always consider the credentials as valid
credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY)
vertexai.init(
project="adroit-crow-413218",
location="us-central1",
api_endpoint=LITELLM_PROXY_BASE,
credentials = credentials,
api_transport="rest",
)
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
image = Image.load_from_file(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
embeddings = model.get_embeddings(
image=image,
contextual_text="Colosseum",
dimension=1408,
)
print(f"Image Embedding: {embeddings.image_embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
```
</TabItem>
</Tabs>
### Text + Image + Video Embeddings
<Tabs>
<TabItem value="sdk" label="SDK">
Text + Image
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image
)
```
Text + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
Image + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
Text + Image
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"],
)
print(response)
```
Text + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
Image + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
</TabItem>
</Tabs>
## **Gemini TTS (Text-to-Speech) Audio Output**
:::info
docs/my-website/docs/providers/vertex_embedding.md
-587
View File
@@ -1,587 +0,0 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI Embedding
## Usage - Embedding
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
from litellm import embedding
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = embedding(
model="vertex_ai/textembedding-gecko",
input=["good morning from litellm"],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: snowflake-arctic-embed-m-long-1731622468876
litellm_params:
model: vertex_ai/<your-model-id>
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK, Langchain Python SDK
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="snowflake-arctic-embed-m-long-1731622468876",
input = ["good morning from litellm", "this is another item"],
)
print(response)
```
</TabItem>
</Tabs>
#### Supported Embedding Models
All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` |
| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` |
| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` |
| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` |
| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` |
| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` |
| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` |
| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` |
| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` |
### Supported OpenAI (Unified) Params
| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) |
|-------|-------------|--------------------|
| `input` | **string or List[string]** | `instances` |
| `dimensions` | **int** | `output_dimensionality` |
| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` |
#### Usage with OpenAI (Unified) Params
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
input_type = "RETRIEVAL_DOCUMENT",
dimensions=1,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"input_type": "RETRIEVAL_QUERY",
}
)
print(response)
```
</TabItem>
</Tabs>
### Supported Vertex Specific Params
| param | type |
|-------|-------------|
| `auto_truncate` | **bool** |
| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** |
| `title` | **str** |
#### Usage with Vertex Specific Params (Use `task_type` and `title`)
You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this:
[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
task_type = "RETRIEVAL_DOCUMENT",
title = "test",
dimensions=1,
auto_truncate=True,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"task_type": "RETRIEVAL_QUERY",
"auto_truncate": True,
"title": "test",
}
)
print(response)
```
</TabItem>
</Tabs>
## **BGE Embeddings**
Use BGE (Baidu General Embedding) models deployed on Vertex AI.
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Using BGE on Vertex AI"
import litellm
response = litellm.embedding(
model="vertex_ai/bge/<your-endpoint-id>",
input=["Hello", "World"],
vertex_project="your-project-id",
vertex_location="your-location"
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: bge-embedding
litellm_params:
model: vertex_ai/bge/<your-endpoint-id>
vertex_project: "your-project-id"
vertex_location: "us-central1"
vertex_credentials: your-credentials.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```bash
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK
```python showLineNumbers title="Making requests to BGE"
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="bge-embedding",
input=["good morning from litellm", "this is another item"]
)
print(response)
```
Using a Private Service Connect (PSC) endpoint
```yaml showLineNumbers title="config.yaml (PSC)"
model_list:
- model_name: bge-small-en-v1.5
litellm_params:
model: vertex_ai/bge/1234567890
api_base: http://10.96.32.8 # Your PSC IP
vertex_project: my-project-id #optional
vertex_location: us-central1 #optional
```
</TabItem>
</Tabs>
## **Multi-Modal Embeddings**
Known Limitations:
- Only supports 1 image / video / image per request
- Only supports GCS or base64 encoded images / videos
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
Using GCS Images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image
)
```
Using base 64 encoded images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
<Tabs>
<TabItem value="OpenAI SDK" label="OpenAI SDK">
Requests with GCS Image / Video URI
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
)
print(response)
```
Requests with base64 encoded images
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "data:image/jpeg;base64,...",
)
print(response)
```
</TabItem>
<TabItem value="langchain" label="Langchain">
Requests with GCS Image / Video URI
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
print(query_result)
```
Requests with base64 encoded images
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"data:image/jpeg;base64,..."
)
print(query_result)
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="proxy-vtx" label="LiteLLM PROXY (Vertex SDK)">
1. Add model to config.yaml
```yaml
default_vertex_config:
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK
```python
import vertexai
from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video
from vertexai.vision_models import VideoSegmentConfig
from google.auth.credentials import Credentials
LITELLM_PROXY_API_KEY = "sk-1234"
LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai"
import datetime
class CredentialsWrapper(Credentials):
def __init__(self, token=None):
super().__init__()
self.token = token
self.expiry = None # or set to a future date if needed
def refresh(self, request):
pass
def apply(self, headers, token=None):
headers['Authorization'] = f'Bearer {self.token}'
@property
def expired(self):
return False # Always consider the token as non-expired
@property
def valid(self):
return True # Always consider the credentials as valid
credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY)
vertexai.init(
project="adroit-crow-413218",
location="us-central1",
api_endpoint=LITELLM_PROXY_BASE,
credentials = credentials,
api_transport="rest",
)
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
image = Image.load_from_file(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
embeddings = model.get_embeddings(
image=image,
contextual_text="Colosseum",
dimension=1408,
)
print(f"Image Embedding: {embeddings.image_embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
```
</TabItem>
</Tabs>
### Text + Image + Video Embeddings
<Tabs>
<TabItem value="sdk" label="SDK">
Text + Image
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image
)
```
Text + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
Image + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
Text + Image
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"],
)
print(response)
```
Text + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
Image + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
</TabItem>
</Tabs>
docs/my-website/docs/providers/voyage.md
+111 -3
View File
@@ -14,12 +14,41 @@ import os
os.environ['VOYAGE_API_KEY'] = ""
response = embedding(
model="voyage/voyage-3-large",
model="voyage/voyage-3.5",
input=["good morning from litellm"],
)
print(response)
```
## Supported Parameters
VoyageAI embeddings support the following optional parameters:
- `input_type`: Specifies the type of input for retrieval optimization
- `"query"`: Use for search queries
- `"document"`: Use for documents being indexed
- `dimensions`: Output embedding dimensions (256, 512, 1024, or 2048)
- `encoding_format`: Output format (`"float"`, `"int8"`, `"uint8"`, `"binary"`, `"ubinary"`)
- `truncation`: Whether to truncate inputs exceeding max tokens (default: `True`)
### Example with Parameters
```python
from litellm import embedding
import os
os.environ['VOYAGE_API_KEY'] = "your-api-key"
# Embedding with custom dimensions and input type
response = embedding(
model="voyage/voyage-3.5",
input=["Your text here"],
dimensions=512,
input_type="document"
)
print(f"Embedding dimensions: {len(response.data[0]['embedding'])}")
```
## Supported Models
All models listed here https://docs.voyageai.com/embeddings/#models-and-specifics are supported
@@ -40,5 +69,84 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
| voyage-2 | `embedding(model="voyage/voyage-2", input)` |
| voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` |
| voyage-01 | `embedding(model="voyage/voyage-01", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` |
## Contextual Embeddings (voyage-context-3)
VoyageAI's `voyage-context-3` model provides contextualized chunk embeddings, where each chunk is embedded with awareness of its surrounding document context. This significantly improves retrieval quality compared to standard context-agnostic embeddings.
### Key Benefits
- Chunks understand their position and role within the full document
- Improved retrieval accuracy for long documents (outperforms competitors by 7-23%)
- Better handling of ambiguous references and cross-chunk dependencies
- Seamless drop-in replacement for standard embeddings in RAG pipelines
### Usage
Contextual embeddings require a **nested input format** where each inner list represents chunks from a single document:
```python
from litellm import embedding
import os
os.environ['VOYAGE_API_KEY'] = "your-api-key"
# Single document with multiple chunks
response = embedding(
model="voyage/voyage-context-3",
input=[
[
"Chapter 1: Introduction to AI",
"This chapter covers the basics of artificial intelligence.",
"We will explore machine learning and deep learning."
]
]
)
print(f"Number of chunk groups: {len(response.data)}")
# Multiple documents
response = embedding(
model="voyage/voyage-context-3",
input=[
["Paris is the capital of France.", "It is known for the Eiffel Tower."],
["Tokyo is the capital of Japan.", "It is a major economic hub."]
]
)
print(f"Processed {len(response.data)} documents")
```
### Specifications
- Model: `voyage-context-3`
- Context length: 32,000 tokens per document
- Output dimensions: 256, 512, 1024 (default), or 2048
- Max inputs: 1,000 per request
- Max total tokens: 120,000
- Max chunks: 16,000
- Pricing: $0.18 per million tokens
### When to Use Contextual Embeddings
**Use `voyage-context-3` when:**
- Processing long documents split into chunks
- Document structure and flow are important
- References between sections matter
- You need to preserve document hierarchy
**Use standard models (voyage-3.5, voyage-3-large) when:**
- Embedding independent pieces of text
- Processing short queries
- Document context is not relevant
- You need faster/cheaper processing
## Model Selection Guide
| Model | Best For | Context Length | Price/M Tokens |
|-------|----------|----------------|----------------|
| voyage-3.5 | General-purpose, multilingual | 32K | $0.06 |
| voyage-3.5-lite | Latency-sensitive applications | 32K | $0.02 |
| voyage-3-large | Best overall quality | 32K | $0.18 |
| voyage-code-3 | Code retrieval and search | 32K | $0.18 |
| voyage-finance-2 | Financial documents | 32K | $0.12 |
| voyage-law-2 | Legal documents | 16K | $0.12 |
| voyage-context-3 | Contextual document embeddings | 32K | $0.18 |
docs/my-website/docs/proxy/model_access.md
+5 -233
View File
@@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Control Model Access
# Restrict Model Access
## **Restrict models by Virtual Key**
@@ -114,238 +114,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
### [API Reference](https://litellm-api.up.railway.app/#/team%20management/new_team_team_new_post)
## **Model Access Groups**
Use model access groups to give users access to select models, and add new ones to it over time (e.g. mistral, llama-2, etc.)
**Step 1. Assign model, access group in config.yaml**
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
model_info:
access_groups: ["beta-models"] # 👈 Model Access Group
- model_name: fireworks-llama-v3-70b-instruct
litellm_params:
model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct
api_key: "os.environ/FIREWORKS"
model_info:
access_groups: ["beta-models"] # 👈 Model Access Group
```
<Tabs>
<TabItem value="key" label="Key Access Groups">
**Create key with access group**
```bash
curl --location 'http://localhost:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-d '{"models": ["beta-models"], # 👈 Model Access Group
"max_budget": 0,}'
```
Test Key
<Tabs>
<TabItem label="Allowed Access" value = "allowed">
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
<TabItem label="Disallowed Access" value = "not-allowed">
:::info
Expect this to fail since gpt-4o is not in the `beta-models` access group
:::
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="team" label="Team Access Groups">
Create Team
```shell
curl --location 'http://localhost:4000/team/new' \
-H 'Authorization: Bearer sk-<key-from-previous-step>' \
-H 'Content-Type: application/json' \
-d '{"models": ["beta-models"]}'
```
Create Key for Team
```shell
curl --location 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-<key-from-previous-step>' \
--header 'Content-Type: application/json' \
--data '{"team_id": "0ac97648-c194-4c90-8cd6-40af7b0d2d2a"}
```
Test Key
<Tabs>
<TabItem label="Allowed Access" value = "allowed">
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
<TabItem label="Disallowed Access" value = "not-allowed">
:::info
Expect this to fail since gpt-4o is not in the `beta-models` access group
:::
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
### ✨ Control Access on Wildcard Models
Control access to all models with a specific prefix (e.g. `openai/*`).
Use this to also give users access to all models, except for a few that you don't want them to use (e.g. `openai/o1-*`).
:::info
Setting model access groups on wildcard models is an Enterprise feature.
See pricing [here](https://litellm.ai/#pricing)
Get a trial key [here](https://litellm.ai/#trial)
:::
1. Setup config.yaml
```yaml
model_list:
- model_name: openai/*
litellm_params:
model: openai/*
api_key: os.environ/OPENAI_API_KEY
model_info:
access_groups: ["default-models"]
- model_name: openai/o1-*
litellm_params:
model: openai/o1-*
api_key: os.environ/OPENAI_API_KEY
model_info:
access_groups: ["restricted-models"]
```
2. Generate a key with access to `default-models`
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"models": ["default-models"],
}'
```
3. Test the key
<Tabs>
<TabItem label="Successful Request" value = "success">
```bash
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "openai/gpt-4",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
<TabItem value="bad-request" label="Rejected Request">
```bash
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "openai/o1-mini",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
</Tabs>
## **View Available Fallback Models**
Use the `/v1/models` endpoint to discover available fallback models for a given model. This helps you understand which backup models are available when your primary model is unavailable or restricted.
@@ -451,4 +219,8 @@ When `include_metadata=true` is specified, the response includes fallback inform
| `include_metadata` | boolean | Include additional model metadata including fallbacks |
| `fallback_type` | string | Filter fallbacks by type: `general`, `context_window`, or `content_policy` |
## Advanced: Model Access Groups
For advanced use cases, use [Model Access Groups](./model_access_groups) to dynamically group multiple models and manage access without restarting the proxy.
## [Role Based Access Control (RBAC)](./jwt_auth_arch)
docs/my-website/docs/proxy/model_access_groups.md
+503
View File
@@ -0,0 +1,503 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Model Access Groups
### Overview
Group multiple models under a single name, then grant keys or teams access to the entire group. Add or remove models from a group without updating individual keys.
Use cases:
- Separate production and development models
- Restrict expensive models to specific teams
- Organize models by provider or capability
- Control access to model families with wildcards (e.g., `openai/*`)
### How It Works
```mermaid
graph LR
subgraph AG1["Access Group: 'prod-models'"]
M1["gpt-4o"]
M2["claude-opus"]
end
subgraph AG2["Access Group: 'dev-models'"]
M3["gpt-4o-mini"]
M4["claude-haiku"]
end
K1["Production API Key"] --> AG1
K2["Development API Key"] --> AG2
style AG1 fill:#e3f2fd
style AG2 fill:#fff8e1
```
**Key Concept:** Group models together → Attach group to key → Key gets access to all models in group
**Step 1. Assign model, access group in config.yaml**
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
model_info:
access_groups: ["beta-models"] # 👈 Model Access Group
- model_name: fireworks-llama-v3-70b-instruct
litellm_params:
model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct
api_key: "os.environ/FIREWORKS"
model_info:
access_groups: ["beta-models"] # 👈 Model Access Group
```
<Tabs>
<TabItem value="key" label="Key Access Groups">
**Create key with access group**
```bash showLineNumbers title="Create Key with Access Group"
curl --location 'http://localhost:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-d '{"models": ["beta-models"], # 👈 Model Access Group
"max_budget": 0,}'
```
Test Key
<Tabs>
<TabItem label="Allowed Access" value = "allowed">
```bash showLineNumbers title="Test Key - Allowed Access"
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
<TabItem label="Disallowed Access" value = "not-allowed">
:::info
Expect this to fail since gpt-4o is not in the `beta-models` access group
:::
```bash showLineNumbers title="Test Key - Disallowed Access"
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="team" label="Team Access Groups">
Create Team
```bash showLineNumbers title="Create Team"
curl --location 'http://localhost:4000/team/new' \
-H 'Authorization: Bearer sk-<key-from-previous-step>' \
-H 'Content-Type: application/json' \
-d '{"models": ["beta-models"]}'
```
Create Key for Team
```bash showLineNumbers title="Create Key for Team"
curl --location 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-<key-from-previous-step>' \
--header 'Content-Type: application/json' \
--data '{"team_id": "0ac97648-c194-4c90-8cd6-40af7b0d2d2a"}
```
Test Key
<Tabs>
<TabItem label="Allowed Access" value = "allowed">
```bash showLineNumbers title="Test Team Key - Allowed Access"
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
<TabItem label="Disallowed Access" value = "not-allowed">
:::info
Expect this to fail since gpt-4o is not in the `beta-models` access group
:::
```bash showLineNumbers title="Test Team Key - Disallowed Access"
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
### ✨ Control Access on Wildcard Models
Control access to all models with a specific prefix (e.g. `openai/*`).
Use this to also give users access to all models, except for a few that you don't want them to use (e.g. `openai/o1-*`).
:::info
Setting model access groups on wildcard models is an Enterprise feature.
See pricing [here](https://litellm.ai/#pricing)
Get a trial key [here](https://litellm.ai/#trial)
:::
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml - Wildcard Models"
model_list:
- model_name: openai/*
litellm_params:
model: openai/*
api_key: os.environ/OPENAI_API_KEY
model_info:
access_groups: ["default-models"]
- model_name: openai/o1-*
litellm_params:
model: openai/o1-*
api_key: os.environ/OPENAI_API_KEY
model_info:
access_groups: ["restricted-models"]
```
2. Generate a key with access to `default-models`
```bash showLineNumbers title="Generate Key for Wildcard Access Group"
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"models": ["default-models"],
}'
```
3. Test the key
<Tabs>
<TabItem label="Successful Request" value = "success">
```bash showLineNumbers title="Test Wildcard Access - Allowed"
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "openai/gpt-4",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
<TabItem value="bad-request" label="Rejected Request">
```bash showLineNumbers title="Test Wildcard Access - Rejected"
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-<key-from-previous-step>" \
-d '{
"model": "openai/o1-mini",
"messages": [
{"role": "user", "content": "Hello"}
]
}'
```
</TabItem>
</Tabs>
## Managing Access Groups via API
:::warning Database Models Only
Access group management APIs only work with models stored in the database (added via `/model/new`).
Models defined in `config.yaml` cannot be managed through these APIs and must be configured directly in the config file.
:::
Use the access group management endpoints to dynamically create, update, and delete access groups without restarting the proxy.
### Tutorial: Complete Access Group Workflow
This tutorial shows how to create an access group, view its details, attach it to a key, and update the models in the group.
**Prerequisites:**
- Models must be added to the database first (not just in config.yaml)
- You need your master key for authorization
#### Step 1: Add Models to Database
First, add some models to the database:
```bash showLineNumbers title="Add Models to Database"
# Add GPT-4 to database
curl -X POST 'http://localhost:4000/model/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model_name": "gpt-4",
"litellm_params": {
"model": "gpt-4",
"api_key": "os.environ/OPENAI_API_KEY"
}
}'
# Add Claude to database
curl -X POST 'http://localhost:4000/model/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model_name": "claude-3-opus",
"litellm_params": {
"model": "claude-3-opus-20240229",
"api_key": "os.environ/ANTHROPIC_API_KEY"
}
}'
```
#### Step 2: Create Access Group
Create an access group containing multiple models:
```bash showLineNumbers title="Create Access Group"
curl -X POST 'http://localhost:4000/access_group/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"access_group": "production-models",
"model_names": ["gpt-4", "claude-3-opus"]
}'
```
**Response:**
```json showLineNumbers title="Response"
{
"access_group": "production-models",
"model_names": ["gpt-4", "claude-3-opus"],
"models_updated": 2
}
```
#### Step 3: View Access Group Info
Check the access group details:
```bash showLineNumbers title="Get Access Group Info"
curl -X GET 'http://localhost:4000/access_group/production-models/info' \
-H 'Authorization: Bearer sk-1234'
```
**Response:**
```json showLineNumbers title="Response"
{
"access_group": "production-models",
"model_names": ["gpt-4", "claude-3-opus"],
"deployment_count": 2
}
```
#### Step 4: Create Key with Access Group
Create an API key that can access all models in the group:
```bash showLineNumbers title="Create Key with Access Group"
curl -X POST 'http://localhost:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"models": ["production-models"],
"max_budget": 100
}'
```
**Response:**
```json showLineNumbers title="Response"
{
"key": "sk-...",
"models": ["production-models"]
}
```
**Test the key:**
```bash showLineNumbers title="Test Key Access"
# This succeeds - gpt-4 is in production-models
curl -X POST 'http://localhost:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-...' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
# This succeeds - claude-3-opus is in production-models
curl -X POST 'http://localhost:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-...' \
-H 'Content-Type: application/json' \
-d '{
"model": "claude-3-opus",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
#### Step 5: Update Access Group
Add or remove models from the access group:
```bash showLineNumbers title="Update Access Group"
curl -X PUT 'http://localhost:4000/access_group/production-models/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model_names": ["gpt-4", "claude-3-opus", "gemini-pro"]
}'
```
**Response:**
```json showLineNumbers title="Response"
{
"access_group": "production-models",
"model_names": ["gpt-4", "claude-3-opus", "gemini-pro"],
"models_updated": 3
}
```
The API key from Step 4 now automatically has access to `gemini-pro` without any changes to the key itself.
### API Reference - Access Group Management
For complete API documentation including all endpoints, parameters, and response schemas, see the [Access Group Management API Reference](https://litellm-api.up.railway.app/#/model%20management/create_model_group_access_group_new_post).
## Managing Access Groups via UI
You can also manage access groups through the LiteLLM Admin UI.
### Step 1: Add Model to Access Group
When adding a model to the database, assign it to an access group using the "Model Access Group" field:
![Add Model with Access Group](../../img/add_model_access.png)
In this example, `gpt-4` is added to the `production-models` access group.
### Step 2: Create Key with Access Group
When creating an API key, specify the access group in the "Models" field:
![Create Key with Access Group](../../img/add_model_key.png)
The key will have access to all models in the `production-models` group.
### Step 3: Test the Key
Use the generated key to make requests:
```bash showLineNumbers title="Test Key with Access Group"
# This succeeds - gpt-4 is in production-models
curl -X POST 'http://localhost:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-...' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
**Response:**
```json showLineNumbers title="Success Response"
{
"id": "chatcmpl-...",
"object": "chat.completion",
"created": 1234567890,
"model": "gpt-4",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Hello! How can I help you today?"
},
"finish_reason": "stop"
}
]
}
```
If you try to access a model not in the access group, the request will be rejected:
```bash showLineNumbers title="Test Rejected Request"
# This fails - gpt-4o is not in production-models
curl -X POST 'http://localhost:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-...' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
**Response:**
```json showLineNumbers title="Error Response"
{
"error": {
"message": "Invalid model for key",
"type": "invalid_request_error"
}
}
```
docs/my-website/docs/proxy/model_access_guide.md
+6 -1
View File
@@ -85,4 +85,9 @@ litellm_settings:
fallbacks: [{"my-custom-model": ["my-other-model"]}]
```
Fallbacks are done sequentially, so the first model group in the list will be tried first. If it fails, the next model group will be tried.
Fallbacks are done sequentially, so the first model group in the list will be tried first. If it fails, the next model group will be tried.
## Advanced: Model Access Groups
For advanced use cases, use [Model Access Groups](./model_access_groups) to dynamically group multiple models and manage access without restarting the proxy.
docs/my-website/docs/proxy/sync_models_github.md
+16 -3
View File
@@ -1,8 +1,21 @@
# Syncing Models to GitHub model_context_window
# Auto Sync New Models (Day-0 Launches)
Sync model pricing data from GitHub's `model_prices_and_context_window.json` file outside of the LiteLLM UI.
Automatically keep your model pricing and context window data up to date without restarting your service. **This allows you to add day-0 support for new models without restarting your service.**
> **📹 Video Tutorial**: [Watch how to sync models via the Admin UI](https://www.loom.com/share/ba41acc1882d41b284bbddbb0e9c27ce?sid=bdae351e-2026-4e39-932b-fcb185ff612c)
## Overview
When providers like OpenAI or Anthropic release new models (e.g., GPT-5, Claude 4), you typically need to restart your LiteLLM service to get the latest pricing and context window data.
With auto-sync, LiteLLM automatically pulls the latest model data from GitHub's [`model_prices_and_context_window.json`](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) without requiring a restart. This means:
- **Zero downtime** when new models are released
- **Always accurate pricing** for cost tracking and budgets
- **Automatic updates** - set it once and forget it
<iframe width="840" height="500" src="https://www.loom.com/embed/ba41acc1882d41b284bbddbb0e9c27ce?sid=bdae351e-2026-4e39-932b-fcb185ff612c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
<br/>
<br/>
## Quick Start
docs/my-website/img/add_model_access.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 576 KiB

docs/my-website/img/add_model_key.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 514 KiB

docs/my-website/package-lock.json
Generated
+4 -40
View File
@@ -10296,18 +10296,6 @@
"node": ">=8.0.0"
}
},
"node_modules/esprima": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
"bin": {
"esparse": "bin/esparse.js",
"esvalidate": "bin/esvalidate.js"
},
"engines": {
"node": ">=4"
}
},
"node_modules/esrecurse": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
@@ -11092,26 +11080,6 @@
"node": ">=6.0"
}
},
"node_modules/gray-matter/node_modules/argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"dependencies": {
"sprintf-js": "~1.0.2"
}
},
"node_modules/gray-matter/node_modules/js-yaml": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
"integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
"dependencies": {
"argparse": "^1.0.7",
"esprima": "^4.0.0"
},
"bin": {
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/hachure-fill": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz",
@@ -12148,9 +12116,10 @@
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="
},
"node_modules/js-yaml": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
"license": "MIT",
"dependencies": {
"argparse": "^2.0.1"
},
@@ -19035,11 +19004,6 @@
"node": ">= 6"
}
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="
},
"node_modules/srcset": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/srcset/-/srcset-4.0.0.tgz",
docs/my-website/package.json
+2 -1
View File
@@ -50,6 +50,7 @@
"overrides": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0"
"mermaid": ">=11.10.0",
"js-yaml": ">=4.1.1"
}
}
docs/my-website/sidebars.js
+10 -2
View File
@@ -31,9 +31,16 @@ const sidebars = {
label: "Guardrails",
items: [
"proxy/guardrails/quick_start",
{
type: "category",
"label": "Contributing to Guardrails",
items: [
"adding_provider/simple_guardrail_tutorial",
"adding_provider/adding_guardrail_support",
]
},
"proxy/guardrails/test_playground",
...[
"adding_provider/adding_guardrail_support",
"proxy/guardrails/aim_security",
"proxy/guardrails/aporia_api",
"proxy/guardrails/azure_content_guardrail",
@@ -252,6 +259,7 @@ const sidebars = {
items: [
"proxy/model_access_guide",
"proxy/model_access",
"proxy/model_access_groups",
"proxy/team_model_add"
]
},
@@ -277,6 +285,7 @@ const sidebars = {
items: [
"proxy/cost_tracking",
"proxy/custom_pricing",
"proxy/sync_models_github",
"proxy/billing",
],
},
@@ -488,7 +497,6 @@ const sidebars = {
"providers/vertex_ai/videos",
"providers/vertex_partner",
"providers/vertex_self_deployed",
"providers/vertex_embedding",
"providers/vertex_image",
"providers/vertex_batch",
"providers/vertex_ocr",
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.4-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.4.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20251114173537_add_request_id_to_daily_tag_spend/migration.sql
+3
View File
@@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN "request_id" TEXT;
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.3"
version = "0.4.4"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.3"
version = "0.4.4"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+11 -1
View File
@@ -485,6 +485,7 @@ vertex_ai_ai21_models: Set = set()
vertex_mistral_models: Set = set()
vertex_openai_models: Set = set()
vertex_minimax_models: Set = set()
vertex_moonshot_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
@@ -500,6 +501,7 @@ watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
deepseek_models: Set = set()
runwayml_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
@@ -648,6 +650,9 @@ def add_known_models():
elif value.get("litellm_provider") == "vertex_ai-minimax_models":
key = key.replace("vertex_ai/", "")
vertex_minimax_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-moonshot_models":
key = key.replace("vertex_ai/", "")
vertex_moonshot_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.add(key)
@@ -687,6 +692,8 @@ def add_known_models():
fal_ai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.add(key)
elif value.get("litellm_provider") == "runwayml":
runwayml_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
llama_models.add(key)
elif value.get("litellm_provider") == "nscale":
@@ -830,6 +837,7 @@ model_list = list(
| deepinfra_models
| perplexity_models
| set(maritalk_models)
| runwayml_models
| vertex_language_models
| watsonx_models
| gemini_models
@@ -904,7 +912,8 @@ models_by_provider: dict = {
| vertex_vision_models
| vertex_language_models
| vertex_deepseek_models
| vertex_minimax_models,
| vertex_minimax_models
| vertex_moonshot_models,
"ai21": ai21_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
@@ -921,6 +930,7 @@ models_by_provider: dict = {
"xai": xai_models,
"fal_ai": fal_ai_models,
"deepseek": deepseek_models,
"runwayml": runwayml_models,
"mistral": mistral_chat_models,
"azure_ai": azure_ai_models,
"voyage": voyage_models,
litellm/completion_extras/litellm_responses_transformation/transformation.py
+3 -1
View File
@@ -544,7 +544,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return Reasoning(**reasoning_effort) # type: ignore[typeddict-item]
# If string is passed, map without summary (default)
if reasoning_effort == "high":
if reasoning_effort == "none":
return Reasoning(effort="none") # type: ignore
elif reasoning_effort == "high":
return Reasoning(effort="high")
elif reasoning_effort == "medium":
return Reasoning(effort="medium")
litellm/images/main.py
+3
View File
@@ -400,6 +400,8 @@ def image_generation( # noqa: PLR0915
or custom_llm_provider == LlmProviders.LITELLM_PROXY.value
or custom_llm_provider in litellm.openai_compatible_providers
):
# Forward OpenAI organization if present (set by proxy pre-call utils)
organization: Optional[str] = kwargs.get("organization", None)
model_response = openai_chat_completions.image_generation(
model=model,
prompt=prompt,
@@ -409,6 +411,7 @@ def image_generation( # noqa: PLR0915
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
organization=organization,
aimg_generation=aimg_generation,
client=client,
)
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
+36 -46
View File
@@ -152,32 +152,27 @@ class LiteLLMMessagesToCompletionTransformationHandler:
)
)
try:
completion_response = await litellm.acompletion(**completion_kwargs)
completion_response = await litellm.acompletion(**completion_kwargs)
if stream:
transformed_stream = (
ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
)
if stream:
transformed_stream = (
ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
)
if transformed_stream is not None:
return transformed_stream
raise ValueError("Failed to transform streaming response")
else:
anthropic_response = (
ANTHROPIC_ADAPTER.translate_completion_output_params(
cast(ModelResponse, completion_response)
)
)
if anthropic_response is not None:
return anthropic_response
raise ValueError("Failed to transform response to Anthropic format")
except Exception as e: # noqa: BLE001
raise ValueError(
f"Error calling litellm.acompletion for non-Anthropic model: {str(e)}"
)
if transformed_stream is not None:
return transformed_stream
raise ValueError("Failed to transform streaming response")
else:
anthropic_response = (
ANTHROPIC_ADAPTER.translate_completion_output_params(
cast(ModelResponse, completion_response)
)
)
if anthropic_response is not None:
return anthropic_response
raise ValueError("Failed to transform response to Anthropic format")
@staticmethod
def anthropic_messages_handler(
@@ -239,29 +234,24 @@ class LiteLLMMessagesToCompletionTransformationHandler:
)
)
try:
completion_response = litellm.completion(**completion_kwargs)
completion_response = litellm.completion(**completion_kwargs)
if stream:
transformed_stream = (
ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
)
if stream:
transformed_stream = (
ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
)
if transformed_stream is not None:
return transformed_stream
raise ValueError("Failed to transform streaming response")
else:
anthropic_response = (
ANTHROPIC_ADAPTER.translate_completion_output_params(
cast(ModelResponse, completion_response)
)
)
if anthropic_response is not None:
return anthropic_response
raise ValueError("Failed to transform response to Anthropic format")
except Exception as e: # noqa: BLE001
raise ValueError(
f"Error calling litellm.completion for non-Anthropic model: {str(e)}"
)
if transformed_stream is not None:
return transformed_stream
raise ValueError("Failed to transform streaming response")
else:
anthropic_response = (
ANTHROPIC_ADAPTER.translate_completion_output_params(
cast(ModelResponse, completion_response)
)
)
if anthropic_response is not None:
return anthropic_response
raise ValueError("Failed to transform response to Anthropic format")
litellm/llms/bedrock/batches/transformation.py
+14 -3
View File
@@ -6,6 +6,7 @@ from httpx import Headers, Response
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.bedrock import (
BedrockCreateBatchRequest,
BedrockCreateBatchResponse,
@@ -140,10 +141,20 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
}
# Build output data config
s3_output_config: BedrockS3OutputDataConfig = BedrockS3OutputDataConfig(
s3Uri=f"s3://{output_bucket}/{output_key}"
)
# Add optional KMS encryption key ID if provided
s3_encryption_key_id = (
litellm_params.get("s3_encryption_key_id")
or get_secret_str("AWS_S3_ENCRYPTION_KEY_ID")
)
if s3_encryption_key_id:
s3_output_config["s3EncryptionKeyId"] = s3_encryption_key_id
output_data_config: BedrockOutputDataConfig = {
"s3OutputDataConfig": BedrockS3OutputDataConfig(
s3Uri=f"s3://{output_bucket}/{output_key}"
)
"s3OutputDataConfig": s3_output_config
}
# Create Bedrock batch request with proper typing
litellm/llms/bedrock/chat/agentcore/sse_iterator.py
+136 -6
View File
@@ -19,21 +19,151 @@ if TYPE_CHECKING:
class AgentCoreSSEStreamIterator:
"""Async iterator for AgentCore SSE streaming responses."""
"""Iterator for AgentCore SSE streaming responses. Supports both sync and async iteration."""
def __init__(self, response: httpx.Response, model: str):
self.response = response
self.model = model
self.finished = False
self.line_iterator = self.response.aiter_lines()
self.line_iterator = None
self.async_line_iterator = None
def __aiter__(self):
def __iter__(self):
"""Initialize sync iteration."""
self.line_iterator = self.response.iter_lines()
return self
async def __anext__(self) -> ModelResponse:
"""Parse SSE events and yield ModelResponse chunks."""
def __aiter__(self):
"""Initialize async iteration."""
self.async_line_iterator = self.response.aiter_lines()
return self
def __next__(self) -> ModelResponse:
"""Sync iteration - parse SSE events and yield ModelResponse chunks."""
try:
async for line in self.line_iterator:
if self.line_iterator is None:
raise StopIteration
for line in self.line_iterator:
line = line.strip()
if not line or not line.startswith('data:'):
continue
# Extract JSON from SSE line
json_str = line[5:].strip()
if not json_str:
continue
try:
data = json.loads(json_str)
# Skip non-dict data
if not isinstance(data, dict):
continue
# Process content delta events
if "event" in data and isinstance(data["event"], dict):
event_payload = data["event"]
content_block_delta = event_payload.get("contentBlockDelta")
if content_block_delta:
delta = content_block_delta.get("delta", {})
text = delta.get("text", "")
if text:
# Yield chunk with text
chunk = ModelResponse(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=self.model,
object="chat.completion.chunk",
)
chunk.choices = [
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content=text, role="assistant"),
)
]
return chunk
# Check for metadata/usage
metadata = event_payload.get("metadata")
if metadata and "usage" in metadata:
# This is the final chunk with usage
chunk = ModelResponse(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=self.model,
object="chat.completion.chunk",
)
chunk.choices = [
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(),
)
]
usage_data: AgentCoreUsage = metadata["usage"] # type: ignore
setattr(chunk, "usage", Usage(
prompt_tokens=usage_data.get("inputTokens", 0),
completion_tokens=usage_data.get("outputTokens", 0),
total_tokens=usage_data.get("totalTokens", 0),
))
self.finished = True
return chunk
# Check for final message (alternative finish signal)
if "message" in data and isinstance(data["message"], dict):
if not self.finished:
chunk = ModelResponse(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=self.model,
object="chat.completion.chunk",
)
chunk.choices = [
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(),
)
]
self.finished = True
return chunk
except json.JSONDecodeError:
verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
continue
# Stream ended naturally
raise StopIteration
except StopIteration:
raise
except httpx.StreamConsumed:
# This is expected when the stream has been fully consumed
raise StopIteration
except httpx.StreamClosed:
# This is expected when the stream is closed
raise StopIteration
except Exception as e:
verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}")
raise StopIteration
async def __anext__(self) -> ModelResponse:
"""Async iteration - parse SSE events and yield ModelResponse chunks."""
try:
if self.async_line_iterator is None:
raise StopAsyncIteration
async for line in self.async_line_iterator:
line = line.strip()
if not line or not line.startswith('data:'):
litellm/llms/fal_ai/__init__.py
+2
View File
@@ -3,6 +3,7 @@ from .image_generation import (
FalAIBaseConfig,
FalAIBriaConfig,
FalAIFluxProV11UltraConfig,
FalAIFluxSchnellConfig,
FalAIImageGenerationConfig,
FalAIImagen4Config,
FalAIRecraftV3Config,
@@ -18,6 +19,7 @@ __all__ = [
"FalAIRecraftV3Config",
"FalAIBriaConfig",
"FalAIFluxProV11UltraConfig",
"FalAIFluxSchnellConfig",
"FalAIStableDiffusionConfig",
"get_fal_ai_image_generation_config",
]
litellm/llms/fal_ai/image_generation/__init__.py
+4
View File
@@ -4,6 +4,7 @@ from litellm.llms.base_llm.image_generation.transformation import (
from .bria_transformation import FalAIBriaConfig
from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig
from .flux_schnell_transformation import FalAIFluxSchnellConfig
from .imagen4_transformation import FalAIImagen4Config
from .recraft_v3_transformation import FalAIRecraftV3Config
from .stable_diffusion_transformation import FalAIStableDiffusionConfig
@@ -16,6 +17,7 @@ __all__ = [
"FalAIRecraftV3Config",
"FalAIBriaConfig",
"FalAIFluxProV11UltraConfig",
"FalAIFluxSchnellConfig",
"FalAIStableDiffusionConfig",
]
@@ -41,6 +43,8 @@ def get_fal_ai_image_generation_config(model: str) -> BaseImageGenerationConfig:
return FalAIBriaConfig()
elif "flux-pro" in model_lower and "ultra" in model_lower:
return FalAIFluxProV11UltraConfig()
elif "flux/schnell" in model_lower or "flux-schnell" in model_lower or "schnell" in model_lower:
return FalAIFluxSchnellConfig()
elif "stable-diffusion" in model_lower:
return FalAIStableDiffusionConfig()
litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py
+88
View File
@@ -0,0 +1,88 @@
from typing import Any
from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig
class FalAIFluxSchnellConfig(FalAIFluxProV11UltraConfig):
"""
Configuration for Fal AI Flux Schnell model.
Flux Schnell shares the same response format as Flux Pro models but expects
the OpenAI `size` parameter to be translated into Fal AI's `image_size`
enum/object.
Model endpoint: fal-ai/flux/schnell
Documentation: https://fal.ai/models/fal-ai/flux/schnell
"""
IMAGE_GENERATION_ENDPOINT: str = "fal-ai/flux/schnell"
_OPENAI_SIZE_TO_IMAGE_SIZE = {
"1024x1024": "square_hd",
"512x512": "square",
"1792x1024": "landscape_16_9",
"1024x1792": "portrait_16_9",
"1024x768": "landscape_4_3",
"768x1024": "portrait_4_3",
}
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
param_mapping = {
"n": "num_images",
"response_format": "output_format",
"size": "image_size",
}
for k in non_default_params.keys():
if k not in optional_params.keys():
if k in supported_params:
mapped_key = param_mapping.get(k, k)
mapped_value = non_default_params[k]
if k == "response_format":
if mapped_value in ["b64_json", "url"]:
mapped_value = "jpeg"
elif k == "size":
mapped_value = self._map_image_size(mapped_value)
optional_params[mapped_key] = mapped_value
elif drop_params:
continue
else:
raise ValueError(
f"Parameter {k} is not supported for model {model}. "
f"Supported parameters are {supported_params}. "
"Set drop_params=True to drop unsupported parameters."
)
return optional_params
def _map_image_size(self, size: Any) -> Any:
if isinstance(size, dict):
return size
if not isinstance(size, str):
return size
if size in self._OPENAI_SIZE_TO_IMAGE_SIZE:
return self._OPENAI_SIZE_TO_IMAGE_SIZE[size]
if "x" in size:
try:
width_str, height_str = size.split("x")
width = int(width_str)
height = int(height_str)
return {"width": width, "height": height}
except (ValueError, AttributeError, ZeroDivisionError):
pass
return "landscape_4_3"
litellm/llms/fal_ai/image_generation/imagen4_transformation.py
+1 -1
View File
@@ -23,7 +23,7 @@ class FalAIImagen4Config(FalAIBaseConfig):
Model variants:
- fal-ai/imagen4/preview (Standard): $0.05 per image
- fal-ai/imagen4/preview/fast (Fast): $0.04 per image
- fal-ai/imagen4/preview/fast (Fast): $0.02 per image
- fal-ai/imagen4/preview/ultra (Ultra): $0.06 per image
Documentation: https://fal.ai/models/fal-ai/imagen4/preview
litellm/llms/groq/chat/transformation.py
+53 -8
View File
@@ -1,9 +1,27 @@
"""
Translate from OpenAI's `/v1/chat/completions` to Groq's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
from typing import (
Any,
Coroutine,
List,
Literal,
Optional,
Tuple,
Union,
cast,
overload,
Iterator,
AsyncIterator,
)
import httpx
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
)
from litellm.llms.openai.common_utils import OpenAIError
from pydantic import BaseModel
import litellm
@@ -16,7 +34,7 @@ from litellm.types.llms.openai import (
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
)
from litellm.types.utils import ModelResponse
from litellm.types.utils import ModelResponse, ModelResponseStream
from ...openai_like.chat.transformation import OpenAILikeChatConfig
@@ -65,6 +83,18 @@ class GroqChatConfig(OpenAILikeChatConfig):
def get_config(cls):
return super().get_config()
def get_model_response_iterator(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
sync_stream: bool,
json_mode: Optional[bool] = False,
) -> Any:
return GroqChatCompletionStreamingHandler(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def get_supported_openai_params(self, model: str) -> list:
base_params = super().get_supported_openai_params(model)
try:
@@ -209,7 +239,6 @@ class GroqChatConfig(OpenAILikeChatConfig):
)
return optional_params
def transform_response(
self,
@@ -239,12 +268,17 @@ class GroqChatConfig(OpenAILikeChatConfig):
json_mode=json_mode,
)
mapped_service_tier: Literal["auto", "default", "flex"] = self._map_groq_service_tier(original_service_tier=getattr(model_response, "service_tier"))
mapped_service_tier: Literal[
"auto", "default", "flex"
] = self._map_groq_service_tier(
original_service_tier=getattr(model_response, "service_tier")
)
setattr(model_response, "service_tier", mapped_service_tier)
return model_response
def _map_groq_service_tier(self, original_service_tier: Optional[str]) -> Literal["auto", "default", "flex"]:
def _map_groq_service_tier(
self, original_service_tier: Optional[str]
) -> Literal["auto", "default", "flex"]:
"""
Ensure groq service tier is OpenAI compatible.
"""
@@ -252,5 +286,16 @@ class GroqChatConfig(OpenAILikeChatConfig):
return "auto"
if original_service_tier not in ["auto", "default", "flex"]:
return "auto"
return cast(Literal["auto", "default", "flex"], original_service_tier)
return cast(Literal["auto", "default", "flex"], original_service_tier)
class GroqChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
error = chunk.get("error")
if error:
raise OpenAIError(
status_code=error.get("code"), message=error.get("message"), body=error
)
return super().chunk_parser(chunk)
litellm/llms/openai/openai.py
+5 -1
View File
@@ -1285,6 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
api_base: Optional[str] = None,
client=None,
max_retries=None,
organization: Optional[str] = None,
):
response = None
try:
@@ -1294,6 +1295,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
client=client,
)
@@ -1328,6 +1330,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
model_response: Optional[ImageResponse] = None,
client=None,
aimg_generation=None,
organization: Optional[str] = None,
) -> ImageResponse:
data = {}
try:
@@ -1337,7 +1340,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(status_code=422, message="max retries must be an int")
if aimg_generation is True:
return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries) # type: ignore
return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization) # type: ignore
openai_client: OpenAI = self._get_openai_client( # type: ignore
is_async=False,
@@ -1345,6 +1348,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
client=client,
)
litellm/llms/sambanova/chat.py
+32 -1
View File
@@ -4,9 +4,13 @@ Sambanova Chat Completions API
this is OpenAI compatible - no translation needed / occurs
"""
from typing import Optional, Union
from typing import Any, Coroutine, List, Literal, Optional, Union, overload
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.openai import AllMessageValues
class SambanovaConfig(OpenAIGPTConfig):
@@ -92,3 +96,30 @@ class SambanovaConfig(OpenAIGPTConfig):
elif param in supported_openai_params:
optional_params[param] = value
return optional_params
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
def _transform_messages(
self,
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
...
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
"""
Transform messages to handle content list conversion.
SambaNova API doesn't support content as a list - only string content.
This converts content lists like [{"type": "text", "text": "..."}] to strings.
"""
messages = handle_messages_with_content_list_to_str_conversion(messages)
return messages
litellm/llms/vertex_ai/batches/handler.py
-8
View File
@@ -61,10 +61,6 @@ class VertexAIBatchPrediction(VertexLLM):
stream=None,
auth_header=None,
url=default_api_base,
model=None,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1",
)
headers = {
@@ -170,10 +166,6 @@ class VertexAIBatchPrediction(VertexLLM):
stream=None,
auth_header=None,
url=default_api_base,
model=None,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1",
)
headers = {
litellm/llms/vertex_ai/common_utils.py
+7 -88
View File
@@ -31,11 +31,9 @@ class VertexAIModelRoute(str, Enum):
PARTNER_MODELS = "partner_models"
GEMINI = "gemini"
GEMMA = "gemma"
BGE = "bge"
MODEL_GARDEN = "model_garden"
NON_GEMINI = "non_gemini"
VERTEX_AI_MODEL_ROUTES = [f"{route.value}/" for route in VertexAIModelRoute]
def get_vertex_ai_model_route(
model: str, litellm_params: Optional[dict] = None
@@ -62,9 +60,6 @@ def get_vertex_ai_model_route(
>>> get_vertex_ai_model_route("openai/gpt-oss-120b")
VertexAIModelRoute.MODEL_GARDEN
>>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"})
VertexAIModelRoute.GEMINI # Numeric endpoints with api_base use HTTP path
"""
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
VertexAIPartnerModels,
@@ -74,20 +69,11 @@ def get_vertex_ai_model_route(
if litellm_params and litellm_params.get("base_model") is not None:
if "gemini" in litellm_params["base_model"]:
return VertexAIModelRoute.GEMINI
# Check if numeric endpoint ID with custom api_base (PSC endpoint)
# Route to GEMINI (HTTP path) to support PSC endpoints properly
if model.isdigit() and litellm_params and litellm_params.get("api_base"):
return VertexAIModelRoute.GEMINI
# Check for partner models (llama, mistral, claude, etc.)
if VertexAIPartnerModels.is_vertex_partner_model(model=model):
return VertexAIModelRoute.PARTNER_MODELS
# Check for BGE models
if "bge/" in model or "bge" in model.lower():
return VertexAIModelRoute.BGE
# Check for gemma models
if "gemma/" in model:
return VertexAIModelRoute.GEMMA
@@ -150,71 +136,6 @@ all_gemini_url_modes = Literal[
]
def get_vertex_base_model_name(model: str) -> str:
"""
Strip routing prefixes from model name for PSC/endpoint URL construction.
Patterns like "bge/", "gemma/", "openai/" are used for internal routing but
should not appear in the actual endpoint URL. Routing prefixes are derived
from VertexAIModelRoute enum values.
Args:
model: The model name with potential prefix (e.g., "bge/123456", "gemma/gemma-3-12b-it")
Returns:
str: The model name without routing prefix (e.g., "123456", "gemma-3-12b-it")
Examples:
>>> get_vertex_base_model_name("bge/378943383978115072")
"378943383978115072"
>>> get_vertex_base_model_name("gemma/gemma-3-12b-it")
"gemma-3-12b-it"
>>> get_vertex_base_model_name("openai/gpt-oss-120b")
"gpt-oss-120b"
>>> get_vertex_base_model_name("1234567890")
"1234567890"
"""
# Derive routing prefixes from VertexAIModelRoute enum
# Map specific routes to their prefixes (some routes like PARTNER_MODELS, GEMINI don't have prefixes)
for route in VERTEX_AI_MODEL_ROUTES:
if model.startswith(route):
return model.replace(route, "", 1)
return model
def _get_embedding_url(
model: str,
vertex_project: Optional[str],
vertex_location: Optional[str],
vertex_api_version: Literal["v1", "v1beta1"],
) -> Tuple[str, str]:
"""
Get URL for embedding models.
Handles special patterns:
- bge/endpoint_id -> strips to endpoint_id for endpoints/ routing
- numeric model -> routes to endpoints/
- regular model -> routes to publishers/google/models/
"""
endpoint = "predict"
# Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction
model = get_vertex_base_model_name(model=model)
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
if model.isdigit():
# https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
return url, endpoint
def _get_vertex_url(
mode: all_gemini_url_modes,
model: str,
@@ -227,7 +148,6 @@ def _get_vertex_url(
endpoint: Optional[str] = None
model = litellm.VertexGeminiConfig.get_model_for_vertex_ai_url(model=model)
if mode == "chat":
### SET RUNTIME ENDPOINT ###
endpoint = "generateContent"
@@ -252,12 +172,11 @@ def _get_vertex_url(
if stream is True:
url += "?alt=sse"
elif mode == "embedding":
return _get_embedding_url(
model=model,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_api_version=vertex_api_version,
)
endpoint = "predict"
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
if model.isdigit():
# https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
elif mode == "image_generation":
endpoint = "predict"
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py
-4
View File
@@ -79,10 +79,6 @@ class ContextCachingEndpoints(VertexBase):
stream=None,
auth_header=auth_header,
url=url,
model=None,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_api_version="v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1",
)
def check_cache(
litellm/llms/vertex_ai/vertex_ai_partner_models/main.py
+3
View File
@@ -39,6 +39,7 @@ class PartnerModelPrefixes(str, Enum):
QWEN_PREFIX = "qwen"
GPT_OSS_PREFIX = "openai/gpt-oss-"
MINIMAX_PREFIX = "minimaxai/"
MOONSHOT_PREFIX = "moonshotai/"
class VertexAIPartnerModels(VertexBase):
@@ -64,6 +65,7 @@ class VertexAIPartnerModels(VertexBase):
or model.startswith(PartnerModelPrefixes.QWEN_PREFIX)
or model.startswith(PartnerModelPrefixes.GPT_OSS_PREFIX)
or model.startswith(PartnerModelPrefixes.MINIMAX_PREFIX)
or model.startswith(PartnerModelPrefixes.MOONSHOT_PREFIX)
):
return True
return False
@@ -76,6 +78,7 @@ class VertexAIPartnerModels(VertexBase):
PartnerModelPrefixes.QWEN_PREFIX,
PartnerModelPrefixes.GPT_OSS_PREFIX,
PartnerModelPrefixes.MINIMAX_PREFIX,
PartnerModelPrefixes.MOONSHOT_PREFIX,
]
if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS):
return True
litellm/llms/vertex_ai/vertex_embeddings/bge.py
-182
View File
@@ -1,182 +0,0 @@
"""
Vertex AI BGE (BAAI General Embedding) Configuration
BGE models deployed on Vertex AI require different input/output format:
- Request: Use "prompt" instead of "content" as the input field
- Response: Embeddings are returned directly as arrays, not wrapped in objects
Model name handling:
- Model names like "bge/endpoint_id" are automatically transformed in common_utils._get_vertex_url()
- This module focuses on request/response transformation only
"""
from typing import List, Optional, Union
from litellm.types.utils import EmbeddingResponse, Usage
from .types import (
EmbeddingParameters,
TaskType,
TextEmbeddingBGEInput,
VertexEmbeddingRequest,
)
class VertexBGEConfig:
"""
Configuration and transformation logic for BGE models on Vertex AI.
BGE (BAAI General Embedding) models use a different request format
where the input field is named "prompt" instead of "content".
Supported model patterns (after provider split in main.py):
- "bge-small-en-v1.5" (model name)
- "bge/204379420394258432" (endpoint ID pattern)
Note: Model name transformation (bge/ -> numeric ID) is handled automatically
in common_utils._get_vertex_url(). This class focuses on request/response format only.
"""
@staticmethod
def is_bge_model(model: str) -> bool:
"""
Check if the model is a BGE (BAAI General Embedding) model.
After provider split in main.py, supports:
- "bge-small-en-v1.5" (model name)
- "bge/204379420394258432" (endpoint ID pattern)
Args:
model: The model name after provider split
Returns:
bool: True if the model is a BGE model
"""
model_lower = model.lower()
# Check for "bge/" prefix (endpoint pattern) or "bge" in model name
return model_lower.startswith("bge/") or "bge" in model_lower
@staticmethod
def transform_request(
input: Union[list, str], optional_params: dict, model: str
) -> VertexEmbeddingRequest:
"""
Transforms an OpenAI request to a Vertex BGE embedding request.
BGE models use "prompt" instead of "content" as the input field.
Args:
input: The input text(s) to embed
optional_params: Optional parameters for the request
model: The model name
Returns:
VertexEmbeddingRequest: The transformed request
"""
vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest()
vertex_text_embedding_input_list: List[TextEmbeddingBGEInput] = []
task_type: Optional[TaskType] = optional_params.get("task_type")
title = optional_params.get("title")
if isinstance(input, str):
input = [input]
for text in input:
embedding_input = VertexBGEConfig._create_embedding_input(
prompt=text, task_type=task_type, title=title
)
vertex_text_embedding_input_list.append(embedding_input)
vertex_request["instances"] = vertex_text_embedding_input_list
vertex_request["parameters"] = EmbeddingParameters(**optional_params)
return vertex_request
@staticmethod
def _create_embedding_input(
prompt: str,
task_type: Optional[TaskType] = None,
title: Optional[str] = None,
) -> TextEmbeddingBGEInput:
"""
Creates a TextEmbeddingBGEInput object for BGE models.
BGE models use "prompt" instead of "content" as the input field.
Args:
prompt: The prompt to be embedded
task_type: The type of task to be performed
title: The title of the document to be embedded
Returns:
TextEmbeddingBGEInput: A TextEmbeddingBGEInput object
"""
text_embedding_input = TextEmbeddingBGEInput(prompt=prompt)
if task_type is not None:
text_embedding_input["task_type"] = task_type
if title is not None:
text_embedding_input["title"] = title
return text_embedding_input
@staticmethod
def transform_response(
response: dict, model: str, model_response: EmbeddingResponse
) -> EmbeddingResponse:
"""
Transforms a Vertex BGE embedding response to OpenAI format.
BGE models return embeddings directly as arrays in predictions:
{
"predictions": [
[0.002, 0.021, ...],
[0.003, 0.022, ...]
]
}
Args:
response: The raw response from Vertex AI
model: The model name
model_response: The EmbeddingResponse object to populate
Returns:
EmbeddingResponse: The transformed response in OpenAI format
Raises:
KeyError: If response doesn't contain 'predictions'
ValueError: If predictions is not a list or contains invalid data
"""
if "predictions" not in response:
raise KeyError("Response missing 'predictions' field")
_predictions = response["predictions"]
if not isinstance(_predictions, list):
raise ValueError(f"Expected 'predictions' to be a list, got {type(_predictions)}")
embedding_response = []
# BGE models don't return token counts, so we estimate or set to 0
input_tokens = 0
for idx, embedding_values in enumerate(_predictions):
if not isinstance(embedding_values, list):
raise ValueError(
f"Expected embedding at index {idx} to be a list, got {type(embedding_values)}"
)
embedding_response.append(
{
"object": "embedding",
"index": idx,
"embedding": embedding_values,
}
)
model_response.object = "list"
model_response.data = embedding_response
model_response.model = model
usage = Usage(
prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens
)
setattr(model_response, "usage", usage)
return model_response
litellm/llms/vertex_ai/vertex_embeddings/transformation.py
+2 -19
View File
@@ -105,16 +105,10 @@ class VertexAITextEmbeddingConfig(BaseModel):
"""
Transforms an openai request to a vertex embedding request.
"""
# Import here to avoid circular import issues with litellm.__init__
from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig
if model.isdigit():
return self._transform_openai_request_to_fine_tuned_embedding_request(
input, optional_params, model
)
if VertexBGEConfig.is_bge_model(model):
return VertexBGEConfig.transform_request(
input=input, optional_params=optional_params, model=model
)
vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest()
vertex_text_embedding_input_list: List[TextEmbeddingInput] = []
@@ -173,9 +167,6 @@ class VertexAITextEmbeddingConfig(BaseModel):
vertex_request["parameters"] = TextEmbeddingFineTunedParameters(
**optional_params
)
# Remove 'shared_session' from parameters if present
if vertex_request["parameters"] is not None and "shared_session" in vertex_request["parameters"]:
del vertex_request["parameters"]["shared_session"] # type: ignore[typeddict-item]
return vertex_request
@@ -192,8 +183,8 @@ class VertexAITextEmbeddingConfig(BaseModel):
Args:
content (str): The content to be embedded.
task_type (Optional[TaskType]): The type of task to be performed.
title (Optional[str]): The title of the document to be embedded.
task_type (Optional[TaskType]): The type of task to be performed".
title (Optional[str]): The title of the document to be embedded
Returns:
TextEmbeddingInput: A TextEmbeddingInput object.
@@ -215,14 +206,6 @@ class VertexAITextEmbeddingConfig(BaseModel):
return self._transform_vertex_response_to_openai_for_fine_tuned_models(
response, model, model_response
)
# Import here to avoid circular import issues with litellm.__init__
from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig
if VertexBGEConfig.is_bge_model(model):
return VertexBGEConfig.transform_response(
response=response, model=model, model_response=model_response
)
_predictions = response["predictions"]
litellm/llms/vertex_ai/vertex_embeddings/types.py
+1 -7
View File
@@ -25,12 +25,6 @@ class TextEmbeddingInput(TypedDict, total=False):
title: Optional[str]
class TextEmbeddingBGEInput(TypedDict, total=False):
prompt: str
task_type: Optional[TaskType]
title: Optional[str]
# Fine-tuned models require a different input format
# Ref: https://console.cloud.google.com/vertex-ai/model-garden?hl=en&project=adroit-crow-413218&pageState=(%22galleryStateKey%22:(%22f%22:(%22g%22:%5B%5D,%22o%22:%5B%5D),%22s%22:%22%22))
class TextEmbeddingFineTunedInput(TypedDict, total=False):
@@ -50,7 +44,7 @@ class EmbeddingParameters(TypedDict, total=False):
class VertexEmbeddingRequest(TypedDict, total=False):
instances: Union[List[TextEmbeddingInput], List[TextEmbeddingBGEInput], List[TextEmbeddingFineTunedInput]]
instances: Union[List[TextEmbeddingInput], List[TextEmbeddingFineTunedInput]]
parameters: Optional[Union[EmbeddingParameters, TextEmbeddingFineTunedParameters]]
litellm/llms/vertex_ai/vertex_gemma_models/main.py
+2 -3
View File
@@ -25,7 +25,7 @@ import httpx # type: ignore
from litellm.utils import ModelResponse
from ..common_utils import VertexAIError, get_vertex_base_model_name
from ..common_utils import VertexAIError
from ..vertex_llm_base import VertexBase
@@ -82,8 +82,7 @@ class VertexAIGemmaModels(VertexBase):
message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""",
)
try:
model = get_vertex_base_model_name(model=model)
model = model.replace("gemma/", "")
vertex_httpx_logic = VertexLLM()
access_token, project_id = vertex_httpx_logic._ensure_access_token(
litellm/llms/vertex_ai/vertex_llm_base.py
+3 -49
View File
@@ -19,7 +19,6 @@ from .common_utils import (
_get_gemini_url,
_get_vertex_url,
all_gemini_url_modes,
get_vertex_base_model_name,
is_global_only_vertex_model,
)
@@ -242,9 +241,6 @@ class VertexBase:
auth_header=None,
url=default_api_base,
model=model,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1", # Partner models typically use v1
)
return api_base
@@ -293,18 +289,9 @@ class VertexBase:
auth_header: Optional[str],
url: str,
model: Optional[str] = None,
vertex_project: Optional[str] = None,
vertex_location: Optional[str] = None,
vertex_api_version: Optional[Literal["v1", "v1beta1"]] = None,
) -> Tuple[Optional[str], str]:
"""
for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317
Handles custom api_base for:
1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}
2. Vertex AI with standard proxies - constructs {api_base}:{endpoint}
3. Vertex AI with PSC endpoints - constructs full path structure
{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
## Returns
- (auth_header, url) - Tuple[Optional[str], str]
@@ -324,37 +311,8 @@ class VertexBase:
if gemini_api_key is not None:
auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment]
else:
# For Vertex AI
# Check if this is a PSC endpoint or custom deployment
# PSC/custom endpoints need the full path structure
if vertex_project and vertex_location and model:
# Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction
model_for_url = get_vertex_base_model_name(model=model)
# Check if model is numeric (endpoint ID) or if api_base doesn't contain googleapis.com
# These are indicators of PSC/custom endpoints
is_psc_or_custom = (
"googleapis.com" not in api_base.lower() or model_for_url.isdigit()
)
if is_psc_or_custom:
# Construct full PSC/custom endpoint URL
# Format: {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
version = vertex_api_version or "v1"
url = "{}/{}/projects/{}/locations/{}/endpoints/{}:{}".format(
api_base.rstrip("/"),
version,
vertex_project,
vertex_location,
model_for_url,
endpoint,
)
else:
# Standard proxy - just append endpoint
url = "{}:{}".format(api_base, endpoint)
else:
# Fallback to simple format if we don't have all parameters
url = "{}:{}".format(api_base, endpoint)
url = "{}:{}".format(api_base, endpoint)
if stream is True:
url = url + "?alt=sse"
return auth_header, url
@@ -381,7 +339,6 @@ class VertexBase:
Returns
token, url
"""
version: Optional[Literal["v1beta1", "v1"]] = None
if custom_llm_provider == "gemini":
url, endpoint = _get_gemini_url(
mode=mode,
@@ -397,7 +354,7 @@ class VertexBase:
)
### SET RUNTIME ENDPOINT ###
version = (
version: Literal["v1beta1", "v1"] = (
"v1beta1" if should_use_v1beta1_features is True else "v1"
)
url, endpoint = _get_vertex_url(
@@ -418,9 +375,6 @@ class VertexBase:
stream=stream,
url=url,
model=model,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_api_version=version,
)
def _handle_reauthentication(
litellm/llms/vertex_ai/vertex_model_garden/main.py
+2 -6
View File
@@ -22,7 +22,7 @@ import httpx # type: ignore
from litellm.utils import ModelResponse
from ..common_utils import VertexAIError, get_vertex_base_model_name
from ..common_utils import VertexAIError
from ..vertex_llm_base import VertexBase
@@ -89,7 +89,7 @@ class VertexAIModelGardenModels(VertexBase):
message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""",
)
try:
model = get_vertex_base_model_name(model=model)
model = model.replace("openai/", "")
vertex_httpx_logic = VertexLLM()
access_token, project_id = vertex_httpx_logic._ensure_access_token(
@@ -123,10 +123,6 @@ class VertexAIModelGardenModels(VertexBase):
stream=stream,
auth_header=None,
url=default_api_base,
model=model,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1beta1",
)
model = ""
return openai_like_chat_completions.completion(
litellm/model_prices_and_context_window_backup.json
+77 -5
View File
@@ -8523,6 +8523,14 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/flux/schnell": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.003,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/imagen4/preview": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
@@ -8531,6 +8539,22 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/imagen4/preview/fast": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.02,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/imagen4/preview/ultra": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.06,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/recraft/v3/text-to-image": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
@@ -23408,6 +23432,19 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
"vertex_ai/moonshotai/kimi-k2-thinking-maas": {
"input_cost_per_token": 6e-07,
"litellm_provider": "vertex_ai-moonshot_models",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_web_search": true
},
"vertex_ai/mistral-medium-3": {
"input_cost_per_token": 4e-07,
"litellm_provider": "vertex_ai-mistral_models",
@@ -23744,6 +23781,22 @@
"mode": "embedding",
"output_cost_per_token": 0.0
},
"voyage/voyage-3.5": {
"input_cost_per_token": 6e-08,
"litellm_provider": "voyage",
"max_input_tokens": 32000,
"max_tokens": 32000,
"mode": "embedding",
"output_cost_per_token": 0.0
},
"voyage/voyage-3.5-lite": {
"input_cost_per_token": 2e-08,
"litellm_provider": "voyage",
"max_input_tokens": 32000,
"max_tokens": 32000,
"mode": "embedding",
"output_cost_per_token": 0.0
},
"voyage/voyage-code-2": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "voyage",
@@ -24789,7 +24842,9 @@
"1280x720",
"720x1280"
],
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
"metadata": {
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
}
},
"runwayml/gen4_aleph": {
"litellm_provider": "runwayml",
@@ -24807,7 +24862,9 @@
"1280x720",
"720x1280"
],
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
"metadata": {
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
}
},
"runwayml/gen3a_turbo": {
"litellm_provider": "runwayml",
@@ -24825,7 +24882,9 @@
"1280x720",
"720x1280"
],
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
"metadata": {
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
}
},
"runwayml/gen4_image": {
"litellm_provider": "runwayml",
@@ -24844,7 +24903,9 @@
"1280x720",
"1920x1080"
],
"comment": "5 credits per 720p image or 8 credits per 1080p image @ $0.01 per credit. Using 5 credits ($0.05) as base cost"
"metadata": {
"comment": "5 credits per 720p image or 8 credits per 1080p image @ $0.01 per credit. Using 5 credits ($0.05) as base cost"
}
},
"runwayml/gen4_image_turbo": {
"litellm_provider": "runwayml",
@@ -24863,6 +24924,17 @@
"1280x720",
"1920x1080"
],
"comment": "2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"
"metadata": {
"comment": "2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"
}
},
"runwayml/eleven_multilingual_v2": {
"litellm_provider": "runwayml",
"mode": "audio_speech",
"input_cost_per_character": 3e-07,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"metadata": {
"comment": "Estimated cost based on standard TTS pricing. RunwayML uses ElevenLabs models."
}
}
}
litellm/proxy/_experimental/out/assets/logos/runway.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.0 KiB

litellm/proxy/common_request_processing.py
+6 -2
View File
@@ -292,6 +292,7 @@ class ProxyBaseLLMRequestProcessing:
proxy_config: ProxyConfig,
route_type: Literal[
"acompletion",
"aembedding",
"aresponses",
"_arealtime",
"aget_responses",
@@ -403,6 +404,7 @@ class ProxyBaseLLMRequestProcessing:
user_api_key_dict: UserAPIKeyAuth,
route_type: Literal[
"acompletion",
"aembedding",
"aresponses",
"_arealtime",
"aget_responses",
@@ -772,10 +774,12 @@ class ProxyBaseLLMRequestProcessing:
@staticmethod
def _get_pre_call_type(
route_type: Literal["acompletion", "aresponses"],
) -> Literal["completion", "responses"]:
route_type: Literal["acompletion", "aembedding", "aresponses"],
) -> Literal["completion", "embeddings", "responses"]:
if route_type == "acompletion":
return "completion"
elif route_type == "aembedding":
return "embeddings"
elif route_type == "aresponses":
return "responses"
litellm/proxy/litellm_pre_call_utils.py
+1 -1
View File
@@ -1158,7 +1158,7 @@ def _enforced_params_check(
)
if enforced_params is None:
return True
if enforced_params is not None and premium_user is not True:
if enforced_params and premium_user is not True:
raise ValueError(
f"Enforced Params is an Enterprise feature. Enforced Params: {enforced_params}. {CommonProxyErrors.not_premium_user.value}"
)
litellm/proxy/management_endpoints/mcp_management_endpoints.py
+3 -1
View File
@@ -68,7 +68,9 @@ if MCP_AVAILABLE:
except AttributeError:
redacted_server = mcp_server.copy(deep=True) # type: ignore[attr-defined]
redacted_server.credentials = None
if hasattr(redacted_server, "credentials"):
setattr(redacted_server, "credentials", None)
return redacted_server
def _redact_mcp_credentials_list(
litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
+688
View File
@@ -0,0 +1,688 @@
"""
Allow proxy admin to manage model access groups
Endpoints here:
- POST /model_group/new - Create a new access group with multiple model names
"""
import json
from typing import Any, Dict, List, Tuple
from fastapi import APIRouter, Depends, HTTPException
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
# Clear cache and reload models to pick up the access group changes
from litellm.proxy.management_endpoints.model_management_endpoints import (
clear_cache,
)
from litellm.proxy.utils import PrismaClient
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
AccessGroupInfo,
DeleteModelGroupResponse,
ListAccessGroupsResponse,
NewModelGroupRequest,
NewModelGroupResponse,
UpdateModelGroupRequest,
)
router = APIRouter()
def validate_models_exist(
model_names: List[str], llm_router
) -> Tuple[bool, List[str]]:
"""
Validate that all requested model names exist in the router.
Checks only exact model name matches.
Returns:
Tuple[bool, List[str]]: (all_valid, missing_models)
"""
if llm_router is None:
return False, model_names
router_model_names = set(llm_router.get_model_names())
missing = [m for m in model_names if m not in router_model_names]
return (len(missing) == 0, missing)
def add_access_group_to_deployment(
model_info: Dict[str, Any], access_group: str
) -> Tuple[Dict[str, Any], bool]:
"""
Add an access group to a deployment's model_info.
Args:
model_info: The model_info dictionary from the deployment
access_group: The access group name to add
Returns:
Tuple[Dict[str, Any], bool]: (updated_model_info, was_modified)
"""
access_groups = model_info.get("access_groups", [])
# Check if access group already exists
if access_group in access_groups:
return model_info, False
# Add the access group
access_groups.append(access_group)
model_info["access_groups"] = access_groups
return model_info, True
async def update_deployments_with_access_group(
model_names: List[str],
access_group: str,
prisma_client: PrismaClient,
) -> int:
"""
Update all deployments for the given model names to include the access group.
Args:
model_names: List of model names whose deployments should be updated
access_group: The access group name to add
prisma_client: Database client
Returns:
int: Number of deployments updated
"""
models_updated = 0
for model_name in model_names:
verbose_proxy_logger.debug(
f"Updating deployments for model_name: {model_name}"
)
# Get all deployments with this model_name
deployments = await prisma_client.db.litellm_proxymodeltable.find_many(
where={"model_name": model_name}
)
verbose_proxy_logger.debug(
f"Found {len(deployments)} deployments for model_name: {model_name}"
)
# If no deployments found, this is a config model (not in DB)
if len(deployments) == 0:
raise HTTPException(
status_code=400,
detail={
"error": f"Can't find model '{model_name}' in Database. Access group management is only supported for database models."
},
)
# Update each deployment
for deployment in deployments:
model_info = deployment.model_info or {}
# Add access group using helper
updated_model_info, was_modified = add_access_group_to_deployment(
model_info=model_info,
access_group=access_group,
)
# Only update in DB if modified
if was_modified:
await prisma_client.db.litellm_proxymodeltable.update(
where={"model_id": deployment.model_id},
data={"model_info": json.dumps(updated_model_info)},
)
models_updated += 1
verbose_proxy_logger.debug(
f"Updated deployment {deployment.model_id} with access group: {access_group}"
)
return models_updated
def remove_access_group_from_deployment(
model_info: Dict[str, Any], access_group: str
) -> Tuple[Dict[str, Any], bool]:
"""
Remove an access group from a deployment's model_info.
Args:
model_info: The model_info dictionary from the deployment
access_group: The access group name to remove
Returns:
Tuple[Dict[str, Any], bool]: (updated_model_info, was_modified)
"""
access_groups = model_info.get("access_groups", [])
# Check if access group exists
if access_group not in access_groups:
return model_info, False
# Remove the access group
access_groups.remove(access_group)
model_info["access_groups"] = access_groups
return model_info, True
async def get_all_access_groups_from_db(
prisma_client: PrismaClient,
) -> Dict[str, AccessGroupInfo]:
"""
Get all access groups from the database.
Returns:
Dict[str, AccessGroupInfo]: Dictionary mapping access_group name to info
"""
# Get all deployments
deployments = await prisma_client.db.litellm_proxymodeltable.find_many()
# Build access group map
access_group_map: Dict[str, Dict[str, Any]] = {}
for deployment in deployments:
model_info = deployment.model_info or {}
access_groups = model_info.get("access_groups", [])
model_name = deployment.model_name
for access_group in access_groups:
if access_group not in access_group_map:
access_group_map[access_group] = {
"model_names": set(),
"deployment_count": 0,
}
access_group_map[access_group]["model_names"].add(model_name)
access_group_map[access_group]["deployment_count"] += 1
# Convert to AccessGroupInfo objects
result = {}
for access_group, data in access_group_map.items():
result[access_group] = AccessGroupInfo(
access_group=access_group,
model_names=sorted(list(data["model_names"])),
deployment_count=data["deployment_count"],
)
return result
@router.post(
"/access_group/new",
tags=["model management"],
dependencies=[Depends(user_api_key_auth)],
response_model=NewModelGroupResponse,
)
async def create_model_group(
data: NewModelGroupRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Create a new access group containing multiple model names.
An access group is a named collection of model groups that can be referenced
by teams/keys for simplified access control.
Example:
```bash
curl -X POST 'http://localhost:4000/access_group/new' \\
-H 'Authorization: Bearer sk-1234' \\
-H 'Content-Type: application/json' \\
-d '{
"access_group": "production-models",
"model_names": ["gpt-4", "claude-3-opus", "gemini-pro"]
}'
```
Parameters:
- access_group: str - The access group name (e.g., "production-models")
- model_names: List[str] - List of existing model groups to include
Returns:
- NewModelGroupResponse with the created access group details
Raises:
- HTTPException 400: If any model names don't exist
- HTTPException 500: If database operations fail
"""
from litellm.proxy.proxy_server import (
llm_router,
prisma_client,
)
verbose_proxy_logger.debug(
f"Creating access group: {data.access_group} with models: {data.model_names}"
)
# Validation: Check if access_group is provided
if not data.access_group or not data.access_group.strip():
raise HTTPException(
status_code=400,
detail={"error": "access_group is required and cannot be empty"},
)
# Validation: Check if model_names list is provided and not empty
if not data.model_names or len(data.model_names) == 0:
raise HTTPException(
status_code=400,
detail={"error": "model_names list is required and cannot be empty"},
)
# Validation: Check if all models exist in the router
all_valid, missing_models = validate_models_exist(
model_names=data.model_names,
llm_router=llm_router,
)
if not all_valid:
raise HTTPException(
status_code=400,
detail={"error": f"Model(s) not found: {', '.join(missing_models)}"},
)
# Check if database is connected
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": "Database not connected. Cannot create access group."},
)
try:
# Check if access group already exists
existing_access_groups = await get_all_access_groups_from_db(
prisma_client=prisma_client
)
if data.access_group in existing_access_groups:
raise HTTPException(
status_code=409,
detail={"error": f"Access group '{data.access_group}' already exists. Use PUT /access_group/{data.access_group}/update to modify it."},
)
# Update deployments using helper function
models_updated = await update_deployments_with_access_group(
model_names=data.model_names,
access_group=data.access_group,
prisma_client=prisma_client,
)
await clear_cache()
verbose_proxy_logger.info(
f"Successfully created access group '{data.access_group}' with {models_updated} models updated"
)
return NewModelGroupResponse(
access_group=data.access_group,
model_names=data.model_names,
models_updated=models_updated,
)
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
f"Error creating access group '{data.access_group}': {str(e)}"
)
raise HTTPException(
status_code=500,
detail={"error": f"Failed to create access group: {str(e)}"},
)
@router.get(
"/access_group/list",
tags=["model management"],
dependencies=[Depends(user_api_key_auth)],
response_model=ListAccessGroupsResponse,
)
async def list_access_groups(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
List all access groups.
Returns a list of all access groups with their model names and deployment counts.
Example:
```bash
curl -X GET 'http://localhost:4000/access_group/list' \\
-H 'Authorization: Bearer sk-1234'
```
Returns:
- ListAccessGroupsResponse with all access groups
"""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": "Database not connected."},
)
try:
access_groups_map = await get_all_access_groups_from_db(
prisma_client=prisma_client
)
# Sort by access group name
access_groups_list = sorted(
access_groups_map.values(),
key=lambda x: x.access_group,
)
return ListAccessGroupsResponse(access_groups=access_groups_list)
except Exception as e:
verbose_proxy_logger.exception(f"Error listing access groups: {str(e)}")
raise HTTPException(
status_code=500,
detail={"error": f"Failed to list access groups: {str(e)}"},
)
@router.get(
"/access_group/{access_group}/info",
tags=["model management"],
dependencies=[Depends(user_api_key_auth)],
response_model=AccessGroupInfo,
)
async def get_access_group_info(
access_group: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Get information about a specific access group.
Example:
```bash
curl -X GET 'http://localhost:4000/access_group/production-models/info' \\
-H 'Authorization: Bearer sk-1234'
```
Parameters:
- access_group: str - The access group name (URL path parameter)
Returns:
- AccessGroupInfo with the access group details
Raises:
- HTTPException 404: If access group not found
"""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": "Database not connected."},
)
try:
access_groups_map = await get_all_access_groups_from_db(
prisma_client=prisma_client
)
if access_group not in access_groups_map:
raise HTTPException(
status_code=404,
detail={"error": f"Access group '{access_group}' not found"},
)
return access_groups_map[access_group]
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
f"Error getting access group info for '{access_group}': {str(e)}"
)
raise HTTPException(
status_code=500,
detail={"error": f"Failed to get access group info: {str(e)}"},
)
@router.put(
"/access_group/{access_group}/update",
tags=["model management"],
dependencies=[Depends(user_api_key_auth)],
response_model=NewModelGroupResponse,
)
async def update_access_group(
access_group: str,
data: UpdateModelGroupRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Update an access group's model names.
This will:
1. Remove the access group from all current deployments
2. Add the access group to all deployments for the new model_names list
Example:
```bash
curl -X PUT 'http://localhost:4000/access_group/production-models/update' \\
-H 'Authorization: Bearer sk-1234' \\
-H 'Content-Type: application/json' \\
-d '{
"model_names": ["gpt-4", "claude-3-sonnet"]
}'
```
Parameters:
- access_group: str - The access group name (URL path parameter)
- model_names: List[str] - New list of model groups to include
Returns:
- NewModelGroupResponse with the updated access group details
Raises:
- HTTPException 400: If any model names don't exist
- HTTPException 404: If access group not found
"""
from litellm.proxy.proxy_server import llm_router, prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": "Database not connected."},
)
verbose_proxy_logger.debug(
f"Updating access group: {access_group} with models: {data.model_names}"
)
# Validation: Check if model_names list is provided and not empty
if not data.model_names or len(data.model_names) == 0:
raise HTTPException(
status_code=400,
detail={"error": "model_names list is required and cannot be empty"},
)
# Validation: Check if access group exists
try:
access_groups_map = await get_all_access_groups_from_db(
prisma_client=prisma_client
)
if access_group not in access_groups_map:
raise HTTPException(
status_code=404,
detail={"error": f"Access group '{access_group}' not found"},
)
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=500,
detail={"error": f"Failed to check access group existence: {str(e)}"},
)
# Validation: Check if all new models exist
all_valid, missing_models = validate_models_exist(
model_names=data.model_names,
llm_router=llm_router,
)
if not all_valid:
raise HTTPException(
status_code=400,
detail={"error": f"Model(s) not found: {', '.join(missing_models)}"},
)
try:
# Step 1: Remove access group from ALL DB deployments (skip config models)
all_deployments = await prisma_client.db.litellm_proxymodeltable.find_many()
for deployment in all_deployments:
model_info = deployment.model_info or {}
updated_model_info, was_modified = remove_access_group_from_deployment(
model_info=model_info,
access_group=access_group,
)
if was_modified:
await prisma_client.db.litellm_proxymodeltable.update(
where={"model_id": deployment.model_id},
data={"model_info": json.dumps(updated_model_info)},
)
# Step 2: Add access group to new model_names
models_updated = await update_deployments_with_access_group(
model_names=data.model_names,
access_group=access_group,
prisma_client=prisma_client,
)
# Clear cache and reload models to pick up the access group changes
await clear_cache()
verbose_proxy_logger.info(
f"Successfully updated access group '{access_group}' with {models_updated} models updated"
)
return NewModelGroupResponse(
access_group=access_group,
model_names=data.model_names,
models_updated=models_updated,
)
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
f"Error updating access group '{access_group}': {str(e)}"
)
raise HTTPException(
status_code=500,
detail={"error": f"Failed to update access group: {str(e)}"},
)
@router.delete(
"/access_group/{access_group}/delete",
tags=["model management"],
dependencies=[Depends(user_api_key_auth)],
response_model=DeleteModelGroupResponse,
)
async def delete_access_group(
access_group: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Delete an access group.
Removes the access group from all deployments that have it.
Example:
```bash
curl -X DELETE 'http://localhost:4000/access_group/production-models/delete' \\
-H 'Authorization: Bearer sk-1234'
```
Parameters:
- access_group: str - The access group name (URL path parameter)
Returns:
- DeleteModelGroupResponse with deletion details
Raises:
- HTTPException 404: If access group not found
"""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": "Database not connected."},
)
verbose_proxy_logger.debug(f"Deleting access group: {access_group}")
# Validation: Check if access group exists
try:
access_groups_map = await get_all_access_groups_from_db(
prisma_client=prisma_client
)
if access_group not in access_groups_map:
raise HTTPException(
status_code=404,
detail={"error": f"Access group '{access_group}' not found"},
)
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=500,
detail={"error": f"Failed to check access group existence: {str(e)}"},
)
try:
# Remove access group from all DB deployments (skip config models)
all_deployments = await prisma_client.db.litellm_proxymodeltable.find_many()
models_updated = 0
for deployment in all_deployments:
model_info = deployment.model_info or {}
updated_model_info, was_modified = remove_access_group_from_deployment(
model_info=model_info,
access_group=access_group,
)
if was_modified:
await prisma_client.db.litellm_proxymodeltable.update(
where={"model_id": deployment.model_id},
data={"model_info": json.dumps(updated_model_info)},
)
models_updated += 1
# Clear cache and reload models to pick up the access group changes
await clear_cache()
verbose_proxy_logger.info(
f"Successfully deleted access group '{access_group}' from {models_updated} deployments"
)
return DeleteModelGroupResponse(
access_group=access_group,
models_updated=models_updated,
message=f"Access group '{access_group}' deleted successfully",
)
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
f"Error deleting access group '{access_group}': {str(e)}"
)
raise HTTPException(
status_code=500,
detail={"error": f"Failed to delete access group: {str(e)}"},
)
litellm/proxy/proxy_server.py
+53 -143
View File
@@ -292,6 +292,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
from litellm.proxy.management_endpoints.model_management_endpoints import (
router as model_management_router,
)
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
router as model_access_group_management_router,
)
from litellm.proxy.management_endpoints.organization_endpoints import (
router as organization_router,
)
@@ -5011,40 +5014,11 @@ async def embeddings( # noqa: PLR0915
global proxy_logging_obj
data: Any = {}
try:
# Use orjson to parse JSON data, orjson speeds up requests significantly
body = await request.body()
data = orjson.loads(body)
verbose_proxy_logger.debug(
"Request received by LiteLLM:\n%s",
json.dumps(data, indent=4),
)
# Include original request and headers in the data
data = await add_litellm_data_to_request(
data=data,
request=request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_config=proxy_config,
)
data["model"] = (
general_settings.get("embedding_model", None) # server default
or user_model # model name passed via cli args
or model # for azure deployments
or data.get("model", None) # default passed in http request
)
if user_model:
data["model"] = user_model
### MODEL ALIAS MAPPING ###
# check if model name in model alias map
# get the actual model name
if data["model"] in litellm.model_alias_map:
data["model"] = litellm.model_alias_map[data["model"]]
# Use shared request body reading helper (same as chat/completions)
data = await _read_request_body(request=request)
### HANDLE TOKEN ARRAY INPUT DECODING ###
# This must happen BEFORE base_process_llm_request() since it modifies the input
router_model_names = llm_router.model_names if llm_router is not None else []
if (
"input" in data
@@ -5054,126 +5028,61 @@ async def embeddings( # noqa: PLR0915
and isinstance(data["input"][0][0], int)
): # check if array of tokens passed in
# check if provider accept list of tokens as input - e.g. for langchain integration
if llm_model_list is not None and data["model"] in router_model_names:
for m in llm_model_list:
if m["model_name"] == data["model"]:
if m["litellm_params"][
"model"
] in litellm.open_ai_embedding_models or any(
m["litellm_params"]["model"].startswith(provider)
if llm_router is not None and data.get("model") in router_model_names:
# Use router's O(1) lookup instead of O(N) iteration through llm_model_list
deployment = llm_router.get_deployment(model_id=data["model"])
if deployment is not None:
litellm_model = deployment.get("litellm_params", {}).get("model", "")
# Check if this provider supports token arrays
supports_token_arrays = (
litellm_model in litellm.open_ai_embedding_models
or any(
litellm_model.startswith(provider)
for provider in LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS
):
pass
else:
# non-openai/azure embedding model called with token input
input_list = []
for i in data["input"]:
input_list.append(
litellm.decode(model="gpt-3.5-turbo", tokens=i)
)
data["input"] = input_list
break
)
)
if not supports_token_arrays:
# non-openai/azure embedding model called with token input - decode tokens
input_list = []
for i in data["input"]:
input_list.append(
litellm.decode(model="gpt-3.5-turbo", tokens=i)
)
data["input"] = input_list
### CALL HOOKS ### - modify incoming data / reject request before calling the model
data = await proxy_logging_obj.pre_call_hook(
# Use unified request processor (same as chat/completions and responses)
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
# Process the request with all optimizations (shared sessions, network tuning, etc.)
response = await base_llm_response_processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
data=data,
call_type=CallTypes.aembedding.value,
)
tasks = []
tasks.append(
proxy_logging_obj.during_call_hook(
data=data,
user_api_key_dict=user_api_key_dict,
call_type="aembedding",
)
)
## ROUTE TO CORRECT ENDPOINT ##
llm_call = await route_request(
data=data,
route_type="aembedding",
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
general_settings=general_settings,
proxy_config=proxy_config,
select_data_generator=select_data_generator,
model=model,
user_model=user_model,
user_temperature=user_temperature,
user_request_timeout=user_request_timeout,
user_max_tokens=user_max_tokens,
user_api_base=user_api_base,
version=version,
)
tasks.append(llm_call)
# wait for call to end
llm_responses = asyncio.gather(
*tasks
) # run the moderation check in parallel to the actual llm api call
responses = await llm_responses
response = responses[1]
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(
litellm_call_id=data.get("litellm_call_id", ""), status="success"
)
)
### RESPONSE HEADERS ###
hidden_params = getattr(response, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
litellm_call_id = hidden_params.get("litellm_call_id", None) or ""
additional_headers: dict = hidden_params.get("additional_headers", {}) or {}
fastapi_response.headers.update(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
model_id=model_id,
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
call_id=litellm_call_id,
request_data=data,
hidden_params=hidden_params,
**additional_headers,
)
)
await check_response_size_is_safe(response=response)
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
# Use unified error handler
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
raise await base_llm_response_processor._handle_llm_api_exception(
e=e,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)
litellm_debug_info = getattr(e, "litellm_debug_info", "")
verbose_proxy_logger.debug(
"\033[1;31mAn error occurred: %s %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`",
e,
litellm_debug_info,
)
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.embeddings(): Exception occured - {}".format(
str(e)
)
)
if isinstance(e, HTTPException):
message = get_error_message_str(e)
raise ProxyException(
message=message,
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)
@router.post(
@@ -10180,6 +10089,7 @@ app.include_router(openai_files_router)
app.include_router(team_callback_router)
app.include_router(budget_management_router)
app.include_router(model_management_router)
app.include_router(model_access_group_management_router)
app.include_router(tag_management_router)
app.include_router(cost_tracking_settings_router)
app.include_router(router_settings_router)
litellm/proxy/public_endpoints/provider_create_metadata.py
+769
View File
@@ -0,0 +1,769 @@
from __future__ import annotations
from typing import Any, Dict, List
from litellm.types.proxy.public_endpoints.public_endpoints import (
ProviderCreateInfo,
ProviderCredentialField,
)
from litellm.types.utils import LlmProviders
DEFAULT_MODEL_PLACEHOLDER = "gpt-3.5-turbo"
_FALLBACK_FIELDS: List[Dict[str, Any]] = [
{
"key": "api_base",
"label": "API Base",
"field_type": "text",
"required": False,
},
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": False,
},
]
PROVIDER_BASE_INFO: Dict[str, Dict[str, Any]] = {
"AIML": {
"provider_display_name": "AI/ML API",
"litellm_provider": "aiml",
"default_model_placeholder": "aiml/flux-pro/v1.1",
},
"Anthropic": {
"provider_display_name": "Anthropic",
"litellm_provider": "anthropic",
"default_model_placeholder": "claude-3-opus",
},
"AssemblyAI": {
"provider_display_name": "AssemblyAI",
"litellm_provider": "assemblyai",
},
"Azure": {
"provider_display_name": "Azure",
"litellm_provider": "azure",
"default_model_placeholder": "azure/my-deployment",
},
"Azure_AI_Studio": {
"provider_display_name": "Azure AI Foundry (Studio)",
"litellm_provider": "azure_ai",
"default_model_placeholder": "azure_ai/command-r-plus",
},
"Bedrock": {
"provider_display_name": "Amazon Bedrock",
"litellm_provider": "bedrock",
"default_model_placeholder": "claude-3-opus",
},
"Cerebras": {
"provider_display_name": "Cerebras",
"litellm_provider": "cerebras",
},
"Cohere": {
"provider_display_name": "Cohere",
"litellm_provider": "cohere",
},
"Dashscope": {
"provider_display_name": "Dashscope",
"litellm_provider": "dashscope",
},
"Databricks": {
"provider_display_name": "Databricks (Qwen API)",
"litellm_provider": "databricks",
},
"DeepInfra": {
"provider_display_name": "DeepInfra",
"litellm_provider": "deepinfra",
"default_model_placeholder": "deepinfra/<any-model-on-deepinfra>",
},
"Deepgram": {
"provider_display_name": "Deepgram",
"litellm_provider": "deepgram",
},
"Deepseek": {
"provider_display_name": "Deepseek",
"litellm_provider": "deepseek",
},
"ElevenLabs": {
"provider_display_name": "ElevenLabs",
"litellm_provider": "elevenlabs",
},
"FalAI": {
"provider_display_name": "Fal AI",
"litellm_provider": "fal_ai",
"default_model_placeholder": "fal_ai/fal-ai/flux-pro/v1.1-ultra",
},
"FireworksAI": {
"provider_display_name": "Fireworks AI",
"litellm_provider": "fireworks_ai",
},
"Google_AI_Studio": {
"provider_display_name": "Google AI Studio",
"litellm_provider": "gemini",
"default_model_placeholder": "gemini-pro",
},
"GradientAI": {
"provider_display_name": "GradientAI",
"litellm_provider": "gradient_ai",
},
"Groq": {
"provider_display_name": "Groq",
"litellm_provider": "groq",
},
"Hosted_Vllm": {
"provider_display_name": "vllm",
"litellm_provider": "hosted_vllm",
},
"Infinity": {
"provider_display_name": "Infinity",
"litellm_provider": "infinity",
},
"JinaAI": {
"provider_display_name": "Jina AI",
"litellm_provider": "jina_ai",
"default_model_placeholder": "jina_ai/",
},
"MistralAI": {
"provider_display_name": "Mistral AI",
"litellm_provider": "mistral",
},
"Ollama": {
"provider_display_name": "Ollama",
"litellm_provider": "ollama",
},
"OpenAI": {
"provider_display_name": "OpenAI",
"litellm_provider": "openai",
},
"OpenAI_Compatible": {
"provider_display_name": "OpenAI-Compatible Endpoints (Together AI, etc.)",
"litellm_provider": "openai",
},
"OpenAI_Text": {
"provider_display_name": "OpenAI Text Completion",
"litellm_provider": "text-completion-openai",
},
"OpenAI_Text_Compatible": {
"provider_display_name": "OpenAI-Compatible Text Completion Models (Together AI, etc.)",
"litellm_provider": "text-completion-openai",
},
"Openrouter": {
"provider_display_name": "Openrouter",
"litellm_provider": "openrouter",
},
"Oracle": {
"provider_display_name": "Oracle Cloud Infrastructure (OCI)",
"litellm_provider": "oci",
"default_model_placeholder": "oci/xai.grok-4",
},
"Perplexity": {
"provider_display_name": "Perplexity",
"litellm_provider": "perplexity",
},
"SageMaker": {
"provider_display_name": "AWS SageMaker",
"litellm_provider": "sagemaker_chat",
"default_model_placeholder": "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b",
},
"Sambanova": {
"provider_display_name": "Sambanova",
"litellm_provider": "sambanova",
},
"Snowflake": {
"provider_display_name": "Snowflake",
"litellm_provider": "snowflake",
"default_model_placeholder": "snowflake/mistral-7b",
},
"TogetherAI": {
"provider_display_name": "TogetherAI",
"litellm_provider": "together_ai",
},
"Triton": {
"provider_display_name": "Triton",
"litellm_provider": "triton",
},
"Vertex_AI": {
"provider_display_name": "Vertex AI (Anthropic, Gemini, etc.)",
"litellm_provider": "vertex_ai",
"default_model_placeholder": "gemini-pro",
},
"VolcEngine": {
"provider_display_name": "VolcEngine",
"litellm_provider": "volcengine",
"default_model_placeholder": "volcengine/<any-model-on-volcengine>",
},
"Voyage": {
"provider_display_name": "Voyage AI",
"litellm_provider": "voyage",
"default_model_placeholder": "voyage/",
},
"xAI": {
"provider_display_name": "xAI",
"litellm_provider": "xai",
},
}
PROVIDER_CREDENTIAL_FIELDS: Dict[str, List[Dict[str, Any]]] = {
"OpenAI": [
{
"key": "api_base",
"label": "API Base",
"field_type": "text",
"placeholder": "https://api.openai.com/v1",
"tooltip": "Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com",
"default_value": "https://api.openai.com/v1",
},
{
"key": "organization",
"label": "OpenAI Organization ID",
"placeholder": "[OPTIONAL] my-unique-org",
},
{
"key": "api_key",
"label": "OpenAI API Key",
"field_type": "password",
"required": True,
},
],
"OpenAI_Text": [
{
"key": "api_base",
"label": "API Base",
"field_type": "text",
"placeholder": "https://api.openai.com/v1",
"tooltip": "Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com",
"default_value": "https://api.openai.com/v1",
},
{
"key": "organization",
"label": "OpenAI Organization ID",
"placeholder": "[OPTIONAL] my-unique-org",
},
{
"key": "api_key",
"label": "OpenAI API Key",
"field_type": "password",
"required": True,
},
],
"Vertex_AI": [
{
"key": "vertex_project",
"label": "Vertex Project",
"placeholder": "adroit-cadet-1234..",
"required": True,
},
{
"key": "vertex_location",
"label": "Vertex Location",
"placeholder": "us-east-1",
"required": True,
},
{
"key": "vertex_credentials",
"label": "Vertex Credentials",
"field_type": "upload",
"required": True,
},
],
"AssemblyAI": [
{
"key": "api_base",
"label": "API Base",
"field_type": "select",
"required": True,
"options": [
"https://api.assemblyai.com",
"https://api.eu.assemblyai.com",
],
},
{
"key": "api_key",
"label": "AssemblyAI API Key",
"field_type": "password",
"required": True,
},
],
"Azure": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://...",
"required": True,
},
{
"key": "api_version",
"label": "API Version",
"placeholder": "2023-07-01-preview",
"tooltip": "By default litellm will use the latest version. If you want to use a different version, you can specify it here",
},
{
"key": "base_model",
"label": "Base Model",
"placeholder": "azure/gpt-3.5-turbo",
},
{
"key": "api_key",
"label": "Azure API Key",
"field_type": "password",
"placeholder": "Enter your Azure API Key",
},
{
"key": "azure_ad_token",
"label": "Azure AD Token",
"field_type": "password",
"placeholder": "Enter your Azure AD Token",
},
],
"Azure_AI_Studio": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://<test>.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21",
"tooltip": "Enter your full Target URI from Azure Foundry here. Example: https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21",
"required": True,
},
{
"key": "api_key",
"label": "Azure API Key",
"field_type": "password",
"required": True,
},
],
"OpenAI_Compatible": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://...",
"required": True,
},
{
"key": "api_key",
"label": "OpenAI API Key",
"field_type": "password",
"required": True,
},
],
"Dashscope": [
{
"key": "api_key",
"label": "Dashscope API Key",
"field_type": "password",
"required": True,
},
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"default_value": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"required": True,
"tooltip": "The base URL for your Dashscope server. Defaults to https://dashscope-intl.aliyuncs.com/compatible-mode/v1 if not specified.",
},
],
"OpenAI_Text_Compatible": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://...",
"required": True,
},
{
"key": "api_key",
"label": "OpenAI API Key",
"field_type": "password",
"required": True,
},
],
"Bedrock": [
{
"key": "aws_access_key_id",
"label": "AWS Access Key ID",
"field_type": "password",
"tooltip": "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
"key": "aws_secret_access_key",
"label": "AWS Secret Access Key",
"field_type": "password",
"tooltip": "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
"key": "aws_session_token",
"label": "AWS Session Token",
"field_type": "password",
"tooltip": "Temporary credentials session token. You can provide the raw token or the environment variable (e.g. `os.environ/MY_SESSION_TOKEN`).",
},
{
"key": "aws_region_name",
"label": "AWS Region Name",
"placeholder": "us-east-1",
"tooltip": "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
"key": "aws_session_name",
"label": "AWS Session Name",
"placeholder": "my-session",
"tooltip": "Name for the AWS session. You can provide the raw value or the environment variable (e.g. `os.environ/MY_SESSION_NAME`).",
},
{
"key": "aws_profile_name",
"label": "AWS Profile Name",
"placeholder": "default",
"tooltip": "AWS profile name to use for authentication. You can provide the raw value or the environment variable (e.g. `os.environ/MY_PROFILE_NAME`).",
},
{
"key": "aws_role_name",
"label": "AWS Role Name",
"placeholder": "MyRole",
"tooltip": "AWS IAM role name to assume. You can provide the raw value or the environment variable (e.g. `os.environ/MY_ROLE_NAME`).",
},
{
"key": "aws_web_identity_token",
"label": "AWS Web Identity Token",
"field_type": "password",
"tooltip": "Web identity token for OIDC authentication. You can provide the raw token or the environment variable (e.g. `os.environ/MY_WEB_IDENTITY_TOKEN`).",
},
{
"key": "aws_bedrock_runtime_endpoint",
"label": "AWS Bedrock Runtime Endpoint",
"placeholder": "https://bedrock-runtime.us-east-1.amazonaws.com",
"tooltip": "Custom Bedrock runtime endpoint URL. You can provide the raw value or the environment variable (e.g. `os.environ/MY_BEDROCK_ENDPOINT`).",
},
],
"SageMaker": [
{
"key": "aws_access_key_id",
"label": "AWS Access Key ID",
"field_type": "password",
"tooltip": "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
"key": "aws_secret_access_key",
"label": "AWS Secret Access Key",
"field_type": "password",
"tooltip": "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
"key": "aws_region_name",
"label": "AWS Region Name",
"placeholder": "us-east-1",
"tooltip": "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
],
"Ollama": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "http://localhost:11434",
"default_value": "http://localhost:11434",
"tooltip": "The base URL for your Ollama server. Defaults to http://localhost:11434 if not specified.",
},
],
"Anthropic": [
{
"key": "api_key",
"label": "API Key",
"placeholder": "sk-",
"field_type": "password",
"required": True,
},
],
"Deepgram": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"ElevenLabs": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Google_AI_Studio": [
{
"key": "api_key",
"label": "API Key",
"placeholder": "aig-",
"field_type": "password",
"required": True,
},
],
"Groq": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"MistralAI": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Deepseek": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Cohere": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Databricks": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"xAI": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"AIML": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Cerebras": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Sambanova": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Perplexity": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"TogetherAI": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Openrouter": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"FireworksAI": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"GradientAI": [
{
"key": "api_base",
"label": "GradientAI Endpoint",
"placeholder": "https://...",
},
{
"key": "api_key",
"label": "GradientAI API Key",
"field_type": "password",
"required": True,
},
],
"Triton": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
},
{
"key": "api_base",
"label": "API Base",
"placeholder": "http://localhost:8000/generate",
},
],
"Hosted_Vllm": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://...",
"required": True,
},
{
"key": "api_key",
"label": "vLLM API Key",
"field_type": "password",
},
],
"Voyage": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"JinaAI": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"VolcEngine": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"DeepInfra": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Oracle": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
"Snowflake": [
{
"key": "api_key",
"label": "Snowflake API Key / JWT Key for Authentication",
"field_type": "password",
"required": True,
},
{
"key": "api_base",
"label": "Snowflake API Endpoint",
"placeholder": "https://1234567890.snowflakecomputing.com/api/v2/cortex/inference:complete",
"tooltip": "Enter the full endpoint with path here. Example: https://1234567890.snowflakecomputing.com/api/v2/cortex/inference:complete",
"required": True,
},
],
"Infinity": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "http://localhost:7997",
},
],
"FalAI": [
{
"key": "api_key",
"label": "API Key",
"field_type": "password",
"required": True,
},
],
}
def _normalize_field(field: Dict[str, Any]) -> ProviderCredentialField:
return ProviderCredentialField(
key=field["key"],
label=field["label"],
placeholder=field.get("placeholder"),
tooltip=field.get("tooltip"),
required=field.get("required", False),
field_type=field.get("field_type", "text"),
options=field.get("options"),
default_value=field.get("default_value"),
)
def get_provider_create_metadata() -> List[ProviderCreateInfo]:
providers: List[ProviderCreateInfo] = []
for provider_key, base_info in PROVIDER_BASE_INFO.items():
raw_fields = PROVIDER_CREDENTIAL_FIELDS.get(provider_key, _FALLBACK_FIELDS)
normalized_fields = [_normalize_field(field) for field in raw_fields]
providers.append(
ProviderCreateInfo(
provider=provider_key,
provider_display_name=base_info["provider_display_name"],
litellm_provider=base_info["litellm_provider"],
default_model_placeholder=base_info.get(
"default_model_placeholder", DEFAULT_MODEL_PLACEHOLDER
),
credential_fields=normalized_fields,
)
)
# Ensure we have metadata entries for all providers defined in LlmProviders.
# If a provider enum value is not already present in the litellm_provider
# field of any entry, create a default entry for it using the fallback
# credential fields (api_key + api_base) and a generated display name.
existing_litellm_providers = {p.litellm_provider for p in providers}
for provider_enum in LlmProviders:
litellm_provider_value = provider_enum.value
if litellm_provider_value in existing_litellm_providers:
continue
normalized_fields = [_normalize_field(field) for field in _FALLBACK_FIELDS]
provider_display_name = provider_enum.value.replace("_", " ").title()
providers.append(
ProviderCreateInfo(
provider=provider_enum.name,
provider_display_name=provider_display_name,
litellm_provider=litellm_provider_value,
default_model_placeholder=DEFAULT_MODEL_PLACEHOLDER,
credential_fields=normalized_fields,
)
)
providers.sort(key=lambda item: item.provider_display_name.lower())
return providers
litellm/proxy/public_endpoints/public_endpoints.py
+20 -1
View File
@@ -3,11 +3,17 @@ from typing import List
from fastapi import APIRouter, Depends, HTTPException
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.public_endpoints.provider_create_metadata import (
get_provider_create_metadata,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
ModelGroupInfoProxy,
)
from litellm.types.proxy.public_endpoints.public_endpoints import PublicModelHubInfo
from litellm.types.proxy.public_endpoints.public_endpoints import (
PublicModelHubInfo,
ProviderCreateInfo,
)
from litellm.types.utils import LlmProviders
router = APIRouter()
@@ -74,3 +80,16 @@ async def get_supported_providers() -> List[str]:
"""
return sorted(provider.value for provider in LlmProviders)
@router.get(
"/public/providers/fields",
tags=["public", "providers"],
response_model=List[ProviderCreateInfo],
)
async def get_provider_fields() -> List[ProviderCreateInfo]:
"""
Return provider metadata required by the dashboard create-model flow.
"""
return get_provider_create_metadata()
litellm/proxy/spend_tracking/spend_management_endpoints.py
+46 -5
View File
@@ -2968,10 +2968,30 @@ async def ui_view_session_spend_logs(
session_id: str = fastapi.Query(
description="Get all spend logs for a particular session",
),
page: int = fastapi.Query(
default=1,
ge=1,
description="Page number for pagination",
),
page_size: int = fastapi.Query(
default=50,
ge=1,
le=100,
description="Number of items per page",
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Get all spend logs for a particular session
Get paginated spend logs for a particular session.
Returns:
{
"data": List[LiteLLM_SpendLogs],
"total": int,
"page": int,
"page_size": int,
"total_pages": int,
}
"""
from litellm.proxy.proxy_server import prisma_client
@@ -2984,11 +3004,32 @@ async def ui_view_session_spend_logs(
# Build query conditions
where_conditions = {"session_id": session_id}
# Query the database
result = await prisma_client.db.litellm_spendlogs.find_many(
where=where_conditions, order={"startTime": "asc"}
# Calculate pagination offsets
skip = (page - 1) * page_size
# Get total count for pagination metadata
total_records = await prisma_client.db.litellm_spendlogs.count(
where=where_conditions
)
return result
# Query the database with pagination
result = await prisma_client.db.litellm_spendlogs.find_many(
where=where_conditions,
order={"startTime": "asc"},
skip=skip,
take=page_size,
)
total_pages = (total_records + page_size - 1) // page_size
return {
"data": result,
"total": total_records,
"page": page,
"page_size": page_size,
"total_pages": total_pages,
}
except Exception as e:
if isinstance(e, HTTPException):
raise e
litellm/responses/mcp/litellm_proxy_mcp_handler.py
+7 -5
View File
@@ -526,8 +526,10 @@ class LiteLLM_Proxy_MCP_Handler:
else:
assistant_message_content.append(content)
# Add assistant message with content and function calls
if assistant_message_content or function_calls:
# Add assistant message only if there's actual content (not empty)
# For example, gemini requires that function call turns come immediately after user turns,
# so we should not add empty assistant messages
if assistant_message_content:
follow_up_input.append(
{
"type": "message",
@@ -536,9 +538,9 @@ class LiteLLM_Proxy_MCP_Handler:
}
)
# Add function calls after assistant message
for function_call in function_calls:
follow_up_input.append(function_call)
# Add function calls (these can come directly after user message for LLM)
for function_call in function_calls:
follow_up_input.append(function_call)
# Add tool results (function call outputs)
for tool_result in tool_results:
litellm/types/llms/bedrock.py
+2 -1
View File
@@ -679,10 +679,11 @@ class BedrockInputDataConfig(TypedDict):
s3InputDataConfig: BedrockS3InputDataConfig
class BedrockS3OutputDataConfig(TypedDict):
class BedrockS3OutputDataConfig(TypedDict, total=False):
"""S3 output data configuration for Bedrock batch jobs."""
s3Uri: str
s3EncryptionKeyId: Optional[str]
class BedrockOutputDataConfig(TypedDict):
litellm/types/llms/openai.py
+1 -1
View File
@@ -1475,7 +1475,7 @@ ResponsesAPIStreamingResponse = Annotated[
]
REASONING_EFFORT = Literal["minimal", "low", "medium", "high"]
REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high"]
class OpenAIRealtimeStreamSession(TypedDict, total=False):
litellm/types/proxy/management_endpoints/model_management_endpoints.py
+32 -1
View File
@@ -1,4 +1,4 @@
from typing import Dict
from typing import Dict, List
from pydantic import BaseModel, Field
@@ -11,3 +11,34 @@ class ModelGroupInfoProxy(ModelGroupInfo):
class UpdateUsefulLinksRequest(BaseModel):
useful_links: Dict[str, str]
class NewModelGroupRequest(BaseModel):
access_group: str # The access group name (e.g., "production-models")
model_names: List[str] # Existing model groups to include (e.g., ["gpt-4", "claude-3"])
class NewModelGroupResponse(BaseModel):
access_group: str
model_names: List[str]
models_updated: int # Number of models updated
class UpdateModelGroupRequest(BaseModel):
model_names: List[str] # Updated list of model groups to include
class DeleteModelGroupResponse(BaseModel):
access_group: str
models_updated: int # Number of deployments where the access group was removed
message: str
class AccessGroupInfo(BaseModel):
access_group: str
model_names: List[str] # List of model names in this access group
deployment_count: int # Total number of deployments with this access group
class ListAccessGroupsResponse(BaseModel):
access_groups: List[AccessGroupInfo]
litellm/types/proxy/public_endpoints/public_endpoints.py
+20 -1
View File
@@ -1,4 +1,4 @@
from typing import Dict, Optional
from typing import Dict, List, Literal, Optional
from pydantic import BaseModel
@@ -8,3 +8,22 @@ class PublicModelHubInfo(BaseModel):
custom_docs_description: Optional[str]
litellm_version: str
useful_links: Optional[Dict[str, str]]
class ProviderCredentialField(BaseModel):
key: str
label: str
placeholder: Optional[str] = None
tooltip: Optional[str] = None
required: bool = False
field_type: Literal["text", "password", "select", "upload"] = "text"
options: Optional[List[str]] = None
default_value: Optional[str] = None
class ProviderCreateInfo(BaseModel):
provider: str
provider_display_name: str
litellm_provider: str
credential_fields: List[ProviderCredentialField]
default_model_placeholder: Optional[str] = None
litellm/types/router.py
+2
View File
@@ -205,6 +205,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
# Batch/File API Params
s3_bucket_name: Optional[str] = None
s3_encryption_key_id: Optional[str] = None
gcs_bucket_name: Optional[str] = None
# Vector Store Params
@@ -262,6 +263,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
auto_router_embedding_model: Optional[str] = None,
# Batch/File API Params
s3_bucket_name: Optional[str] = None,
s3_encryption_key_id: Optional[str] = None,
gcs_bucket_name: Optional[str] = None,
**params,
):
model_prices_and_context_window.json
+77 -5
View File
@@ -8523,6 +8523,14 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/flux/schnell": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.003,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/imagen4/preview": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
@@ -8531,6 +8539,22 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/imagen4/preview/fast": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.02,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/imagen4/preview/ultra": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.06,
"supported_endpoints": [
"/v1/images/generations"
]
},
"fal_ai/fal-ai/recraft/v3/text-to-image": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
@@ -23408,6 +23432,19 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
"vertex_ai/moonshotai/kimi-k2-thinking-maas": {
"input_cost_per_token": 6e-07,
"litellm_provider": "vertex_ai-moonshot_models",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_web_search": true
},
"vertex_ai/mistral-medium-3": {
"input_cost_per_token": 4e-07,
"litellm_provider": "vertex_ai-mistral_models",
@@ -23744,6 +23781,22 @@
"mode": "embedding",
"output_cost_per_token": 0.0
},
"voyage/voyage-3.5": {
"input_cost_per_token": 6e-08,
"litellm_provider": "voyage",
"max_input_tokens": 32000,
"max_tokens": 32000,
"mode": "embedding",
"output_cost_per_token": 0.0
},
"voyage/voyage-3.5-lite": {
"input_cost_per_token": 2e-08,
"litellm_provider": "voyage",
"max_input_tokens": 32000,
"max_tokens": 32000,
"mode": "embedding",
"output_cost_per_token": 0.0
},
"voyage/voyage-code-2": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "voyage",
@@ -24789,7 +24842,9 @@
"1280x720",
"720x1280"
],
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
"metadata": {
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
}
},
"runwayml/gen4_aleph": {
"litellm_provider": "runwayml",
@@ -24807,7 +24862,9 @@
"1280x720",
"720x1280"
],
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
"metadata": {
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
}
},
"runwayml/gen3a_turbo": {
"litellm_provider": "runwayml",
@@ -24825,7 +24882,9 @@
"1280x720",
"720x1280"
],
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
"metadata": {
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
}
},
"runwayml/gen4_image": {
"litellm_provider": "runwayml",
@@ -24844,7 +24903,9 @@
"1280x720",
"1920x1080"
],
"comment": "5 credits per 720p image or 8 credits per 1080p image @ $0.01 per credit. Using 5 credits ($0.05) as base cost"
"metadata": {
"comment": "5 credits per 720p image or 8 credits per 1080p image @ $0.01 per credit. Using 5 credits ($0.05) as base cost"
}
},
"runwayml/gen4_image_turbo": {
"litellm_provider": "runwayml",
@@ -24863,6 +24924,17 @@
"1280x720",
"1920x1080"
],
"comment": "2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"
"metadata": {
"comment": "2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"
}
},
"runwayml/eleven_multilingual_v2": {
"litellm_provider": "runwayml",
"mode": "audio_speech",
"input_cost_per_character": 3e-07,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"metadata": {
"comment": "Estimated cost based on standard TTS pricing. RunwayML uses ElevenLabs models."
}
}
}
poetry.lock
Generated
+641 -110
View File
File diff suppressed because it is too large Load Diff
pyproject.toml
+3 -3
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.79.3"
version = "1.79.4"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@@ -59,7 +59,7 @@ websockets = {version = "^13.1.0", optional = true}
boto3 = {version = "1.36.0", optional = true}
redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
mcp = {version = "^1.10.0", optional = true, python = ">=3.10"}
litellm-proxy-extras = {version = "0.4.3", optional = true}
litellm-proxy-extras = {version = "0.4.4", optional = true}
rich = {version = "13.7.1", optional = true}
litellm-enterprise = {version = "0.1.20", optional = true}
diskcache = {version = "^5.6.1", optional = true}
@@ -159,7 +159,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.79.3"
version = "1.79.4"
version_files = [
"pyproject.toml:^version"
]
requirements.txt
+1 -1
View File
@@ -43,7 +43,7 @@ sentry_sdk==2.21.0 # for sentry error handling
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
cryptography==44.0.1
tzdata==2025.1 # IANA time zone database
litellm-proxy-extras==0.4.3 # for proxy extras - e.g. prisma migrations
litellm-proxy-extras==0.4.4 # for proxy extras - e.g. prisma migrations
### LITELLM PACKAGE DEPENDENCIES
python-dotenv==1.0.1 # for env
tiktoken==0.8.0 # for calculating usage
tests/audio_tests/runwayml_speech.mp3
BIN
View File
Binary file not shown.
tests/audio_tests/test_audio_speech.py
+1 -1
View File
@@ -426,7 +426,7 @@ async def test_runwayml_tts_async():
assert speech_file_path.exists()
assert speech_file_path.stat().st_size > 0
print(f"Azure TTS audio saved to: {speech_file_path}")
print(f"RunwayML TTS audio saved to: {speech_file_path}")
# assert response cost is greater than 0
print("Response cost: ", response._hidden_params["response_cost"])
tests/batches_tests/test_bedrock_files_and_batches.py
+50
View File
@@ -193,3 +193,53 @@ async def test_bedrock_retrieve_batch():
assert batch_response.input_file_id == "s3://test-bucket/input/test-input.jsonl"
assert batch_response.output_file_id == "s3://test-bucket/output/"
def test_bedrock_batch_with_encryption_key_in_post_request():
"""
Test that s3_encryption_key_id is included in the AWS POST request payload.
"""
import json
import litellm
test_kms_key_id = "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012"
captured_request_body = None
def mock_post(*args, **kwargs):
nonlocal captured_request_body
if "data" in kwargs:
captured_request_body = kwargs["data"]
mock_response = MagicMock()
mock_response.json.return_value = {
"jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job",
"jobName": "test-job",
"status": "Submitted"
}
mock_response.status_code = 200
mock_response.raise_for_status.return_value = None
return mock_response
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", side_effect=mock_post):
response = litellm.create_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="s3://test-bucket/input/test.jsonl",
custom_llm_provider="bedrock",
model="us.anthropic.claude-3-5-sonnet-20240620-v1:0",
s3_encryption_key_id=test_kms_key_id,
aws_batch_role_arn="arn:aws:iam::123456789012:role/test-role"
)
assert captured_request_body is not None, "Request body was not captured"
request_data = json.loads(captured_request_body)
print("REQUEST DATA to bedrock batch creation", json.dumps(request_data, indent=4))
assert "outputDataConfig" in request_data
assert "s3OutputDataConfig" in request_data["outputDataConfig"]
assert "s3EncryptionKeyId" in request_data["outputDataConfig"]["s3OutputDataConfig"]
assert request_data["outputDataConfig"]["s3OutputDataConfig"]["s3EncryptionKeyId"] == test_kms_key_id
print("SUCCESS: s3_encryption_key_id properly included in AWS POST request")
tests/image_gen_tests/test_fal_ai_image_generation.py
+1
View File
@@ -14,6 +14,7 @@ from litellm import aimage_generation
"model",
[
"fal_ai/fal-ai/flux-pro/v1.1-ultra",
"fal_ai/fal-ai/flux/schnell",
"fal_ai/fal-ai/recraft/v3/text-to-image",
"fal_ai/bria/text-to-image/3.2",
"fal_ai/fal-ai/stable-diffusion-v35-medium"
tests/llm_translation/test_openai.py
+42
View File
@@ -337,6 +337,48 @@ def test_openai_max_retries_0(mock_get_openai_client):
assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0
@patch("litellm.main.openai_chat_completions._get_openai_client")
def test_openai_image_generation_forwards_organization(mock_get_openai_client):
"""Ensure organization flows to OpenAI client for image generation."""
class _DummyImages:
def generate(self, **kwargs): # type: ignore
class _Resp:
def model_dump(self_inner): # minimal OpenAI ImagesResponse shape
return {
"created": 123,
"data": [{"url": "http://example.com/image.png"}],
"usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
}
return _Resp()
class _DummyClient:
def __init__(self):
self.api_key = "sk-test"
class _BaseURL:
_uri_reference = "https://api.openai.com/v1"
self._base_url = _BaseURL()
self.images = _DummyImages()
mock_get_openai_client.return_value = _DummyClient()
org = "org_test_123"
resp = litellm.image_generation(
model="gpt-image-1",
prompt="A cute baby sea otter",
organization=org,
)
# Assert organization forwarded into OpenAI client factory
assert mock_get_openai_client.call_args.kwargs.get("organization") == org
# Basic sanity on response shape
assert hasattr(resp, "data") and len(resp.data) == 1
@pytest.mark.parametrize("model", ["o1", "o3-mini"])
def test_o1_parallel_tool_calls(model):
litellm.completion(
tests/llm_translation/test_sambanova_chat_transformation.py
+127
View File
@@ -0,0 +1,127 @@
"""
Unit tests for SambaNova chat message transformation
"""
import pytest
from litellm.llms.sambanova.chat import SambanovaConfig
class TestSambanovaContentListHandling:
"""
Test that SambaNova properly transforms content lists to strings
"""
def test_content_list_to_string_transformation(self):
"""
Test content list with text objects is converted to string.
SambaNova API doesn't support content as a list - only string content.
"""
config = SambanovaConfig()
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello, how are you?"}
]
}
]
transformed_messages = config._transform_messages(
messages=messages,
model="sambanova/gpt-oss-120b",
is_async=False
)
assert len(transformed_messages) == 1
assert transformed_messages[0]["role"] == "user"
assert isinstance(transformed_messages[0]["content"], str)
assert transformed_messages[0]["content"] == "Hello, how are you?"
def test_content_list_multiple_text_blocks(self):
"""
Test content list with multiple text blocks is converted to concatenated string.
"""
config = SambanovaConfig()
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello, "},
{"type": "text", "text": "how are you?"}
]
}
]
transformed_messages = config._transform_messages(
messages=messages,
model="sambanova/gpt-oss-120b",
is_async=False
)
assert transformed_messages[0]["content"] == "Hello, how are you?"
def test_string_content_unchanged(self):
"""
Test that string content is passed through unchanged.
"""
config = SambanovaConfig()
messages = [
{
"role": "user",
"content": "Hello, how are you?"
}
]
transformed_messages = config._transform_messages(
messages=messages,
model="sambanova/gpt-oss-120b",
is_async=False
)
assert transformed_messages[0]["content"] == "Hello, how are you?"
def test_multiple_messages_transformation(self):
"""
Test transformation of multiple messages with mixed content types.
"""
config = SambanovaConfig()
messages = [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": [
{"type": "text", "text": "What is the weather?"}
]
},
{
"role": "assistant",
"content": "I need your location."
},
{
"role": "user",
"content": [
{"type": "text", "text": "I'm in "},
{"type": "text", "text": "San Francisco"}
]
}
]
transformed_messages = config._transform_messages(
messages=messages,
model="sambanova/gpt-oss-120b",
is_async=False
)
assert len(transformed_messages) == 4
assert transformed_messages[0]["content"] == "You are a helpful assistant."
assert transformed_messages[1]["content"] == "What is the weather?"
assert transformed_messages[2]["content"] == "I need your location."
assert transformed_messages[3]["content"] == "I'm in San Francisco"
tests/proxy_unit_tests/test_key_generate_prisma.py
+7 -1
View File
@@ -132,6 +132,11 @@ def prisma_client():
### add connection pool + pool timeout args
params = {"connection_limit": 100, "pool_timeout": 60}
database_url = os.getenv("DATABASE_URL")
# If DATABASE_URL is not set, use a default test database URL
if not database_url:
database_url = "postgresql://postgres:postgres@localhost:5432/circle_test"
modified_url = append_query_params(database_url, params)
os.environ["DATABASE_URL"] = modified_url
@@ -666,7 +671,8 @@ def test_call_with_end_user_over_budget(prisma_client):
except Exception as e:
print(f"raised error: {e}, traceback: {traceback.format_exc()}")
error_detail = e.message
assert "Budget has been exceeded! Current" in error_detail
assert "ExceededBudget: End User=" in error_detail
assert "over budget" in error_detail
assert isinstance(e, ProxyException)
assert e.type == ProxyErrorTypes.budget_exceeded
print(vars(e))
tests/proxy_unit_tests/test_proxy_exception_mapping.py
+3
View File
@@ -157,6 +157,9 @@ def test_embedding_auth_exception_azure(mock_aembedding, client):
metadata=mock.ANY,
proxy_server_request=mock.ANY,
secret_fields=mock.ANY,
request_timeout=mock.ANY,
litellm_call_id=mock.ANY,
litellm_logging_obj=mock.ANY,
)
print("Response from proxy=", response)
tests/proxy_unit_tests/test_proxy_server.py
+25 -10
View File
@@ -545,17 +545,22 @@ def test_embedding(mock_aembedding, client_no_auth):
"input": ["good morning from litellm"],
}
pre_call_return_value = {
**test_data,
"metadata": {"source": "unit-test"},
"proxy_server_request": {"path": "/v1/embeddings"},
"secret_fields": [],
}
async def _pre_call_hook_side_effect(**kwargs):
data = kwargs["data"]
metadata = {**(data.get("metadata") or {}), "source": "unit-test"}
data["metadata"] = metadata
proxy_request = {**(data.get("proxy_server_request") or {})}
proxy_request["path"] = "/v1/embeddings"
data["proxy_server_request"] = proxy_request
return data
async def _post_call_success_side_effect(**kwargs):
return kwargs["response"]
with patch.object(
litellm.proxy.proxy_server.proxy_logging_obj,
"pre_call_hook",
new=AsyncMock(return_value=pre_call_return_value),
new=AsyncMock(side_effect=_pre_call_hook_side_effect),
) as mock_pre_call_hook, patch.object(
litellm.proxy.proxy_server.proxy_logging_obj,
"during_call_hook",
@@ -563,7 +568,7 @@ def test_embedding(mock_aembedding, client_no_auth):
) as mock_during_hook, patch.object(
litellm.proxy.proxy_server.proxy_logging_obj,
"post_call_success_hook",
new=AsyncMock(return_value=None),
new=AsyncMock(side_effect=_post_call_success_side_effect),
):
response = client_no_auth.post("/v1/embeddings", json=test_data)
@@ -571,6 +576,9 @@ def test_embedding(mock_aembedding, client_no_auth):
model="azure/text-embedding-ada-002",
input=["good morning from litellm"],
specific_deployment=True,
litellm_call_id=mock.ANY,
litellm_logging_obj=mock.ANY,
request_timeout=mock.ANY,
metadata=mock.ANY,
proxy_server_request=mock.ANY,
secret_fields=mock.ANY,
@@ -580,6 +588,9 @@ def test_embedding(mock_aembedding, client_no_auth):
print(len(result["data"][0]["embedding"]))
assert len(result["data"][0]["embedding"]) > 10 # this usually has len==1536 so
call_metadata = mock_aembedding.call_args.kwargs["metadata"]
assert call_metadata.get("source") == "unit-test"
pre_call_kwargs = mock_pre_call_hook.await_args_list[0].kwargs
assert (
pre_call_kwargs.get("call_type") == "aembedding"
@@ -587,8 +598,8 @@ def test_embedding(mock_aembedding, client_no_auth):
during_call_kwargs = mock_during_hook.await_args_list[0].kwargs
assert (
during_call_kwargs.get("call_type") == "aembedding"
), f"expected during_call_hook to receive call_type='aembedding', got {during_call_kwargs.get('call_type')}"
during_call_kwargs.get("call_type") == "embeddings"
), f"expected during_call_hook to receive call_type='embeddings', got {during_call_kwargs.get('call_type')}"
except Exception as e:
pytest.fail(f"LiteLLM Proxy test failed. Exception - {str(e)}")
@@ -609,11 +620,15 @@ def test_bedrock_embedding(mock_aembedding, client_no_auth):
mock_aembedding.assert_called_once_with(
model="amazon-embeddings",
input=["good morning from litellm"],
litellm_call_id=mock.ANY,
litellm_logging_obj=mock.ANY,
request_timeout=mock.ANY,
metadata=mock.ANY,
proxy_server_request=mock.ANY,
secret_fields=mock.ANY,
)
assert response.status_code == 200
print(response.status_code, response.text)
result = response.json()
print(len(result["data"][0]["embedding"]))
assert len(result["data"][0]["embedding"]) > 10 # this usually has len==1536 so
tests/test_litellm/llms/openai/test_gpt5_transformation.py
+30
View File
@@ -153,3 +153,33 @@ def test_gpt5_codex_supports_function_calling(config: OpenAIConfig):
assert "functions" in supported_params
assert "function_call" in supported_params
assert "tools" in supported_params
def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig):
"""Test that GPT-5.1 supports reasoning_effort='none' parameter.
Related issue: https://github.com/BerriAI/litellm/issues/16633
GPT-5.1 introduced 'none' as the new default reasoning effort setting
for faster, lower-latency responses.
"""
# Test that reasoning_effort is a supported parameter
assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5.1")
# Test that reasoning_effort="none" passes through correctly
params = config.map_openai_params(
non_default_params={"reasoning_effort": "none"},
optional_params={},
model="gpt-5.1",
drop_params=False,
)
assert params["reasoning_effort"] == "none"
# Test with other valid values for GPT-5.1
for effort in ["low", "medium", "high"]:
params = config.map_openai_params(
non_default_params={"reasoning_effort": effort},
optional_params={},
model="gpt-5.1",
drop_params=False,
)
assert params["reasoning_effort"] == effort
tests/test_litellm/llms/openai_like/chat/test_transformation.py → tests/test_litellm/llms/openai_like/chat/test_openai_like_chat_transformation.py
View File
tests/test_litellm/llms/vertex_ai/test_bge_embedding.py
-251
View File
@@ -1,251 +0,0 @@
"""
Test BGE embeddings with Vertex AI using custom api_base.
This test ensures that BGE embeddings work correctly with Vertex AI
and that the request body is properly formatted.
"""
import json
import os
import sys
from unittest.mock import MagicMock, patch
sys.path.insert(
0, os.path.abspath("../../../..")
)
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def test_vertex_ai_bge_embedding_with_custom_api_base():
"""
Test Vertex AI BGE embeddings with custom api_base.
This test verifies that when using a BGE model with Vertex AI and
a custom api_base, the request is properly formatted and sent to
the correct endpoint.
"""
client = HTTPHandler()
def mock_auth_token(*args, **kwargs):
return "fake-token", "fake-project"
with patch.object(client, "post") as mock_post, patch(
"litellm.llms.vertex_ai.vertex_embeddings.embedding_handler.VertexEmbedding._ensure_access_token",
side_effect=mock_auth_token
):
mock_response = MagicMock()
mock_response.status_code = 200
# BGE models return embeddings directly as arrays, not wrapped in objects
mock_response.json.return_value = {
"predictions": [
[0.1, 0.2, 0.3, 0.4, 0.5],
[0.6, 0.7, 0.8, 0.9, 1.0]
],
"deployedModelId": "849506872875548672",
"model": "projects/1060139831167/locations/us-central1/models/baai_bge-small-en-v1.5",
"modelDisplayName": "baai_bge-small-en-v1.5",
"modelVersionId": "1"
}
mock_post.return_value = mock_response
response = litellm.embedding(
model="vertex_ai/bge-small-en-v1.5",
input=["Hello", "World"],
api_base="http://10.96.32.8",
client=client
)
mock_post.assert_called_once()
call_args = mock_post.call_args
kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1]
if "url" in kwargs:
api_url_called = kwargs["url"]
elif len(call_args[0]) > 0:
api_url_called = call_args[0][0]
else:
api_url_called = "Unknown"
# Vertex AI may use 'json' or 'data' parameter
if "json" in kwargs:
request_data = kwargs["json"]
elif "data" in kwargs:
request_data = json.loads(kwargs["data"])
else:
request_data = {}
print("\n" + "="*50)
print("Mock Request Body Received:")
print("="*50)
print(json.dumps(request_data, indent=2))
print("="*50)
print(f"API Base: {api_url_called}")
print("="*50 + "\n")
assert "instances" in request_data
assert len(request_data["instances"]) == 2
# BGE models should use "prompt" instead of "content"
assert "prompt" in request_data["instances"][0]
assert request_data["instances"][0]["prompt"] == "Hello"
assert "prompt" in request_data["instances"][1]
assert request_data["instances"][1]["prompt"] == "World"
assert isinstance(response.data, list)
assert len(response.data) == 2
assert "embedding" in response.data[0]
def test_vertex_ai_bge_with_endpoint_id_pattern():
"""
Test BGE with vertex_ai/bge/endpoint_id pattern.
This test verifies that the pattern vertex_ai/bge/204379420394258432
correctly triggers BGE transformations and routes to the endpoint.
"""
client = HTTPHandler()
def mock_auth_token(*args, **kwargs):
return "fake-token", "fake-project"
with patch.object(client, "post") as mock_post, patch(
"litellm.llms.vertex_ai.vertex_embeddings.embedding_handler.VertexEmbedding._ensure_access_token",
side_effect=mock_auth_token
):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"predictions": [
[0.1, 0.2, 0.3, 0.4, 0.5],
[0.6, 0.7, 0.8, 0.9, 1.0]
],
"deployedModelId": "204379420394258432",
"model": "projects/1060139831167/locations/europe-west4/models/baai_bge-base-en",
"modelDisplayName": "baai_bge-base-en",
"modelVersionId": "1"
}
mock_post.return_value = mock_response
response = litellm.embedding(
model="vertex_ai/bge/204379420394258432",
input=["Hello", "World"],
vertex_project="1060139831167",
vertex_location="europe-west4",
client=client
)
mock_post.assert_called_once()
call_args = mock_post.call_args
kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1]
if "url" in kwargs:
api_url_called = kwargs["url"]
elif len(call_args[0]) > 0:
api_url_called = call_args[0][0]
else:
api_url_called = "Unknown"
# Vertex AI may use 'json' or 'data' parameter
if "json" in kwargs:
request_data = kwargs["json"]
elif "data" in kwargs:
request_data = json.loads(kwargs["data"])
else:
request_data = {}
print("\n" + "="*50)
print("BGE Endpoint Pattern Test:")
print("="*50)
print(f"Model: vertex_ai/bge/204379420394258432")
print(f"API URL: {api_url_called}")
print("Request Body:")
print(json.dumps(request_data, indent=2))
print("="*50 + "\n")
# Verify URL contains the endpoint ID and uses endpoints/ path
assert "204379420394258432" in api_url_called, f"Endpoint ID not in URL: {api_url_called}"
assert "endpoints" in api_url_called, f"Expected 'endpoints' in URL, got: {api_url_called}"
# Verify BGE-specific request format (uses "prompt" not "content")
assert "instances" in request_data
assert "prompt" in request_data["instances"][0]
assert request_data["instances"][0]["prompt"] == "Hello"
# Verify response
assert isinstance(response.data, list)
assert len(response.data) == 2
def test_vertex_ai_bge_psc_endpoint_url_construction():
"""
Test that BGE models with PSC endpoints construct correct URL without bge/ prefix.
Verifies that vertex_ai/bge/378943383978115072 with api_base http://10.128.16.2
constructs URL: http://10.128.16.2/v1/projects/{project}/locations/{location}/endpoints/378943383978115072:predict
The bge/ prefix should be stripped from the endpoint URL.
"""
client = HTTPHandler()
def mock_auth_token(*args, **kwargs):
return "fake-token", "gen-lang-client-0682925754"
with patch.object(client, "post") as mock_post, patch(
"litellm.llms.vertex_ai.vertex_embeddings.embedding_handler.VertexEmbedding._ensure_access_token",
side_effect=mock_auth_token
):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"predictions": [
[0.1, 0.2, 0.3, 0.4, 0.5]
]
}
mock_post.return_value = mock_response
response = litellm.embedding(
model="vertex_ai/bge/378943383978115072",
input=["The food was delicious and the waiter.."],
api_base="http://10.128.16.2",
vertex_project="gen-lang-client-0682925754",
vertex_location="us-central1",
client=client
)
mock_post.assert_called_once()
call_args = mock_post.call_args
kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1]
if "url" in kwargs:
api_url_called = kwargs["url"]
elif len(call_args[0]) > 0:
api_url_called = call_args[0][0]
else:
api_url_called = "Unknown"
print("\n" + "="*50)
print("PSC Endpoint URL Construction Test:")
print("="*50)
print(f"Model: vertex_ai/bge/378943383978115072")
print(f"API Base: http://10.128.16.2")
print(f"Constructed URL: {api_url_called}")
print("="*50 + "\n")
# Verify the URL is constructed correctly
expected_url = "http://10.128.16.2/v1/projects/gen-lang-client-0682925754/locations/us-central1/endpoints/378943383978115072:predict"
assert api_url_called == expected_url, f"Expected URL: {expected_url}, Got: {api_url_called}"
# Verify bge/ prefix is NOT in the URL
assert "bge/" not in api_url_called, f"URL should not contain 'bge/' prefix: {api_url_called}"
# Verify response works
assert isinstance(response.data, list)
assert len(response.data) == 1
tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py
-111
View File
@@ -1,111 +0,0 @@
"""
Test BGE response transformation validation.
This test verifies that the BGE response transformer properly validates
and handles different response formats.
"""
import os
import sys
sys.path.insert(
0, os.path.abspath("../../../..")
)
import pytest
from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig
from litellm.types.utils import EmbeddingResponse
def test_is_bge_model_detection():
"""
Test BGE model detection for post-provider-split patterns.
After main.py splits the provider, model strings are passed without the provider prefix.
Model name transformation (bge/ -> numeric ID) is handled in common_utils._get_vertex_url().
"""
# Should detect BGE models (after provider split)
assert VertexBGEConfig.is_bge_model("bge-small-en-v1.5") is True
assert VertexBGEConfig.is_bge_model("bge/204379420394258432") is True
assert VertexBGEConfig.is_bge_model("BGE-large-en-v1.5") is True # case insensitive
# Should not detect non-BGE models
assert VertexBGEConfig.is_bge_model("textembedding-gecko") is False
assert VertexBGEConfig.is_bge_model("gemma") is False
assert VertexBGEConfig.is_bge_model("123456789") is False
def test_bge_response_transformation_success():
"""
Test successful BGE response transformation.
Verifies that a valid BGE response is properly transformed
to OpenAI format.
"""
response = {
"predictions": [
[0.1, 0.2, 0.3],
[0.4, 0.5, 0.6]
],
"deployedModelId": "123456",
"model": "projects/test/models/bge-base"
}
model_response = EmbeddingResponse()
result = VertexBGEConfig.transform_response(
response=response,
model="bge-small-en-v1.5",
model_response=model_response
)
assert result.object == "list"
assert len(result.data) == 2
assert result.data[0]["embedding"] == [0.1, 0.2, 0.3]
assert result.data[1]["embedding"] == [0.4, 0.5, 0.6]
assert result.data[0]["index"] == 0
assert result.data[1]["index"] == 1
assert result.model == "bge-small-en-v1.5"
def test_bge_response_missing_predictions():
"""
Test BGE response transformation with missing predictions field.
Verifies that a KeyError is raised when the response doesn't
contain the required 'predictions' field.
"""
response = {
"deployedModelId": "123456",
"model": "projects/test/models/bge-base"
}
model_response = EmbeddingResponse()
with pytest.raises(KeyError, match="Response missing 'predictions' field"):
VertexBGEConfig.transform_response(
response=response,
model="bge-small-en-v1.5",
model_response=model_response
)
def test_bge_response_invalid_predictions_type():
"""
Test BGE response transformation with invalid predictions type.
Verifies that a ValueError is raised when predictions is not a list.
"""
response = {
"predictions": "not-a-list"
}
model_response = EmbeddingResponse()
with pytest.raises(ValueError, match="Expected 'predictions' to be a list"):
VertexBGEConfig.transform_response(
response=response,
model="bge-small-en-v1.5",
model_response=model_response
)
tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
+17
View File
@@ -967,6 +967,10 @@ async def test_vertex_ai_partner_model_detection():
assert VertexAIPartnerModels.is_vertex_partner_model("meta/llama-3.1-405b")
# Test Minimax models
assert VertexAIPartnerModels.is_vertex_partner_model("minimaxai/minimax-m2-maas")
# Test Moonshot models
assert VertexAIPartnerModels.is_vertex_partner_model(
"moonshotai/kimi-k2-thinking-maas"
)
# Test Gemini models (should NOT be detected as partner model)
assert not VertexAIPartnerModels.is_vertex_partner_model("gemini-1.5-pro")
@@ -989,3 +993,16 @@ def test_vertex_ai_minimax_uses_openai_handler():
assert VertexAIPartnerModels.should_use_openai_handler(
"minimaxai/minimax-m2-maas"
)
def test_vertex_ai_moonshot_uses_openai_handler():
"""
Ensure Moonshot partner models re-use the OpenAI-format handler.
"""
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
VertexAIPartnerModels,
)
assert VertexAIPartnerModels.should_use_openai_handler(
"moonshotai/kimi-k2-thinking-maas"
)
tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py
-258
View File
@@ -1,258 +0,0 @@
"""
Unit tests for Vertex AI Private Service Connect (PSC) endpoint support
Tests that LiteLLM properly constructs URLs when using custom api_base
for PSC endpoints.
"""
import pytest
import sys
import os
# Add the litellm package to the path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../.."))
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
class TestVertexAIPSCEndpointSupport:
"""Test cases for PSC endpoint URL construction"""
def test_psc_endpoint_url_construction_basic(self):
"""Test basic PSC endpoint URL construction for predict endpoint"""
vertex_base = VertexBase()
psc_api_base = "http://10.96.32.8"
endpoint_id = "1234567890"
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=psc_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="predict",
stream=False,
auth_header="test-token",
url="", # This will be replaced
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1",
)
expected_url = f"{psc_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_psc_endpoint_url_construction_with_streaming(self):
"""Test PSC endpoint URL construction with streaming enabled"""
vertex_base = VertexBase()
psc_api_base = "http://10.96.32.8"
endpoint_id = "1234567890"
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=psc_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="streamGenerateContent",
stream=True,
auth_header="test-token",
url="",
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1",
)
expected_url = f"{psc_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:streamGenerateContent?alt=sse"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_psc_endpoint_url_construction_v1beta1(self):
"""Test PSC endpoint URL construction with v1beta1 API version"""
vertex_base = VertexBase()
psc_api_base = "http://10.96.32.8"
endpoint_id = "1234567890"
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=psc_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="predict",
stream=False,
auth_header="test-token",
url="",
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1beta1",
)
expected_url = f"{psc_api_base}/v1beta1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_psc_endpoint_url_with_https(self):
"""Test PSC endpoint URL construction with HTTPS"""
vertex_base = VertexBase()
psc_api_base = "https://10.96.32.8"
endpoint_id = "1234567890"
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=psc_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="predict",
stream=False,
auth_header="test-token",
url="",
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1",
)
expected_url = f"{psc_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_psc_endpoint_with_trailing_slash(self):
"""Test that trailing slashes in api_base are handled correctly"""
vertex_base = VertexBase()
psc_api_base = "http://10.96.32.8/"
endpoint_id = "1234567890"
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=psc_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="predict",
stream=False,
auth_header="test-token",
url="",
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1",
)
# rstrip('/') should remove the trailing slash
expected_url = f"{psc_api_base.rstrip('/')}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_standard_proxy_with_googleapis(self):
"""Test that standard proxies with googleapis.com in URL use simple format"""
vertex_base = VertexBase()
proxy_api_base = "https://my-proxy.googleapis.com"
endpoint_id = "gemini-pro" # Not numeric
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=proxy_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="generateContent",
stream=False,
auth_header="test-token",
url="",
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1",
)
# Should use simple format: api_base:endpoint
expected_url = f"{proxy_api_base}:generateContent"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_custom_proxy_with_numeric_model(self):
"""Test that numeric model IDs trigger PSC-style URL construction"""
vertex_base = VertexBase()
proxy_api_base = "https://my-custom-proxy.example.com"
endpoint_id = "9876543210" # Numeric endpoint ID
project_id = "test-project"
location = "us-central1"
auth_header, url = vertex_base._check_custom_proxy(
api_base=proxy_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="predict",
stream=False,
auth_header="test-token",
url="",
model=endpoint_id,
vertex_project=project_id,
vertex_location=location,
vertex_api_version="v1",
)
# Numeric model should trigger full path construction
expected_url = f"{proxy_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict"
assert (
url == expected_url
), f"Expected {expected_url}, but got {url}"
def test_no_api_base_returns_original_url(self):
"""Test that when api_base is None, the original URL is returned"""
vertex_base = VertexBase()
original_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/test/locations/us-central1/publishers/google/models/gemini-pro:generateContent"
auth_header, url = vertex_base._check_custom_proxy(
api_base=None,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="generateContent",
stream=False,
auth_header="test-token",
url=original_url,
model="gemini-pro",
vertex_project="test-project",
vertex_location="us-central1",
vertex_api_version="v1",
)
# When api_base is None, original URL should be returned unchanged
assert url == original_url, f"Expected {original_url}, but got {url}"
def test_auth_header_preserved(self):
"""Test that auth_header is properly preserved"""
vertex_base = VertexBase()
psc_api_base = "http://10.96.32.8"
test_auth_header = "Bearer test-token-12345"
auth_header, url = vertex_base._check_custom_proxy(
api_base=psc_api_base,
custom_llm_provider="vertex_ai",
gemini_api_key=None,
endpoint="predict",
stream=False,
auth_header=test_auth_header,
url="",
model="1234567890",
vertex_project="test-project",
vertex_location="us-central1",
vertex_api_version="v1",
)
assert (
auth_header == test_auth_header
), f"Auth header should be preserved, got {auth_header}"
tests/test_litellm/llms/xai/responses/test_transformation.py → tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py
View File
tests/test_litellm/proxy/management_endpoints/test_access_group_management.py
+80
View File
@@ -0,0 +1,80 @@
"""
Test access group management endpoints
"""
import os
import sys
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
sys.path.insert(
0, os.path.abspath("../../../..")
) # Adds the parent directory to the system path
from litellm import Router
@pytest.mark.asyncio
async def test_create_duplicate_access_group_fails():
"""
Test that creating an access group with a name that already exists returns 409 error.
Scenario: User creates "production-models" access group, then tries to create it again.
Should fail with 409 Conflict.
"""
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
create_model_group,
)
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
NewModelGroupRequest,
)
# Mock dependencies - use exact model name (not wildcard)
mock_router = Router(
model_list=[
{
"model_name": "gpt-4", # Exact model name
"litellm_params": {
"model": "gpt-4",
"api_key": "fake-key",
},
}
]
)
mock_prisma = MagicMock()
mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(
return_value=[
MagicMock(
model_id="1",
model_name="gpt-4",
model_info={"access_groups": ["production-models"]}, # Already exists
)
]
)
mock_user = UserAPIKeyAuth(
user_id="test_admin",
user_role=LitellmUserRoles.PROXY_ADMIN,
)
request_data = NewModelGroupRequest(
access_group="production-models",
model_names=["gpt-4"],
)
# Mock the imported dependencies from proxy_server (where they're actually imported from)
with patch("litellm.proxy.proxy_server.llm_router", mock_router), \
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma):
# Should raise 409 Conflict
with pytest.raises(HTTPException) as exc_info:
await create_model_group(data=request_data, user_api_key_dict=mock_user)
assert exc_info.value.status_code == 409
assert "already exists" in str(exc_info.value.detail)
tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
+45
View File
@@ -520,6 +520,51 @@ class TestListMCPServers:
assert mock_server.credentials == {"auth_value": "top-secret"}
assert result.status == "healthy"
@pytest.mark.asyncio
async def test_fetch_single_mcp_server_handles_missing_credentials_field(self):
mock_server = generate_mock_mcp_server_db_record(
server_id="server-2", alias="Server 2"
)
# Simulate ORM object without credentials attribute (e.g., older schema)
delattr(mock_server, "credentials")
mock_prisma_client = MagicMock()
mock_health_result = {
"status": "healthy",
"last_health_check": datetime.now().isoformat(),
"error": None,
}
mock_user_auth = generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN
)
with patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw",
return_value=mock_prisma_client,
), patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server",
AsyncMock(return_value=mock_server),
), patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server",
AsyncMock(return_value=mock_health_result),
), patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
return_value=True,
):
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
fetch_mcp_server,
)
result = await fetch_mcp_server(
server_id="server-2", user_api_key_dict=mock_user_auth
)
assert result.server_id == "server-2"
# credentials attribute should still be absent and no exception raised
assert not hasattr(result, "credentials")
assert result.status == "healthy"
class TestMCPHealthCheckEndpoints:
"""Test MCP health check endpoints"""
tests/test_litellm/proxy/public_endpoints/test_provider_create_metadata.py
+55
View File
@@ -0,0 +1,55 @@
import os
import sys
from copy import deepcopy
import pytest
sys.path.insert(0, os.path.abspath("../../.."))
import litellm.proxy.public_endpoints.provider_create_metadata as pcm # noqa: E402
from litellm.proxy.public_endpoints.provider_create_metadata import ( # noqa: E402
_normalize_field,
get_provider_create_metadata,
)
def test_get_provider_create_metadata_includes_openai_fields():
metadata = get_provider_create_metadata()
openai_info = next(item for item in metadata if item.provider == "OpenAI")
assert openai_info.provider_display_name == "OpenAI"
assert openai_info.litellm_provider == "openai"
keys = {field.key for field in openai_info.credential_fields}
assert {"api_base", "api_key"}.issubset(keys)
def test_get_provider_create_metadata_returns_sorted_display_names():
metadata = get_provider_create_metadata()
display_names = [item.provider_display_name for item in metadata]
assert display_names == sorted(display_names, key=str.lower)
def test_get_provider_create_metadata_uses_fallback_fields(monkeypatch):
overridden_fields = deepcopy(pcm.PROVIDER_CREDENTIAL_FIELDS)
overridden_fields.pop("Azure", None)
monkeypatch.setattr(pcm, "PROVIDER_CREDENTIAL_FIELDS", overridden_fields)
metadata = get_provider_create_metadata()
azure_info = next(item for item in metadata if item.provider == "Azure")
fallback_keys = [field.key for field in azure_info.credential_fields]
assert fallback_keys == ["api_base", "api_key"]
assert all(field.required is False for field in azure_info.credential_fields)
def test_normalize_field_applies_defaults():
normalized = _normalize_field({"key": "api_key", "label": "API Key"})
assert normalized.key == "api_key"
assert normalized.label == "API Key"
assert normalized.field_type == "text"
assert normalized.required is False
assert normalized.placeholder is None
assert normalized.options is None
tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py
+41
View File
@@ -23,3 +23,44 @@ def test_get_supported_providers_returns_enum_values():
expected_providers = sorted(provider.value for provider in LlmProviders)
assert response.json() == expected_providers
def test_get_provider_fields_returns_metadata():
app = FastAPI()
app.include_router(router)
client = TestClient(app)
response = client.get("/public/providers/fields")
assert response.status_code == 200
payload = response.json()
assert isinstance(payload, list)
provider_lookup = {item["provider"]: item for item in payload}
assert "OpenAI" in provider_lookup
openai_fields = provider_lookup["OpenAI"]
assert openai_fields["provider_display_name"] == "OpenAI"
assert openai_fields["litellm_provider"] == "openai"
credential_keys = {field["key"] for field in openai_fields["credential_fields"]}
assert {"api_base", "api_key"}.issubset(credential_keys)
# Every provider exposed by `/public/providers` (i.e. every LlmProviders value)
# should have a corresponding entry in `/public/providers/fields`.
expected_litellm_providers = {provider.value for provider in LlmProviders}
actual_litellm_providers = {item["litellm_provider"] for item in payload}
assert expected_litellm_providers.issubset(actual_litellm_providers)
# Sanity check for runwayml specifically – it should be present and use the
# default API base + API key credential fields at minimum.
runway_entries = [
item for item in payload if item["litellm_provider"] == "runwayml"
]
assert (
len(runway_entries) >= 1
), "Expected runwayml provider metadata in /public/providers/fields"
runway_credential_keys = {
field["key"] for field in runway_entries[0]["credential_fields"]
}
assert {"api_base", "api_key"}.issubset(runway_credential_keys)
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
+53
View File
@@ -628,6 +628,59 @@ async def test_ui_view_spend_logs_pagination(client, monkeypatch):
assert data["page"] == 2
@pytest.mark.asyncio
async def test_ui_view_session_spend_logs_pagination(client, monkeypatch):
mock_spend_logs = [
{
"id": "log1",
"request_id": "req1",
"session_id": "session-123",
"startTime": "2024-01-01T00:00:00Z",
},
{
"id": "log2",
"request_id": "req2",
"session_id": "session-123",
"startTime": "2024-01-02T00:00:00Z",
},
]
class MockDB:
async def count(self, *args, **kwargs):
assert kwargs.get("where") == {"session_id": "session-123"}
return len(mock_spend_logs)
async def find_many(self, *args, **kwargs):
assert kwargs.get("where") == {"session_id": "session-123"}
assert kwargs.get("order") == {"startTime": "asc"}
assert kwargs.get("skip") == 1 # page=2, page_size=1
assert kwargs.get("take") == 1
return [mock_spend_logs[1]]
class MockPrismaClient:
def __init__(self):
self.db = MockDB()
self.db.litellm_spendlogs = self.db
mock_prisma_client = MockPrismaClient()
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
response = client.get(
"/spend/logs/session/ui",
params={"session_id": "session-123", "page": 2, "page_size": 1},
headers={"Authorization": "Bearer sk-test"},
)
assert response.status_code == 200
data = response.json()
assert data["total"] == 2
assert data["page"] == 2
assert data["page_size"] == 1
assert data["total_pages"] == 2
assert len(data["data"]) == 1
assert data["data"][0]["request_id"] == "req2"
@pytest.mark.asyncio
async def test_ui_view_spend_logs_date_range_filter(client, monkeypatch):
# Create mock data with different dates
tests/test_litellm/proxy/test_proxy_server.py
+14 -7
View File
@@ -325,13 +325,20 @@ def test_embedding_input_array_of_tokens(mock_aembedding, client_no_auth):
response = client_no_auth.post("/v1/embeddings", json=test_data)
mock_aembedding.assert_called_once_with(
model="vllm_embed_model",
input=[[2046, 13269, 158208]],
metadata=mock.ANY,
proxy_server_request=mock.ANY,
secret_fields=mock.ANY,
)
# DEPRECATED - mock_aembedding.assert_called_once_with is too strict, and will fail when new kwargs are added to embeddings
# mock_aembedding.assert_called_once_with(
# model="vllm_embed_model",
# input=[[2046, 13269, 158208]],
# metadata=mock.ANY,
# proxy_server_request=mock.ANY,
# secret_fields=mock.ANY,
# )
# Assert that aembedding was called, and that input was not modified
mock_aembedding.assert_called_once()
call_args, call_kwargs = mock_aembedding.call_args
assert call_kwargs["model"] == "vllm_embed_model"
assert call_kwargs["input"] == [[2046, 13269, 158208]]
assert response.status_code == 200
result = response.json()
print(len(result["data"][0]["embedding"]))
tests/test_litellm/test_responses_id_security.py
+18 -12
View File
@@ -115,14 +115,17 @@ class TestEncryptResponseId:
"litellm.proxy.hooks.responses_id_security.encrypt_value_helper"
) as mock_encrypt:
mock_encrypt.return_value = "encrypted_base64_value"
with patch.object(
responses_id_security, "_get_signing_key", return_value="test-key"
):
result = responses_id_security._encrypt_response_id(
mock_response, mock_user_api_key_dict
)
result = responses_id_security._encrypt_response_id(
mock_response, mock_user_api_key_dict
)
assert result.id == "resp_encrypted_base64_value"
assert result.id.startswith("resp_")
mock_encrypt.assert_called_once()
assert result.id == "resp_encrypted_base64_value"
assert result.id.startswith("resp_")
mock_encrypt.assert_called_once()
def test_encrypt_response_id_maintains_prefix(
self, responses_id_security, mock_user_api_key_dict
@@ -136,12 +139,15 @@ class TestEncryptResponseId:
"litellm.proxy.hooks.responses_id_security.encrypt_value_helper"
) as mock_encrypt:
mock_encrypt.return_value = "encrypted_value_456"
with patch.object(
responses_id_security, "_get_signing_key", return_value="test-key"
):
result = responses_id_security._encrypt_response_id(
mock_response, mock_user_api_key_dict
)
result = responses_id_security._encrypt_response_id(
mock_response, mock_user_api_key_dict
)
assert result.id.startswith("resp_")
assert result.id.startswith("resp_")
class TestCheckUserAccessToResponseId:
ui/litellm-dashboard/public/assets/logos/runway.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.0 KiB

ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.tsx
+1 -11
View File
@@ -1,14 +1,4 @@
import {
Button,
Icon,
Table,
TableBody,
TableCell,
TableHead,
TableHeaderCell,
TableRow,
Text,
} from "@tremor/react";
import { Button, Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Text } from "@tremor/react";
import { Tooltip } from "antd";
import { formatNumberWithCommas } from "@/utils/dataUtils";
import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline";
ui/litellm-dashboard/src/components/OldTeams.tsx
+23 -14
View File
@@ -755,6 +755,7 @@ const Teams: React.FC<TeamProps> = ({
<TableHeaderCell>Models</TableHeaderCell>
<TableHeaderCell>Organization</TableHeaderCell>
<TableHeaderCell>Info</TableHeaderCell>
<TableHeaderCell>Actions</TableHeaderCell>
</TableRow>
</TableHead>
@@ -937,20 +938,28 @@ const Teams: React.FC<TeamProps> = ({
<TableCell>
{userRole == "Admin" ? (
<>
<Icon
icon={PencilAltIcon}
size="sm"
onClick={() => {
setSelectedTeamId(team.team_id);
setEditTeam(true);
}}
/>
<Icon
onClick={() => handleDelete(team.team_id)}
icon={TrashIcon}
size="sm"
data-testid="delete-team-button"
/>
<Tooltip title="Edit team">
{" "}
<Icon
icon={PencilAltIcon}
size="sm"
className="cursor-pointer hover:text-blue-600"
onClick={() => {
setSelectedTeamId(team.team_id);
setEditTeam(true);
}}
/>
</Tooltip>
<Tooltip title="Delete team">
{" "}
<Icon
onClick={() => handleDelete(team.team_id)}
icon={TrashIcon}
size="sm"
className="cursor-pointer hover:text-red-600"
data-testid="delete-team-button"
/>
</Tooltip>
</>
) : null}
</TableCell>
ui/litellm-dashboard/src/components/add_model/add_model_tab.test.tsx
+9
View File
@@ -19,6 +19,15 @@ vi.mock("../networking", async () => {
modelAvailableCall: vi.fn().mockResolvedValue({
data: [{ id: "model-group-1" }, { id: "model-group-2" }],
}),
getProviderCreateMetadata: vi.fn().mockResolvedValue([
{
provider: "OpenAI",
provider_display_name: "OpenAI",
litellm_provider: "openai",
default_model_placeholder: "gpt-3.5-turbo",
credential_fields: [],
},
]),
};
});
ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx
+106 -29
View File
@@ -1,4 +1,4 @@
import React, { useEffect, useState } from "react";
import React, { useEffect, useMemo, useState } from "react";
import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal } from "antd";
import type { FormInstance } from "antd";
import type { UploadProps } from "antd/es/upload";
@@ -9,7 +9,14 @@ import ProviderSpecificFields from "./provider_specific_fields";
import AdvancedSettings from "./advanced_settings";
import { Providers, providerLogoMap } from "../provider_info_helpers";
import type { Team } from "../key_team_helpers/key_list";
import { CredentialItem, getGuardrailsList, modelAvailableCall, tagListCall } from "../networking";
import {
type CredentialItem,
type ProviderCreateInfo,
getGuardrailsList,
getProviderCreateMetadata,
modelAvailableCall,
tagListCall,
} from "../networking";
import ConnectionErrorDisplay from "./model_connection_test";
import { TEST_MODES } from "./add_model_modes";
import { Row, Col } from "antd";
@@ -68,6 +75,11 @@ const AddModelTab: React.FC<AddModelTabProps> = ({
// Using a unique ID to force the ConnectionErrorDisplay to remount and run a fresh test
const [connectionTestId, setConnectionTestId] = useState<string>("");
// Provider metadata for driving the provider select from backend config
const [providerMetadata, setProviderMetadata] = useState<ProviderCreateInfo[] | null>(null);
const [isProviderMetadataLoading, setIsProviderMetadataLoading] = useState<boolean>(false);
const [providerMetadataError, setProviderMetadataError] = useState<string | null>(null);
useEffect(() => {
const fetchGuardrails = async () => {
try {
@@ -95,6 +107,37 @@ const AddModelTab: React.FC<AddModelTabProps> = ({
fetchTags();
}, [accessToken]);
useEffect(() => {
let isMounted = true;
const fetchProviderMetadata = async () => {
setIsProviderMetadataLoading(true);
setProviderMetadataError(null);
try {
const metadata = await getProviderCreateMetadata();
if (!isMounted) {
return;
}
setProviderMetadata(metadata);
} catch (error) {
console.error("Failed to fetch provider metadata:", error);
if (isMounted) {
setProviderMetadataError("Failed to load providers");
}
} finally {
if (isMounted) {
setIsProviderMetadataLoading(false);
}
}
};
fetchProviderMetadata();
return () => {
isMounted = false;
};
}, []);
// Test connection when button is clicked
const handleTestConnection = async () => {
setIsTestingConnection(true);
@@ -118,6 +161,13 @@ const AddModelTab: React.FC<AddModelTabProps> = ({
fetchModelAccessGroups();
}, [accessToken]);
const sortedProviderMetadata: ProviderCreateInfo[] = useMemo(() => {
if (!providerMetadata) {
return [];
}
return [...providerMetadata].sort((a, b) => a.provider_display_name.localeCompare(b.provider_display_name));
}, [providerMetadata]);
const isAdmin = all_admin_roles.includes(userRole);
const handleAutoRouterOk = () => {
@@ -166,41 +216,68 @@ const AddModelTab: React.FC<AddModelTabProps> = ({
labelAlign="left"
>
<AntdSelect
showSearch={true}
value={selectedProvider}
showSearch
loading={isProviderMetadataLoading}
placeholder={isProviderMetadataLoading ? "Loading providers..." : "Select a provider"}
optionFilterProp="data-label"
onChange={(value) => {
setSelectedProvider(value);
setProviderModelsFn(value);
setSelectedProvider(value as Providers);
setProviderModelsFn(value as Providers);
form.setFieldsValue({
custom_llm_provider: value,
});
form.setFieldsValue({
model: [],
model_name: undefined,
});
}}
>
{Object.entries(Providers).map(([providerEnum, providerDisplayName]) => (
<AntdSelect.Option key={providerEnum} value={providerEnum}>
<div className="flex items-center space-x-2">
<img
src={providerLogoMap[providerDisplayName]}
alt={`${providerEnum} logo`}
className="w-5 h-5"
onError={(e) => {
// Create a div with provider initial as fallback
const target = e.target as HTMLImageElement;
const parent = target.parentElement;
if (parent) {
const fallbackDiv = document.createElement("div");
fallbackDiv.className =
"w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs";
fallbackDiv.textContent = providerDisplayName.charAt(0);
parent.replaceChild(fallbackDiv, target);
}
}}
/>
<span>{providerDisplayName}</span>
</div>
{providerMetadataError && sortedProviderMetadata.length === 0 && (
<AntdSelect.Option key="__error" value="">
{providerMetadataError}
</AntdSelect.Option>
))}
)}
{sortedProviderMetadata.map((providerInfo) => {
const displayName = providerInfo.provider_display_name;
const providerKey = providerInfo.provider;
const logoSrc = providerLogoMap[displayName] ?? "";
return (
<AntdSelect.Option key={providerKey} value={providerKey} data-label={displayName}>
<div className="flex items-center space-x-2">
{logoSrc ? (
<img
src={logoSrc}
alt={`${displayName} logo`}
className="w-5 h-5"
onError={(e) => {
const target = e.currentTarget as HTMLImageElement;
const parent = target.parentElement;
if (!parent || !parent.contains(target)) {
return;
}
try {
const fallbackDiv = document.createElement("div");
fallbackDiv.className =
"w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs";
fallbackDiv.textContent = displayName.charAt(0);
parent.replaceChild(fallbackDiv, target);
} catch (error) {
console.error("Failed to replace provider logo fallback:", error);
}
}}
/>
) : (
<div className="w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs">
{displayName.charAt(0)}
</div>
)}
<span>{displayName}</span>
</div>
</AntdSelect.Option>
);
})}
</AntdSelect>
</Form.Item>
<LiteLLMModelNameField
ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx
+58
View File
@@ -0,0 +1,58 @@
import { describe, expect, it, vi } from "vitest";
import { prepareModelAddRequest } from "./handle_add_model_submit";
vi.mock("../molecules/notifications_manager", () => ({
default: {
fromBackend: vi.fn(),
},
}));
describe("prepareModelAddRequest", () => {
it("returns deployment data for the most basic form", async () => {
const formValues = {
model_mappings: [
{
public_name: "Public Model",
litellm_model: "litellm/public",
},
],
model_name: "custom-model-name",
base_model: "gpt-4",
team_id: "team-123",
model_access_group: ["group-1"],
input_cost_per_token: "2000000",
output_cost_per_token: "1000000",
};
const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
expect(deployments).toHaveLength(1);
const [deployment] = deployments!;
expect(deployment.modelName).toBe("Public Model");
expect(deployment.litellmParamsObj.model).toBe("custom-model-name");
expect(deployment.litellmParamsObj.input_cost_per_token).toBe(2);
expect(deployment.litellmParamsObj.output_cost_per_token).toBe(1);
expect(deployment.modelInfoObj.base_model).toBe("gpt-4");
expect(deployment.modelInfoObj.access_groups).toEqual(["group-1"]);
expect(deployment.modelInfoObj.team_id).toBe("team-123");
});
it("uses a lowercase fallback for unrecognized custom providers", async () => {
const fallbackValues = {
model_mappings: [
{
public_name: "Petals Model",
litellm_model: "petals/model",
},
],
model_name: "petals/model",
custom_llm_provider: "Petals",
};
const deployments = await prepareModelAddRequest({ ...fallbackValues }, "token", null);
expect(deployments).toHaveLength(1);
const [deployment] = deployments!;
expect(deployment.litellmParamsObj.custom_llm_provider).toBe("petals");
});
});
ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx
+8 -5
View File
@@ -1,6 +1,6 @@
import { provider_map, Providers } from "../provider_info_helpers";
import { modelCreateCall, Model } from "../networking";
import NotificationManager from "../molecules/notifications_manager";
import { Model, modelCreateCall } from "../networking";
import { provider_map } from "../provider_info_helpers";
export const prepareModelAddRequest = async (formValues: Record<string, any>, accessToken: string, form: any) => {
try {
@@ -14,8 +14,10 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
// Handle wildcard case
if (formValues["model"] && formValues["model"].includes("all-wildcard")) {
const customProvider: Providers = formValues["custom_llm_provider"];
const litellm_custom_provider = provider_map[customProvider as keyof typeof Providers];
const customProviderKey = formValues["custom_llm_provider"] as string;
const mappedProvider =
provider_map[customProviderKey as keyof typeof provider_map] ?? customProviderKey.toLowerCase();
const litellm_custom_provider = mappedProvider;
const wildcardModel = litellm_custom_provider + "/*";
formValues["model_name"] = wildcardModel;
modelMappings.push({
@@ -59,7 +61,8 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
litellmParamsObj["model"] = value;
} else if (key == "custom_llm_provider") {
console.log("custom_llm_provider:", value);
const mappingResult = provider_map[value]; // Get the corresponding value from the mapping
const providerKey = value as string;
const mappingResult = provider_map[providerKey as keyof typeof provider_map] ?? providerKey.toLowerCase();
litellmParamsObj["custom_llm_provider"] = mappingResult;
console.log("custom_llm_provider mappingResult:", mappingResult);
} else if (key == "model") {
ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx
+94 -1
View File
@@ -1,9 +1,102 @@
import { render, waitFor } from "@testing-library/react";
import { describe, it, expect, beforeAll } from "vitest";
import { describe, it, expect, beforeAll, vi } from "vitest";
import { Form } from "antd";
import { Providers } from "../provider_info_helpers";
import ProviderSpecificFields from "./provider_specific_fields";
vi.mock("../networking", async () => {
const actual = await vi.importActual("../networking");
return {
...actual,
getProviderCreateMetadata: vi.fn().mockResolvedValue([
{
provider: "OpenAI",
provider_display_name: Providers.OpenAI,
litellm_provider: "openai",
default_model_placeholder: "gpt-3.5-turbo",
credential_fields: [
{
key: "api_base",
label: "API Base",
field_type: "text",
placeholder: "https://api.openai.com/v1",
tooltip:
"Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com",
default_value: "https://api.openai.com/v1",
},
{
key: "organization",
label: "OpenAI Organization ID",
placeholder: "[OPTIONAL] my-unique-org",
},
{
key: "api_key",
label: "OpenAI API Key",
field_type: "password",
required: true,
},
],
},
{
provider: "Hosted_Vllm",
provider_display_name: Providers.Hosted_Vllm,
litellm_provider: "hosted_vllm",
default_model_placeholder: "vllm/any-model",
credential_fields: [
{
key: "api_base",
label: "API Base",
placeholder: "https://...",
},
{
key: "api_key",
label: "vLLM API Key",
field_type: "password",
},
],
},
{
provider: "Azure",
provider_display_name: Providers.Azure,
litellm_provider: "azure",
default_model_placeholder: "azure/my-deployment",
credential_fields: [
{
key: "api_base",
label: "API Base",
placeholder: "https://...",
required: true,
},
{
key: "api_version",
label: "API Version",
placeholder: "2023-07-01-preview",
tooltip:
"By default litellm will use the latest version. If you want to use a different version, you can specify it here",
},
{
key: "base_model",
label: "Base Model",
placeholder: "azure/gpt-3.5-turbo",
},
{
key: "api_key",
label: "Azure API Key",
field_type: "password",
placeholder: "Enter your Azure API Key",
},
{
key: "azure_ad_token",
label: "Azure AD Token",
field_type: "password",
placeholder: "Enter your Azure AD Token",
},
],
},
]),
};
});
// Mock window.matchMedia for Ant Design components
beforeAll(() => {
Object.defineProperty(window, "matchMedia", {
ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
+141 -536
View File
@@ -4,7 +4,12 @@ import { TextInput, Text } from "@tremor/react";
import { Row, Col, Typography, Button as Button2, Upload, UploadProps } from "antd";
import { UploadOutlined } from "@ant-design/icons";
import { provider_map, Providers } from "../provider_info_helpers";
import { CredentialItem } from "../networking";
import {
CredentialItem,
ProviderCreateInfo,
ProviderCredentialFieldMetadata,
getProviderCreateMetadata,
} from "../networking";
const { Link } = Typography;
interface ProviderSpecificFieldsProps {
@@ -28,6 +33,33 @@ export interface CredentialValues {
value: string;
}
const mapFieldMetadataToUiField = (field: ProviderCredentialFieldMetadata): ProviderCredentialField => {
const type: ProviderCredentialField["type"] =
field.field_type === "password"
? "password"
: field.field_type === "select"
? "select"
: field.field_type === "upload"
? "upload"
: "text";
return {
key: field.key,
label: field.label,
placeholder: field.placeholder ?? undefined,
tooltip: field.tooltip ?? undefined,
required: field.required ?? false,
type,
options: field.options ?? undefined,
defaultValue: field.default_value ?? undefined,
};
};
// In-memory cache of provider credential fields keyed by provider display name.
// This lets us reuse the data across multiple mounts and also supports
// non-React helpers like createCredentialFromModel.
const providerFieldsByDisplayName: Record<string, ProviderCredentialField[]> = {};
export const createCredentialFromModel = (provider: string, modelData: any): CredentialItem => {
console.log("provider", provider);
console.log("modelData", modelData);
@@ -35,8 +67,8 @@ export const createCredentialFromModel = (provider: string, modelData: any): Cre
if (!enumKey) {
throw new Error(`Provider ${provider} not found in provider_map`);
}
const providerEnum = Providers[enumKey as keyof typeof Providers];
const providerFields = PROVIDER_CREDENTIAL_FIELDS[providerEnum] || [];
const providerDisplayName = Providers[enumKey as keyof typeof Providers];
const providerFields = providerFieldsByDisplayName[providerDisplayName] || [];
const credentialValues: object = {};
console.log("providerFields", providerFields);
@@ -63,544 +95,103 @@ export const createCredentialFromModel = (provider: string, modelData: any): Cre
return credential;
};
const PROVIDER_CREDENTIAL_FIELDS: Record<Providers, ProviderCredentialField[]> = {
[Providers.OpenAI]: [
{
key: "api_base",
label: "API Base",
type: "text",
placeholder: "https://api.openai.com/v1",
tooltip: "Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com",
defaultValue: "https://api.openai.com/v1",
},
{
key: "organization",
label: "OpenAI Organization ID",
placeholder: "[OPTIONAL] my-unique-org",
},
{
key: "api_key",
label: "OpenAI API Key",
type: "password",
required: true,
},
],
[Providers.OpenAI_Text]: [
{
key: "api_base",
label: "API Base",
type: "text",
placeholder: "https://api.openai.com/v1",
tooltip: "Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com",
defaultValue: "https://api.openai.com/v1",
},
{
key: "organization",
label: "OpenAI Organization ID",
placeholder: "[OPTIONAL] my-unique-org",
},
{
key: "api_key",
label: "OpenAI API Key",
type: "password",
required: true,
},
],
[Providers.Vertex_AI]: [
{
key: "vertex_project",
label: "Vertex Project",
placeholder: "adroit-cadet-1234..",
required: true,
},
{
key: "vertex_location",
label: "Vertex Location",
placeholder: "us-east-1",
required: true,
},
{
key: "vertex_credentials",
label: "Vertex Credentials",
required: true,
type: "upload",
},
],
[Providers.AssemblyAI]: [
{
key: "api_base",
label: "API Base",
type: "select",
required: true,
options: ["https://api.assemblyai.com", "https://api.eu.assemblyai.com"],
},
{
key: "api_key",
label: "AssemblyAI API Key",
type: "password",
required: true,
},
],
[Providers.Azure]: [
{
key: "api_base",
label: "API Base",
placeholder: "https://...",
required: true,
},
{
key: "api_version",
label: "API Version",
placeholder: "2023-07-01-preview",
tooltip:
"By default litellm will use the latest version. If you want to use a different version, you can specify it here",
},
{
key: "base_model",
label: "Base Model",
placeholder: "azure/gpt-3.5-turbo",
},
{
key: "api_key",
label: "Azure API Key",
type: "password",
placeholder: "Enter your Azure API Key",
required: false,
},
{
key: "azure_ad_token",
label: "Azure AD Token",
type: "password",
placeholder: "Enter your Azure AD Token",
required: false,
},
],
[Providers.Azure_AI_Studio]: [
{
key: "api_base",
label: "API Base",
placeholder: "https://<test>.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21",
tooltip:
"Enter your full Target URI from Azure Foundry here. Example: https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21",
required: true,
},
{
key: "api_key",
label: "Azure API Key",
type: "password",
required: true,
},
],
[Providers.OpenAI_Compatible]: [
{
key: "api_base",
label: "API Base",
placeholder: "https://...",
required: true,
},
{
key: "api_key",
label: "OpenAI API Key",
type: "password",
required: true,
},
],
[Providers.Dashscope]: [
{
key: "api_key",
label: "Dashscope API Key",
type: "password",
required: true,
},
{
key: "api_base",
label: "API Base",
placeholder: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
defaultValue: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
required: true,
tooltip:
"The base URL for your Dashscope server. Defaults to https://dashscope-intl.aliyuncs.com/compatible-mode/v1 if not specified.",
},
],
[Providers.OpenAI_Text_Compatible]: [
{
key: "api_base",
label: "API Base",
placeholder: "https://...",
required: true,
},
{
key: "api_key",
label: "OpenAI API Key",
type: "password",
required: true,
},
],
[Providers.Bedrock]: [
{
key: "aws_access_key_id",
label: "AWS Access Key ID",
type: "password",
required: false,
tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
key: "aws_secret_access_key",
label: "AWS Secret Access Key",
type: "password",
required: false,
tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
key: "aws_session_token",
label: "AWS Session Token",
type: "password",
required: false,
tooltip:
"Temporary credentials session token. You can provide the raw token or the environment variable (e.g. `os.environ/MY_SESSION_TOKEN`).",
},
{
key: "aws_region_name",
label: "AWS Region Name",
placeholder: "us-east-1",
required: false,
tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
key: "aws_session_name",
label: "AWS Session Name",
placeholder: "my-session",
required: false,
tooltip:
"Name for the AWS session. You can provide the raw value or the environment variable (e.g. `os.environ/MY_SESSION_NAME`).",
},
{
key: "aws_profile_name",
label: "AWS Profile Name",
placeholder: "default",
required: false,
tooltip:
"AWS profile name to use for authentication. You can provide the raw value or the environment variable (e.g. `os.environ/MY_PROFILE_NAME`).",
},
{
key: "aws_role_name",
label: "AWS Role Name",
placeholder: "MyRole",
required: false,
tooltip:
"AWS IAM role name to assume. You can provide the raw value or the environment variable (e.g. `os.environ/MY_ROLE_NAME`).",
},
{
key: "aws_web_identity_token",
label: "AWS Web Identity Token",
type: "password",
required: false,
tooltip:
"Web identity token for OIDC authentication. You can provide the raw token or the environment variable (e.g. `os.environ/MY_WEB_IDENTITY_TOKEN`).",
},
{
key: "aws_bedrock_runtime_endpoint",
label: "AWS Bedrock Runtime Endpoint",
placeholder: "https://bedrock-runtime.us-east-1.amazonaws.com",
required: false,
tooltip:
"Custom Bedrock runtime endpoint URL. You can provide the raw value or the environment variable (e.g. `os.environ/MY_BEDROCK_ENDPOINT`).",
},
],
[Providers.SageMaker]: [
{
key: "aws_access_key_id",
label: "AWS Access Key ID",
type: "password",
required: false,
tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
key: "aws_secret_access_key",
label: "AWS Secret Access Key",
type: "password",
required: false,
tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
{
key: "aws_region_name",
label: "AWS Region Name",
placeholder: "us-east-1",
required: false,
tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).",
},
],
[Providers.Ollama]: [
{
key: "api_base",
label: "API Base",
placeholder: "http://localhost:11434",
defaultValue: "http://localhost:11434",
required: false,
tooltip: "The base URL for your Ollama server. Defaults to http://localhost:11434 if not specified.",
},
],
[Providers.Anthropic]: [
{
key: "api_key",
label: "API Key",
placeholder: "sk-",
type: "password",
required: true,
},
],
[Providers.Deepgram]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.ElevenLabs]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Google_AI_Studio]: [
{
key: "api_key",
label: "API Key",
placeholder: "aig-",
type: "password",
required: true,
},
],
[Providers.Groq]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.MistralAI]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Deepseek]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Cohere]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Databricks]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.xAI]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.AIML]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Cerebras]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Sambanova]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Perplexity]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.TogetherAI]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Openrouter]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.FireworksAI]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.GradientAI]: [
{
key: "api_base",
label: "GradientAI Endpoint",
placeholder: "https://...",
required: false,
},
{
key: "api_key",
label: "GradientAI API Key",
type: "password",
required: true,
},
],
[Providers.Triton]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: false,
},
{
key: "api_base",
label: "API Base",
placeholder: "http://localhost:8000/generate",
required: false,
},
],
[Providers.Hosted_Vllm]: [
{
key: "api_base",
label: "API Base",
placeholder: "https://...",
required: true,
},
{
key: "api_key",
label: "vLLM API Key",
type: "password",
required: false,
},
],
[Providers.Voyage]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.JinaAI]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.VolcEngine]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.DeepInfra]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Oracle]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
[Providers.Snowflake]: [
{
key: "api_key",
label: "Snowflake API Key / JWT Key for Authentication",
type: "password",
required: true,
},
{
key: "api_base",
label: "Snowflake API Endpoint",
placeholder: "https://1234567890.snowflakecomputing.com/api/v2/cortex/inference:complete",
tooltip:
"Enter the full endpoint with path here. Example: https://1234567890.snowflakecomputing.com/api/v2/cortex/inference:complete",
required: true,
},
],
[Providers.Infinity]: [
{
key: "api_base",
label: "API Base",
placeholder: "http://localhost:7997",
},
],
[Providers.FalAI]: [
{
key: "api_key",
label: "API Key",
type: "password",
required: true,
},
],
};
const ProviderSpecificFields: React.FC<ProviderSpecificFieldsProps> = ({ selectedProvider, uploadProps }) => {
const selectedProviderEnum = Providers[selectedProvider as keyof typeof Providers] as Providers;
const form = Form.useFormInstance(); // Get form instance from context
// Simply use the fields as defined in PROVIDER_CREDENTIAL_FIELDS
const [providerMetadata, setProviderMetadata] = React.useState<ProviderCreateInfo[] | null>(null);
const [isLoading, setIsLoading] = React.useState<boolean>(false);
const [loadError, setLoadError] = React.useState<string | null>(null);
React.useEffect(() => {
const hasCachedFields = Object.keys(providerFieldsByDisplayName).length > 0;
if (hasCachedFields) {
// We already have fields cached globally; no need to refetch.
// This is important so we can reuse credential field definitions
// across mounts and in non-React helpers.
return;
}
let isMounted = true;
const fetchProviderFields = async () => {
setIsLoading(true);
setLoadError(null);
try {
const metadata = await getProviderCreateMetadata();
if (!isMounted) {
return;
}
setProviderMetadata(metadata);
// Populate cache keyed by provider display name and identifiers
metadata.forEach((providerInfo) => {
const displayName = providerInfo.provider_display_name;
const mappedFields = providerInfo.credential_fields.map(mapFieldMetadataToUiField);
// Primary key: human-readable display name
providerFieldsByDisplayName[displayName] = mappedFields;
// Also cache by backend identifiers so lookups by provider slug work
if (providerInfo.provider) {
providerFieldsByDisplayName[providerInfo.provider] = mappedFields;
}
if (providerInfo.litellm_provider) {
providerFieldsByDisplayName[providerInfo.litellm_provider] = mappedFields;
}
});
} catch (error) {
console.error("Failed to load provider credential fields:", error);
if (isMounted) {
setLoadError("Failed to load provider credential fields");
}
} finally {
if (isMounted) {
setIsLoading(false);
}
}
};
fetchProviderFields();
return () => {
isMounted = false;
};
}, []);
const allFields = React.useMemo(() => {
return PROVIDER_CREDENTIAL_FIELDS[selectedProviderEnum] || [];
}, [selectedProviderEnum]);
// First try to resolve from the in-memory cache. We support both the
// enum/display-name form and the raw provider slug (e.g. "petals").
const cachedFields =
providerFieldsByDisplayName[selectedProviderEnum] ?? providerFieldsByDisplayName[selectedProvider];
if (cachedFields) {
return cachedFields;
}
if (!providerMetadata) {
return [];
}
const providerInfo = providerMetadata.find(
(p) =>
p.provider_display_name === selectedProviderEnum ||
p.provider === selectedProvider ||
p.litellm_provider === selectedProvider,
);
if (!providerInfo) {
return [];
}
const mapped = providerInfo.credential_fields.map(mapFieldMetadataToUiField);
providerFieldsByDisplayName[providerInfo.provider_display_name] = mapped;
if (providerInfo.provider) {
providerFieldsByDisplayName[providerInfo.provider] = mapped;
}
if (providerInfo.litellm_provider) {
providerFieldsByDisplayName[providerInfo.litellm_provider] = mapped;
}
return mapped;
}, [selectedProviderEnum, selectedProvider, providerMetadata]);
const handleUpload = {
name: "file",
@@ -633,6 +224,20 @@ const ProviderSpecificFields: React.FC<ProviderSpecificFieldsProps> = ({ selecte
return (
<>
{isLoading && allFields.length === 0 && (
<Row>
<Col span={24}>
<Text className="mb-2">Loading provider fields...</Text>
</Col>
</Row>
)}
{loadError && allFields.length === 0 && (
<Row>
<Col span={24}>
<Text className="mb-2 text-red-500">{loadError}</Text>
</Col>
</Row>
)}
{allFields.map((field) => (
<React.Fragment key={field.key}>
<Form.Item
ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx
+12 -11
View File
@@ -159,7 +159,7 @@ const GuardrailTable: React.FC<GuardrailTableProps> = ({
},
{
id: "actions",
header: "",
header: "Actions",
cell: ({ row }) => {
const guardrail = row.original;
const isConfigGuardrail = guardrail.guardrail_definition_location === GuardrailDefinitionLocation.CONFIG;
@@ -177,16 +177,17 @@ const GuardrailTable: React.FC<GuardrailTableProps> = ({
/>
</Tooltip>
) : (
<Icon
icon={TrashIcon}
size="sm"
onClick={() =>
guardrail.guardrail_id &&
onDeleteClick(guardrail.guardrail_id, guardrail.guardrail_name || "Unnamed Guardrail")
}
className="cursor-pointer hover:text-red-500"
tooltip="Delete guardrail"
/>
<Tooltip title="Delete guardrail">
<Icon
icon={TrashIcon}
size="sm"
onClick={() =>
guardrail.guardrail_id &&
onDeleteClick(guardrail.guardrail_id, guardrail.guardrail_name || "Unnamed Guardrail")
}
className="cursor-pointer hover:text-red-500"
/>
</Tooltip>
)}
</div>
);
ui/litellm-dashboard/src/components/model_add/credentials.test.tsx
+50
View File
@@ -0,0 +1,50 @@
import { CredentialItem } from "@/components/networking";
import { render, waitFor } from "@testing-library/react";
import { UploadProps } from "antd/es/upload";
import { describe, expect, it, vi } from "vitest";
import CredentialsPanel from "./credentials";
const DEFAULT_UPLOAD_PROPS = {} as UploadProps;
describe("CredentialsPanel", () => {
it("renders without crashing and fetches credentials when token exists", async () => {
const fetchCredentials = vi.fn(() => Promise.resolve());
const { getByRole, getByText } = render(
<CredentialsPanel
accessToken="test-token"
uploadProps={DEFAULT_UPLOAD_PROPS}
credentialList={[]}
fetchCredentials={fetchCredentials}
/>,
);
await waitFor(() => {
expect(getByRole("button", { name: /add credential/i })).toBeInTheDocument();
expect(getByText("Credential Name")).toBeInTheDocument();
expect(getByText("Provider")).toBeInTheDocument();
});
});
it("displays provided credentials and still calls the fetch helper", async () => {
const fetchCredentials = vi.fn(() => Promise.resolve());
const credentials: CredentialItem[] = [
{
credential_name: "openai-key",
credential_values: {},
credential_info: { custom_llm_provider: "openai" },
},
];
const { getByText } = render(
<CredentialsPanel
accessToken="another-token"
uploadProps={DEFAULT_UPLOAD_PROPS}
credentialList={credentials}
fetchCredentials={fetchCredentials}
/>,
);
await waitFor(() => expect(getByText("openai-key")).toBeInTheDocument());
});
});
ui/litellm-dashboard/src/components/model_add/credentials.tsx
-2
View File
@@ -143,7 +143,6 @@ const CredentialsPanel: React.FC<CredentialsPanelProps> = ({
<TableRow>
<TableHeaderCell>Credential Name</TableHeaderCell>
<TableHeaderCell>Provider</TableHeaderCell>
<TableHeaderCell>Description</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
@@ -160,7 +159,6 @@ const CredentialsPanel: React.FC<CredentialsPanelProps> = ({
<TableCell>
{renderProviderBadge((credential.credential_info?.custom_llm_provider as string) || "-")}
</TableCell>
<TableCell>{credential.credential_info?.description || "-"}</TableCell>
<TableCell>
<Button
icon={PencilAltIcon}
ui/litellm-dashboard/src/components/model_info_view.tsx
+11 -14
View File
@@ -419,15 +419,20 @@ export default function ModelInfoView({
alt={`${modelData.provider} logo`}
className="w-4 h-4"
onError={(e) => {
// Create a div with provider initial as fallback
const target = e.target as HTMLImageElement;
const target = e.currentTarget as HTMLImageElement;
const parent = target.parentElement;
if (parent) {
if (!parent || !parent.contains(target)) {
return;
}
try {
const fallbackDiv = document.createElement("div");
fallbackDiv.className =
"w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs";
fallbackDiv.textContent = modelData.provider?.charAt(0) || "-";
parent.replaceChild(fallbackDiv, target);
} catch (error) {
console.error("Failed to replace provider logo fallback:", error);
}
}}
/>
@@ -493,22 +498,14 @@ export default function ModelInfoView({
<Title>Model Settings</Title>
<div className="flex gap-2">
{isAutoRouter && canEditModel && !isEditing && (
<TremorButton
variant="primary"
onClick={() => setIsAutoRouterModalOpen(true)}
className="flex items-center"
>
<TremorButton onClick={() => setIsAutoRouterModalOpen(true)} className="flex items-center">
Edit Auto Router
</TremorButton>
)}
{canEditModel ? (
!isEditing && (
<TremorButton
variant="secondary"
onClick={() => setIsEditing(true)}
className="flex items-center"
>
Edit Model
<TremorButton onClick={() => setIsEditing(true)} className="flex items-center">
Edit Settings
</TremorButton>
)
) : (

Some files were not shown because too many files have changed in this diff Show More