mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-18 10:23:39 +00:00
Merge branch 'main' into wandb-inference
This commit is contained in:
Anubhav Singh
@@ -671,6 +671,7 @@ jobs:
|
||||
pip install mypy
|
||||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install "google-genai==1.22.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
|
||||
@@ -345,6 +345,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
|
||||
| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | |
|
||||
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
||||
| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | |
|
||||
| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | | | | |
|
||||
|
||||
[**Read the Docs**](https://docs.litellm.ai/docs/)
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
base_url="http://0.0.0.0:4000",
|
||||
api_key="sk-1234",
|
||||
)
|
||||
|
||||
BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
|
||||
|
||||
# Upload file
|
||||
batch_input_file = client.files.create(
|
||||
file=open("./bedrock_batch_completions.jsonl", "rb"),
|
||||
purpose="batch",
|
||||
extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
|
||||
)
|
||||
print(batch_input_file)
|
||||
|
||||
# Create batch
|
||||
batch = client.batches.create(
|
||||
input_file_id=batch_input_file.id,
|
||||
endpoint="/v1/chat/completions",
|
||||
completion_window="24h",
|
||||
metadata={"description": "Test batch job"},
|
||||
)
|
||||
print(batch)
|
||||
@@ -0,0 +1,128 @@
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Use LiteLLM Proxy MCP Gateway to call MCP tools.
|
||||
|
||||
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
|
||||
"""
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # paste your litellm proxy api key here
|
||||
base_url="http://localhost:4000" # paste your litellm proxy base url here
|
||||
)
|
||||
print("Making API request to Responses API with MCP tools")
|
||||
|
||||
response = client.responses.create(
|
||||
model="gpt-5",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "give me TLDR of what BerriAI/litellm repo is about",
|
||||
"type": "message"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "litellm",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never"
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
tool_choice="required"
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print("response chunk: ", chunk)
|
||||
@@ -0,0 +1,256 @@
|
||||
# LiteLLM Release Notes Generation Instructions
|
||||
|
||||
This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style.
|
||||
|
||||
## Required Inputs
|
||||
|
||||
1. **Release Version** (e.g., `v1.76.3-stable`)
|
||||
2. **PR Diff/Changelog** - List of PRs with titles and contributors
|
||||
3. **Previous Version Commit Hash** - To compare model pricing changes
|
||||
4. **Reference Release Notes** - Previous release notes to follow style/format
|
||||
|
||||
## Step-by-Step Process
|
||||
|
||||
### 1. Initial Setup and Analysis
|
||||
|
||||
```bash
|
||||
# Check git diff for model pricing changes
|
||||
git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
|
||||
```
|
||||
|
||||
**Key Analysis Points:**
|
||||
- New models added (look for new entries)
|
||||
- Deprecated models removed (look for deleted entries)
|
||||
- Pricing updates (look for cost changes)
|
||||
- Feature support changes (tool calling, reasoning, etc.)
|
||||
|
||||
### 2. Release Notes Structure
|
||||
|
||||
Follow this exact structure based on `docs/my-website/release_notes/v1.76.1-stable/index.md`:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "v1.76.X-stable - [Key Theme]"
|
||||
slug: "v1-76-X"
|
||||
date: YYYY-MM-DDTHH:mm:ss
|
||||
authors: [standard author block]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
## Deploy this version
|
||||
[Docker and pip installation tabs]
|
||||
|
||||
## Key Highlights
|
||||
[3-5 bullet points of major features]
|
||||
|
||||
## Major Changes
|
||||
[Critical changes users need to know]
|
||||
|
||||
## Performance Improvements
|
||||
[Performance-related changes]
|
||||
|
||||
## New Models / Updated Models
|
||||
[Detailed model tables and provider updates]
|
||||
|
||||
## LLM API Endpoints
|
||||
[API-related features and fixes]
|
||||
|
||||
## Management Endpoints / UI
|
||||
[Admin interface and management changes]
|
||||
|
||||
## Logging / Guardrail Integrations
|
||||
[Observability and security features]
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
[Infrastructure improvements]
|
||||
|
||||
## General Proxy Improvements
|
||||
[Other proxy-related changes]
|
||||
|
||||
## New Contributors
|
||||
[List of first-time contributors]
|
||||
|
||||
## Full Changelog
|
||||
[Link to GitHub comparison]
|
||||
```
|
||||
|
||||
### 3. Categorization Rules
|
||||
|
||||
**Performance Improvements:**
|
||||
- RPS improvements
|
||||
- Memory optimizations
|
||||
- CPU usage optimizations
|
||||
- Timeout controls
|
||||
- Worker configuration
|
||||
|
||||
**New Models/Updated Models:**
|
||||
- Extract from model_prices_and_context_window.json diff
|
||||
- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
|
||||
- Group by provider
|
||||
- Note pricing corrections
|
||||
- Highlight deprecated models
|
||||
|
||||
**Provider Features:**
|
||||
- Group by provider (Gemini, OpenAI, Anthropic, etc.)
|
||||
- Link to provider docs: `../../docs/providers/[provider_name]`
|
||||
- Separate features from bug fixes
|
||||
|
||||
**API Endpoints:**
|
||||
- Images API
|
||||
- Video Generation (if applicable)
|
||||
- Responses API
|
||||
- Passthrough endpoints
|
||||
- General chat completions
|
||||
|
||||
**UI/Management:**
|
||||
- Authentication changes
|
||||
- Dashboard improvements
|
||||
- Team management
|
||||
- Key management
|
||||
|
||||
**Integrations:**
|
||||
- Logging providers (Datadog, Braintrust, etc.)
|
||||
- Guardrails
|
||||
- Cost tracking
|
||||
- Observability
|
||||
|
||||
### 4. Documentation Linking Strategy
|
||||
|
||||
**Link to docs when:**
|
||||
- New provider support added
|
||||
- Significant feature additions
|
||||
- API endpoint changes
|
||||
- Integration additions
|
||||
|
||||
**Link format:** `../../docs/[category]/[specific_doc]`
|
||||
|
||||
**Common doc paths:**
|
||||
- `../../docs/providers/[provider]` - Provider-specific docs
|
||||
- `../../docs/image_generation` - Image generation
|
||||
- `../../docs/video_generation` - Video generation (if exists)
|
||||
- `../../docs/response_api` - Responses API
|
||||
- `../../docs/proxy/logging` - Logging integrations
|
||||
- `../../docs/proxy/guardrails` - Guardrails
|
||||
- `../../docs/pass_through/[provider]` - Passthrough endpoints
|
||||
|
||||
### 5. Model Table Generation
|
||||
|
||||
From git diff analysis, create tables like:
|
||||
|
||||
```markdown
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
|
||||
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
|
||||
| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
|
||||
```
|
||||
|
||||
**Extract from JSON:**
|
||||
- `max_input_tokens` → Context Window
|
||||
- `input_cost_per_token` × 1,000,000 → Input cost
|
||||
- `output_cost_per_token` × 1,000,000 → Output cost
|
||||
- `supports_*` fields → Features
|
||||
- Special pricing fields (per image, per second) for generation models
|
||||
|
||||
### 6. PR Categorization Logic
|
||||
|
||||
**By Keywords in PR Title:**
|
||||
- `[Perf]`, `Performance`, `RPS` → Performance Improvements
|
||||
- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
|
||||
- `[Feat]`, `[Feature]`, `Add support` → Features section
|
||||
- `[Docs]` → Documentation (usually exclude from main sections)
|
||||
- Provider names (Gemini, OpenAI, etc.) → Group under provider
|
||||
|
||||
**By PR Content Analysis:**
|
||||
- New model additions → New Models section
|
||||
- UI changes → Management Endpoints/UI
|
||||
- Logging/observability → Logging/Guardrail Integrations
|
||||
- Rate limiting/budgets → Performance/Reliability
|
||||
- Authentication → Management Endpoints
|
||||
|
||||
### 7. Writing Style Guidelines
|
||||
|
||||
**Tone:**
|
||||
- Professional but accessible
|
||||
- Focus on user impact
|
||||
- Highlight breaking changes clearly
|
||||
- Use active voice
|
||||
|
||||
**Formatting:**
|
||||
- Use consistent markdown formatting
|
||||
- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)`
|
||||
- Use code blocks for configuration examples
|
||||
- Bold important terms and section headers
|
||||
|
||||
**Warnings/Notes:**
|
||||
- Add warning boxes for breaking changes
|
||||
- Include migration instructions when needed
|
||||
- Provide override options for default changes
|
||||
|
||||
### 8. Quality Checks
|
||||
|
||||
**Before finalizing:**
|
||||
- Verify all PR links work
|
||||
- Check documentation links are valid
|
||||
- Ensure model pricing is accurate
|
||||
- Confirm provider names are consistent
|
||||
- Review for typos and formatting issues
|
||||
|
||||
### 9. Common Patterns to Follow
|
||||
|
||||
**Performance Changes:**
|
||||
```markdown
|
||||
- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link)
|
||||
```
|
||||
|
||||
**New Models:**
|
||||
Always include pricing table and feature highlights
|
||||
|
||||
**Breaking Changes:**
|
||||
```markdown
|
||||
:::warning
|
||||
This release has a known issue...
|
||||
:::
|
||||
```
|
||||
|
||||
**Provider Features:**
|
||||
```markdown
|
||||
- **[Provider Name](../../docs/providers/provider)**
|
||||
- Feature description - [PR #XXXXX](link)
|
||||
```
|
||||
|
||||
### 10. Missing Documentation Check
|
||||
|
||||
**Review for missing docs:**
|
||||
- New providers without documentation
|
||||
- New API endpoints without examples
|
||||
- Complex features without guides
|
||||
- Integration setup instructions
|
||||
|
||||
**Flag for documentation needs:**
|
||||
- New provider integrations
|
||||
- Significant API changes
|
||||
- Complex configuration options
|
||||
- Migration requirements
|
||||
|
||||
## Example Command Workflow
|
||||
|
||||
```bash
|
||||
# 1. Get model changes
|
||||
git diff <commit> HEAD -- model_prices_and_context_window.json
|
||||
|
||||
# 2. Analyze PR list for categorization
|
||||
# 3. Create release notes following template
|
||||
# 4. Link to appropriate documentation
|
||||
# 5. Review for missing documentation needs
|
||||
```
|
||||
|
||||
## Output Requirements
|
||||
|
||||
- Follow exact markdown structure from reference
|
||||
- Include all PR links and contributors
|
||||
- Provide accurate model pricing tables
|
||||
- Link to relevant documentation
|
||||
- Highlight breaking changes with warnings
|
||||
- Include deployment instructions
|
||||
- End with full changelog link
|
||||
|
||||
This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly.
|
||||
@@ -7,7 +7,7 @@ Covers Batches, Files
|
||||
|
||||
| Feature | Supported | Notes |
|
||||
|-------|-------|-------|
|
||||
| Supported Providers | OpenAI, Azure, Vertex | - |
|
||||
| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
|
||||
| ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
|
||||
| Logging | ✅ | Works across all logging integrations |
|
||||
|
||||
@@ -178,6 +178,7 @@ print("list_batches_response=", list_batches_response)
|
||||
### [Azure OpenAI](./providers/azure#azure-batches-api)
|
||||
### [OpenAI](#quick-start)
|
||||
### [Vertex AI](./providers/vertex#batch-apis)
|
||||
### [Bedrock](./providers/bedrock_batches)
|
||||
|
||||
|
||||
## How Cost Tracking for Batches API Works
|
||||
|
||||
@@ -65,6 +65,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea
|
||||
| Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || ||
|
||||
| Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| ||
|
||||
| Bytez | ✅| ✅ || ✅| ✅ | | | ✅|| || || || || || ||
|
||||
| OVHCloud AI Endpoints | ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
||||
|
||||
:::note
|
||||
|
||||
|
||||
+129
-33
@@ -195,70 +195,169 @@ litellm_settings:
|
||||
|
||||
## Using your MCP
|
||||
|
||||
### Use on LiteLLM UI
|
||||
|
||||
Follow this walkthrough to use your MCP on LiteLLM UI
|
||||
|
||||
<iframe width="840" height="500" src="https://www.loom.com/embed/57e0763267254bc79dbe6658d0b8758c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
|
||||
|
||||
### Use with Responses API
|
||||
|
||||
Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
|
||||
|
||||
Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
|
||||
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai" label="OpenAI API">
|
||||
|
||||
#### Connect via OpenAI Responses API
|
||||
|
||||
Use the OpenAI Responses API to connect to your LiteLLM MCP server:
|
||||
<TabItem value="curl" label="cURL">
|
||||
|
||||
```bash title="cURL Example" showLineNumbers
|
||||
curl --location 'https://api.openai.com/v1/responses' \
|
||||
curl --location 'http://localhost:4000/v1/responses' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
--header "Authorization: Bearer sk-1234" \
|
||||
--data '{
|
||||
"model": "gpt-4o",
|
||||
"model": "gpt-5",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "give me TLDR of what BerriAI/litellm repo is about",
|
||||
"type": "message"
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "litellm",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never",
|
||||
"headers": {
|
||||
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
|
||||
}
|
||||
"require_approval": "never"
|
||||
}
|
||||
],
|
||||
"input": "Run available tools",
|
||||
"stream": true,
|
||||
"tool_choice": "required"
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="python" label="Python SDK">
|
||||
|
||||
<TabItem value="litellm" label="LiteLLM Proxy">
|
||||
```python title="Python SDK Example" showLineNumbers
|
||||
"""
|
||||
Use LiteLLM Proxy MCP Gateway to call MCP tools.
|
||||
|
||||
#### Connect via LiteLLM Proxy Responses API
|
||||
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
|
||||
"""
|
||||
import openai
|
||||
|
||||
Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint.
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # paste your litellm proxy api key here
|
||||
base_url="http://localhost:4000" # paste your litellm proxy base url here
|
||||
)
|
||||
print("Making API request to Responses API with MCP tools")
|
||||
|
||||
```bash title="cURL Example" showLineNumbers
|
||||
curl --location '<your-litellm-proxy-base-url>/v1/responses' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
--data '{
|
||||
"model": "gpt-4o",
|
||||
"tools": [
|
||||
response = client.responses.create(
|
||||
model="gpt-5",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "give me TLDR of what BerriAI/litellm repo is about",
|
||||
"type": "message"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "litellm",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never",
|
||||
"headers": {
|
||||
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
|
||||
}
|
||||
"require_approval": "never"
|
||||
}
|
||||
],
|
||||
"input": "Run available tools",
|
||||
stream=True,
|
||||
tool_choice="required"
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print("response chunk: ", chunk)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
#### Specifying MCP Tools
|
||||
|
||||
You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
|
||||
|
||||
To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="curl" label="cURL">
|
||||
|
||||
```bash title="cURL Example with allowed_tools" showLineNumbers
|
||||
curl --location 'http://localhost:4000/v1/responses' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header "Authorization: Bearer sk-1234" \
|
||||
--data '{
|
||||
"model": "gpt-5",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "give me TLDR of what BerriAI/litellm repo is about",
|
||||
"type": "message"
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "litellm",
|
||||
"server_url": "litellm_proxy/mcp",
|
||||
"require_approval": "never",
|
||||
"allowed_tools": ["GitMCP-fetch_litellm_documentation"]
|
||||
}
|
||||
],
|
||||
"stream": true,
|
||||
"tool_choice": "required"
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="python" label="Python SDK">
|
||||
|
||||
<TabItem value="cursor" label="Cursor IDE">
|
||||
```python title="Python SDK Example with allowed_tools" showLineNumbers
|
||||
import openai
|
||||
|
||||
#### Connect via Cursor IDE
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
|
||||
response = client.responses.create(
|
||||
model="gpt-5",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "give me TLDR of what BerriAI/litellm repo is about",
|
||||
"type": "message"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "litellm",
|
||||
"server_url": "litellm_proxy/mcp",
|
||||
"require_approval": "never",
|
||||
"allowed_tools": ["GitMCP-fetch_litellm_documentation"]
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
tool_choice="required"
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Use with Cursor IDE
|
||||
|
||||
Use tools directly from Cursor IDE with LiteLLM MCP:
|
||||
|
||||
@@ -281,9 +380,6 @@ Use tools directly from Cursor IDE with LiteLLM MCP:
|
||||
}
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
#### How it works when server_url="litellm_proxy"
|
||||
|
||||
When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Bedrock Batches
|
||||
|
||||
Use Amazon Bedrock Batch Inference API through LiteLLM.
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Description | Amazon Bedrock Batch Inference allows you to run inference on large datasets asynchronously |
|
||||
| Provider Doc | [AWS Bedrock Batch Inference ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html) |
|
||||
|
||||
## Overview
|
||||
|
||||
Use this to:
|
||||
|
||||
- Run batch inference on large datasets with Bedrock models
|
||||
- Control batch model access by key/user/team (same as chat completion models)
|
||||
- Manage S3 storage for batch input/output files
|
||||
|
||||
## (Proxy Admin) Usage
|
||||
|
||||
Here's how to give developers access to your Bedrock Batch models.
|
||||
|
||||
### 1. Setup config.yaml
|
||||
|
||||
- Specify `mode: batch` for each model: Allows developers to know this is a batch model
|
||||
- Configure S3 bucket and AWS credentials for batch operations
|
||||
|
||||
```yaml showLineNumbers title="litellm_config.yaml"
|
||||
model_list:
|
||||
- model_name: "bedrock-batch-claude"
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
|
||||
#########################################################
|
||||
########## batch specific params ########################
|
||||
s3_bucket_name: litellm-proxy
|
||||
s3_region_name: us-west-2
|
||||
s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
|
||||
model_info:
|
||||
mode: batch # 👈 SPECIFY MODE AS BATCH, to tell user this is a batch model
|
||||
```
|
||||
|
||||
**Required Parameters:**
|
||||
|
||||
| Parameter | Description |
|
||||
|-----------|-------------|
|
||||
| `s3_bucket_name` | S3 bucket for batch input/output files |
|
||||
| `s3_region_name` | AWS region for S3 bucket |
|
||||
| `s3_access_key_id` | AWS access key for S3 bucket |
|
||||
| `s3_secret_access_key` | AWS secret key for S3 bucket |
|
||||
| `aws_batch_role_arn` | IAM role ARN for Bedrock batch operations. Bedrock Batch APIs require an IAM role ARN to be set. |
|
||||
| `mode: batch` | Indicates to LiteLLM this is a batch model |
|
||||
|
||||
### 2. Create Virtual Key
|
||||
|
||||
```bash showLineNumbers title="create_virtual_key.sh"
|
||||
curl -L -X POST 'https://{PROXY_BASE_URL}/key/generate' \
|
||||
-H 'Authorization: Bearer ${PROXY_API_KEY}' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"models": ["bedrock-batch-claude"]}'
|
||||
```
|
||||
|
||||
You can now use the virtual key to access the batch models (See Developer flow).
|
||||
|
||||
## (Developer) Usage
|
||||
|
||||
Here's how to create a LiteLLM managed file and execute Bedrock Batch CRUD operations with the file.
|
||||
|
||||
### 1. Create request.jsonl
|
||||
|
||||
- Check models available via `/model_group/info`
|
||||
- See all models with `mode: batch`
|
||||
- Set `model` in .jsonl to the model from `/model_group/info`
|
||||
|
||||
```json showLineNumbers title="bedrock_batch_completions.jsonl"
|
||||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are an unhelpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
|
||||
```
|
||||
|
||||
Expectation:
|
||||
|
||||
- LiteLLM translates this to the bedrock deployment specific value (e.g. `bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0`)
|
||||
|
||||
### 2. Upload File
|
||||
|
||||
Specify `target_model_names: "<model-name>"` to enable LiteLLM managed files and request validation.
|
||||
|
||||
model-name should be the same as the model-name in the request.jsonl
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="python" label="Python">
|
||||
|
||||
```python showLineNumbers title="bedrock_batch.py"
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
base_url="http://0.0.0.0:4000",
|
||||
api_key="sk-1234",
|
||||
)
|
||||
|
||||
# Upload file
|
||||
batch_input_file = client.files.create(
|
||||
file=open("./bedrock_batch_completions.jsonl", "rb"), # {"model": "bedrock-batch-claude"} <-> {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"}
|
||||
purpose="batch",
|
||||
extra_body={"target_model_names": "bedrock-batch-claude"}
|
||||
)
|
||||
print(batch_input_file)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="curl" label="Curl">
|
||||
|
||||
```bash showLineNumbers title="Upload File"
|
||||
curl http://localhost:4000/v1/files \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-F purpose="batch" \
|
||||
-F file="@bedrock_batch_completions.jsonl" \
|
||||
-F extra_body='{"target_model_names": "bedrock-batch-claude"}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
**Where is the file written?**:
|
||||
|
||||
The file is written to S3 bucket specified in your config and prepared for Bedrock batch inference.
|
||||
|
||||
### 3. Create the batch
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="python" label="Python">
|
||||
|
||||
```python showLineNumbers title="bedrock_batch.py"
|
||||
...
|
||||
# Create batch
|
||||
batch = client.batches.create(
|
||||
input_file_id=batch_input_file.id,
|
||||
endpoint="/v1/chat/completions",
|
||||
completion_window="24h",
|
||||
metadata={"description": "Test batch job"},
|
||||
)
|
||||
print(batch)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="curl" label="Curl">
|
||||
|
||||
```bash showLineNumbers title="Create Batch Request"
|
||||
curl http://localhost:4000/v1/batches \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input_file_id": "file-abc123",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"metadata": {"description": "Test batch job"}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## FAQ
|
||||
|
||||
### Where are my files written?
|
||||
|
||||
When a `target_model_names` is specified, the file is written to the S3 bucket configured in your Bedrock batch model configuration.
|
||||
|
||||
### What models are supported?
|
||||
|
||||
LiteLLM only supports Bedrock Anthropic Models for Batch API. If you want other bedrock models file an issue [here](https://github.com/BerriAI/litellm/issues/new/choose).
|
||||
|
||||
## Further Reading
|
||||
|
||||
- [AWS Bedrock Batch Inference Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)
|
||||
- [LiteLLM Managed Batches](../proxy/managed_batches)
|
||||
- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)
|
||||
@@ -1,4 +1,4 @@
|
||||
# Dashscope
|
||||
# Dashscope (Qwen API)
|
||||
https://dashscope.console.aliyun.com/
|
||||
|
||||
**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# 🆕 OVHCloud AI Endpoints
|
||||
Leading French Cloud provider in Europe with data sovereignty and privacy.
|
||||
|
||||
You can explore the last models we made available in our [catalog](https://endpoints.ai.cloud.ovh.net/catalog).
|
||||
|
||||
:::tip
|
||||
|
||||
We support ALL OVHCloud AI Endpoints models, just set `model=ovhcloud/<any-model-on-ai-endpoints>` as a prefix when sending litellm requests.
|
||||
For the complete models catalog, visit https://endpoints.ai.cloud.ovh.net/catalog. **
|
||||
|
||||
:::
|
||||
|
||||
## Sample usage
|
||||
### Chat completion
|
||||
You can define your API key by setting the `OVHCLOUD_API_KEY` environment variable or by overriding the `api_key` parameter. You can generate a key on the [OVHCloud Manager](https://www.ovh.com/manager).
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
# Our API is free but ratelimited for calls without an API key.
|
||||
os.environ['OVHCLOUD_API_KEY'] = "your-api-key"
|
||||
|
||||
response = completion(
|
||||
model = "ovhcloud/Meta-Llama-3_3-70B-Instruct",
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?",
|
||||
}
|
||||
],
|
||||
max_tokens = 10,
|
||||
stop = [],
|
||||
temperature = 0.2,
|
||||
top_p = 0.9,
|
||||
user = "user",
|
||||
api_key = "your-api-key" # Optional if set through the enviromnent variable.
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Streaming
|
||||
Set the parameter `stream` to `True` to stream a response.
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['OVHCLOUD_API_KEY'] = "your-api-key"
|
||||
|
||||
response = completion(
|
||||
model = "ovhcloud/Meta-Llama-3_3-70B-Instruct",
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?",
|
||||
}
|
||||
],
|
||||
max_tokens = 10,
|
||||
stop = [],
|
||||
temperature = 0.2,
|
||||
top_p = 0.9,
|
||||
user = "user",
|
||||
api_key = "your-api-key" # Optional if set through the enviromnent variable,
|
||||
stream = True
|
||||
)
|
||||
|
||||
for part in response:
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Tool Calling
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import json
|
||||
|
||||
def get_current_weather(location, unit="celsius"):
|
||||
if unit == "celsius":
|
||||
return {"location": location, "temperature": "22", "unit": "celsius"}
|
||||
else:
|
||||
return {"location": location, "temperature": "72", "unit": "fahrenheit"}
|
||||
|
||||
def print_message(role, content, is_tool_call=False, function_name=None):
|
||||
if role == "user":
|
||||
print(f"🧑 User: {content}")
|
||||
elif role == "assistant":
|
||||
if is_tool_call:
|
||||
print(f"🤖 Assistant: I will call the function '{function_name}' to get some informations.")
|
||||
else:
|
||||
print(f"🤖 Assistant: {content}")
|
||||
elif role == "tool":
|
||||
print(f"🔧 Tool ({function_name}): {content}")
|
||||
print()
|
||||
|
||||
messages = [{"role": "user", "content": "What's the weather like in Paris?"}]
|
||||
model = "ovhcloud/Meta-Llama-3_3-70B-Instruct"
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and country, e.g. Montréal, Canada",
|
||||
},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
print("🌟 Beginning of the conversation")
|
||||
|
||||
# Initial user message
|
||||
print_message("user", messages[0]["content"])
|
||||
|
||||
# First request to the model
|
||||
print("📡 Sending first request to the model...")
|
||||
response = completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
response_message = response.choices[0].message
|
||||
tool_calls = response_message.tool_calls
|
||||
|
||||
if tool_calls:
|
||||
available_functions = {
|
||||
"get_current_weather": get_current_weather,
|
||||
}
|
||||
|
||||
# Display the tool calls suggested by the model
|
||||
for tool_call in tool_calls:
|
||||
print_message("assistant", "", is_tool_call=True, function_name=tool_call.function.name)
|
||||
print(f" 📋 Arguments: {tool_call.function.arguments}")
|
||||
print()
|
||||
|
||||
# Add assistant message with tool calls to the conversation history
|
||||
assistant_message = {
|
||||
"role": "assistant",
|
||||
"content": response_message.content,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": tool_call.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tool_call.function.name,
|
||||
"arguments": tool_call.function.arguments
|
||||
}
|
||||
} for tool_call in tool_calls
|
||||
]
|
||||
}
|
||||
|
||||
messages.append(assistant_message)
|
||||
|
||||
# Execute each tool call and add the results to the conversation history
|
||||
for tool_call in tool_calls:
|
||||
function_name = tool_call.function.name
|
||||
function_to_call = available_functions[function_name]
|
||||
function_args = json.loads(tool_call.function.arguments)
|
||||
|
||||
print(f"🔧 Executing function '{function_name}'...")
|
||||
function_response = function_to_call(
|
||||
location=function_args.get("location"),
|
||||
unit=function_args.get("unit"),
|
||||
)
|
||||
|
||||
# Display tool response
|
||||
print_message("tool", json.dumps(function_response, indent=2), function_name=function_name)
|
||||
|
||||
messages.append({
|
||||
"tool_call_id": tool_call.id,
|
||||
"role": "tool",
|
||||
"name": function_name,
|
||||
"content": json.dumps(function_response),
|
||||
})
|
||||
|
||||
print("📡 Sending second request to the model with results...")
|
||||
|
||||
# Second request with function results
|
||||
second_response = completion(
|
||||
model=model,
|
||||
messages=messages
|
||||
)
|
||||
|
||||
# Display final response
|
||||
final_content = second_response.choices[0].message.content
|
||||
print_message("assistant", final_content)
|
||||
|
||||
else:
|
||||
print("❌ No function call detected")
|
||||
print_message("assistant", response_message.content)
|
||||
```
|
||||
|
||||
### Vision Example
|
||||
|
||||
```python
|
||||
from base64 import b64encode
|
||||
from mimetypes import guess_type
|
||||
import litellm
|
||||
|
||||
# Auxiliary function to get b64 images
|
||||
def data_url_from_image(file_path):
|
||||
mime_type, _ = guess_type(file_path)
|
||||
if mime_type is None:
|
||||
raise ValueError("Could not determine MIME type of the file")
|
||||
|
||||
with open(file_path, "rb") as image_file:
|
||||
encoded_string = b64encode(image_file.read()).decode("utf-8")
|
||||
|
||||
data_url = f"data:{mime_type};base64,{encoded_string}"
|
||||
return data_url
|
||||
|
||||
response = litellm.completion(
|
||||
model = "ovhcloud/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What's in this image?"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": data_url_from_image("your_image.jpg"),
|
||||
"format": "image/jpeg"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
stream=False
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
|
||||
### Structured Output
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="ovhcloud/Meta-Llama-3_3-70B-Instruct",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a specialist in extracting structured data from unstructured text. "
|
||||
"Your task is to identify relevant entities and categories, then format them "
|
||||
"according to the requested structure."
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Room 12 contains books, a desk, and a lamp."
|
||||
},
|
||||
],
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"title": "data",
|
||||
"name": "data_extraction",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"section": {"type": "string"},
|
||||
"products": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["section", "products"],
|
||||
"additionalProperties": False
|
||||
},
|
||||
"strict": False
|
||||
}
|
||||
},
|
||||
stream=False
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### Embeddings
|
||||
|
||||
```python
|
||||
from litellm import embedding
|
||||
|
||||
response = embedding(
|
||||
model="ovhcloud/BGE-M3",
|
||||
input=["sample text to embed", "another sample text to embed"]
|
||||
)
|
||||
|
||||
print(response.data)
|
||||
```
|
||||
|
||||
## Usage with LiteLLM Proxy Server
|
||||
|
||||
Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server
|
||||
|
||||
1. Modify the config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-model
|
||||
litellm_params:
|
||||
model: ovhcloud/<your-model-name> # add ovhcloud/ prefix to route as OVHCloud provider
|
||||
api_key: api-key # api key to send your model
|
||||
```
|
||||
|
||||
|
||||
2. Start the proxy
|
||||
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Send Request to LiteLLM Proxy Server
|
||||
|
||||
<Tabs>
|
||||
|
||||
<TabItem value="openai" label="OpenAI Python v1.0.0+">
|
||||
|
||||
```python
|
||||
import openai
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
|
||||
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="my-model",
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="curl">
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "my-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
@@ -8,9 +8,9 @@ LiteLLM supports all models on VLLM.
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | vLLM is a fast and easy-to-use library for LLM inference and serving. [Docs](https://docs.vllm.ai/en/latest/index.html) |
|
||||
| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` (for vLLM sdk usage) |
|
||||
| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` ([DEPRECATED] for vLLM sdk usage) |
|
||||
| Provider Doc | [vLLM ↗](https://docs.vllm.ai/en/latest/index.html) |
|
||||
| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank` |
|
||||
| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank`, `/audio/transcriptions` |
|
||||
|
||||
|
||||
# Quick Start
|
||||
|
||||
@@ -4,6 +4,10 @@ import TabItem from '@theme/TabItem';
|
||||
|
||||
# ✨ SSO for Admin UI
|
||||
|
||||
:::info
|
||||
From v1.76.0, SSO is now Free for up to 5 users.
|
||||
:::
|
||||
|
||||
:::info
|
||||
|
||||
✨ SSO is on LiteLLM Enterprise
|
||||
|
||||
@@ -473,6 +473,7 @@ router_settings:
|
||||
| EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
|
||||
| EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails.
|
||||
| EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails.
|
||||
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
|
||||
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
|
||||
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
|
||||
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
|
||||
|
||||
@@ -11,13 +11,13 @@ The proxy also supports json logs. [See here](#json-logs)
|
||||
|
||||
**via cli**
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
$ litellm --debug
|
||||
```
|
||||
|
||||
**via env**
|
||||
|
||||
```python
|
||||
```python showLineNumbers
|
||||
os.environ["LITELLM_LOG"] = "INFO"
|
||||
```
|
||||
|
||||
@@ -25,25 +25,25 @@ os.environ["LITELLM_LOG"] = "INFO"
|
||||
|
||||
**via cli**
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
$ litellm --detailed_debug
|
||||
```
|
||||
|
||||
**via env**
|
||||
|
||||
```python
|
||||
```python showLineNumbers
|
||||
os.environ["LITELLM_LOG"] = "DEBUG"
|
||||
```
|
||||
|
||||
### Debug Logs
|
||||
|
||||
Run the proxy with `--detailed_debug` to view detailed debug logs
|
||||
```shell
|
||||
```shell showLineNumbers
|
||||
litellm --config /path/to/config.yaml --detailed_debug
|
||||
```
|
||||
|
||||
When making requests you should see the POST request sent by LiteLLM to the LLM on the Terminal output
|
||||
```shell
|
||||
```shell showLineNumbers
|
||||
POST Request Sent from LiteLLM:
|
||||
curl -X POST \
|
||||
https://api.openai.com/v1/chat/completions \
|
||||
@@ -51,25 +51,63 @@ https://api.openai.com/v1/chat/completions \
|
||||
-d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "this is a test request, write a short poem"}]}'
|
||||
```
|
||||
|
||||
## Debug single request
|
||||
|
||||
Pass in `litellm_request_debug=True` in the request body
|
||||
|
||||
```bash showLineNumbers
|
||||
curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model":"fake-openai-endpoint",
|
||||
"messages": [{"role": "user","content": "How many r in the word strawberry?"}],
|
||||
"litellm_request_debug": true
|
||||
}'
|
||||
```
|
||||
|
||||
This will emit the raw request sent by LiteLLM to the API Provider and raw response received from the API Provider for **just** this request in the logs.
|
||||
|
||||
|
||||
```bash showLineNumbers
|
||||
INFO: Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit)
|
||||
20:14:06 - LiteLLM:WARNING: litellm_logging.py:938 -
|
||||
|
||||
POST Request Sent from LiteLLM:
|
||||
curl -X POST \
|
||||
https://exampleopenaiendpoint-production.up.railway.app/chat/completions \
|
||||
-H 'Authorization: Be****ey' -H 'Content-Type: application/json' \
|
||||
-d '{'model': 'fake', 'messages': [{'role': 'user', 'content': 'How many r in the word strawberry?'}], 'stream': False}'
|
||||
|
||||
|
||||
20:14:06 - LiteLLM:WARNING: litellm_logging.py:1015 - RAW RESPONSE:
|
||||
{"id":"chatcmpl-817fc08f0d6c451485d571dab39b26a1","object":"chat.completion","created":1677652288,"model":"gpt-3.5-turbo-0301","system_fingerprint":"fp_44709d6fcb","choices":[{"index":0,"message":{"role":"assistant","content":"\n\nHello there, how may I assist you today?"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}}
|
||||
|
||||
|
||||
INFO: 127.0.0.1:56155 - "POST /chat/completions HTTP/1.1" 200 OK
|
||||
|
||||
```
|
||||
|
||||
|
||||
## JSON LOGS
|
||||
|
||||
Set `JSON_LOGS="True"` in your env:
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
export JSON_LOGS="True"
|
||||
```
|
||||
**OR**
|
||||
|
||||
Set `json_logs: true` in your yaml:
|
||||
|
||||
```yaml
|
||||
```yaml showLineNumbers
|
||||
litellm_settings:
|
||||
json_logs: true
|
||||
```
|
||||
|
||||
Start proxy
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
$ litellm
|
||||
```
|
||||
|
||||
@@ -80,7 +118,7 @@ The proxy will now all logs in json format.
|
||||
Turn off fastapi's default 'INFO' logs
|
||||
|
||||
1. Turn on 'json logs'
|
||||
```yaml
|
||||
```yaml showLineNumbers
|
||||
litellm_settings:
|
||||
json_logs: true
|
||||
```
|
||||
@@ -89,20 +127,20 @@ litellm_settings:
|
||||
|
||||
Only get logs if an error occurs.
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
LITELLM_LOG="ERROR"
|
||||
```
|
||||
|
||||
3. Start proxy
|
||||
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
$ litellm
|
||||
```
|
||||
|
||||
Expected Output:
|
||||
|
||||
```bash
|
||||
```bash showLineNumbers
|
||||
# no info statements
|
||||
```
|
||||
|
||||
@@ -119,14 +157,14 @@ This can be caused due to all your models hitting rate limit errors, causing the
|
||||
How to control this?
|
||||
- Adjust the cooldown time
|
||||
|
||||
```yaml
|
||||
```yaml showLineNumbers
|
||||
router_settings:
|
||||
cooldown_time: 0 # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
- Disable Cooldowns [NOT RECOMMENDED]
|
||||
|
||||
```yaml
|
||||
```yaml showLineNumbers
|
||||
router_settings:
|
||||
disable_cooldowns: True
|
||||
```
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# Forward Client Headers to LLM API
|
||||
|
||||
Control which model groups can forward client headers to the underlying LLM provider APIs.
|
||||
|
||||
## Overview
|
||||
|
||||
By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting.
|
||||
|
||||
## Configuration
|
||||
|
||||
## Enable Globally
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
forward_client_headers_to_llm_api: true
|
||||
```
|
||||
|
||||
## Enable for a Model Group
|
||||
|
||||
Add the `forward_client_headers_to_llm_api` setting under `model_group_settings` in your configuration:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4o-mini
|
||||
litellm_params:
|
||||
model: openai/gpt-4o-mini
|
||||
api_key: "your-api-key"
|
||||
- model_name: "wildcard-models/*"
|
||||
litellm_params:
|
||||
model: "openai/*"
|
||||
api_key: "your-api-key"
|
||||
|
||||
litellm_settings:
|
||||
model_group_settings:
|
||||
forward_client_headers_to_llm_api:
|
||||
- gpt-4o-mini
|
||||
- wildcard-models/*
|
||||
```
|
||||
|
||||
## Supported Model Patterns
|
||||
|
||||
The configuration supports various model matching patterns:
|
||||
|
||||
### 1. Exact Model Names
|
||||
```yaml
|
||||
forward_client_headers_to_llm_api:
|
||||
- gpt-4o-mini
|
||||
- claude-3-sonnet
|
||||
```
|
||||
|
||||
### 2. Wildcard Patterns
|
||||
```yaml
|
||||
forward_client_headers_to_llm_api:
|
||||
- "openai/*" # All OpenAI models
|
||||
- "anthropic/*" # All Anthropic models
|
||||
- "wildcard-group/*" # All models in wildcard-group
|
||||
```
|
||||
|
||||
### 3. Team Model Aliases
|
||||
If your team has model aliases configured, the forwarding will work with both the original model name and the alias.
|
||||
|
||||
## Forwarded Headers
|
||||
|
||||
When enabled for a model group, LiteLLM forwards the following types of headers:
|
||||
|
||||
### Custom Headers (x- prefix)
|
||||
- Any header starting with `x-` (except `x-stainless-*` which can cause OpenAI SDK issues)
|
||||
- Examples: `x-custom-header`, `x-request-id`, `x-trace-id`
|
||||
|
||||
### Provider-Specific Headers
|
||||
- **Anthropic**: `anthropic-beta` headers
|
||||
- **OpenAI**: `openai-organization` (when enabled via `forward_openai_org_id: true`)
|
||||
|
||||
### User Information Headers (Optional)
|
||||
When `add_user_information_to_llm_headers` is enabled, LiteLLM adds:
|
||||
- `x-litellm-user-id`
|
||||
- `x-litellm-org-id`
|
||||
- Other user metadata as `x-litellm-*` headers
|
||||
|
||||
## Security Considerations
|
||||
|
||||
⚠️ **Important Security Notes:**
|
||||
|
||||
1. **Sensitive Data**: Only enable header forwarding for trusted model groups, as headers may contain sensitive information
|
||||
2. **API Keys**: Never include API keys or secrets in forwarded headers
|
||||
3. **PII**: Be cautious about forwarding headers that might contain personally identifiable information
|
||||
4. **Provider Limits**: Some providers have restrictions on custom headers
|
||||
|
||||
## Example Use Cases
|
||||
|
||||
### 1. Request Tracing
|
||||
Forward tracing headers to track requests across your system:
|
||||
|
||||
```bash
|
||||
curl -X POST "https://your-proxy.com/v1/chat/completions" \
|
||||
-H "Authorization: Bearer your-key" \
|
||||
-H "x-trace-id: abc123" \
|
||||
-H "x-request-source: mobile-app" \
|
||||
-d '{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "Hello"}]
|
||||
}'
|
||||
```
|
||||
|
||||
### 2. Custom Metadata
|
||||
Pass custom metadata to your LLM provider:
|
||||
|
||||
```bash
|
||||
curl -X POST "https://your-proxy.com/v1/chat/completions" \
|
||||
-H "Authorization: Bearer your-key" \
|
||||
-H "x-customer-id: customer-123" \
|
||||
-H "x-environment: production" \
|
||||
-d '{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "Hello"}]
|
||||
}'
|
||||
```
|
||||
|
||||
### 3. Anthropic Beta Features
|
||||
Enable beta features for Anthropic models:
|
||||
|
||||
```bash
|
||||
curl -X POST "https://your-proxy.com/v1/chat/completions" \
|
||||
-H "Authorization: Bearer your-key" \
|
||||
-H "anthropic-beta: tools-2024-04-04" \
|
||||
-d '{
|
||||
"model": "claude-3-sonnet",
|
||||
"messages": [{"role": "user", "content": "Hello"}]
|
||||
}'
|
||||
```
|
||||
|
||||
## Complete Configuration Example
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
# Fixed model with header forwarding
|
||||
- model_name: byok-fixed-gpt-4o-mini
|
||||
litellm_params:
|
||||
model: openai/gpt-4o-mini
|
||||
api_base: "https://your-openai-endpoint.com"
|
||||
api_key: "your-api-key"
|
||||
|
||||
# Wildcard model group with header forwarding
|
||||
- model_name: "byok-wildcard/*"
|
||||
litellm_params:
|
||||
model: "openai/*"
|
||||
api_base: "https://your-openai-endpoint.com"
|
||||
api_key: "your-api-key"
|
||||
|
||||
# Standard model without header forwarding
|
||||
- model_name: standard-gpt-4
|
||||
litellm_params:
|
||||
model: openai/gpt-4
|
||||
api_key: "your-api-key"
|
||||
|
||||
litellm_settings:
|
||||
# Enable user info headers globally (optional)
|
||||
add_user_information_to_llm_headers: true
|
||||
|
||||
model_group_settings:
|
||||
forward_client_headers_to_llm_api:
|
||||
- byok-fixed-gpt-4o-mini
|
||||
- byok-wildcard/*
|
||||
# Note: standard-gpt-4 is NOT included, so no headers forwarded
|
||||
|
||||
general_settings:
|
||||
# Enable OpenAI organization header forwarding (optional)
|
||||
forward_openai_org_id: true
|
||||
```
|
||||
|
||||
## Testing Header Forwarding
|
||||
|
||||
To test if headers are being forwarded:
|
||||
|
||||
1. **Enable Debug Logging**: Set `set_verbose: true` in your config
|
||||
2. **Check Provider Logs**: Monitor your LLM provider's request logs
|
||||
3. **Use Webhook Sites**: For testing, you can use webhook.site URLs as api_base to see forwarded headers
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Headers Not Being Forwarded
|
||||
|
||||
1. **Check Model Name**: Ensure the model name in your request matches the configuration
|
||||
2. **Verify Pattern Matching**: Wildcard patterns must match exactly
|
||||
3. **Review Logs**: Enable verbose logging to see header processing
|
||||
|
||||
### Provider Errors
|
||||
|
||||
1. **Invalid Headers**: Some providers reject unknown headers
|
||||
2. **Header Limits**: Providers may have limits on header count/size
|
||||
3. **Authentication**: Ensure forwarded headers don't conflict with authentication
|
||||
|
||||
## Related Features
|
||||
|
||||
- [Request Headers](./request_headers.md) - Complete list of supported request headers
|
||||
- [Response Headers](./response_headers.md) - Headers returned by LiteLLM
|
||||
- [Team Model Aliases](./team_model_add.md) - Configure model aliases for teams
|
||||
- [Model Access Control](./model_access.md) - Control which users can access which models
|
||||
|
||||
## API Reference
|
||||
|
||||
The header forwarding is controlled by the `ModelGroupSettings` configuration:
|
||||
|
||||
```python
|
||||
class ModelGroupSettings(BaseModel):
|
||||
forward_client_headers_to_llm_api: Optional[List[str]] = None
|
||||
```
|
||||
|
||||
Where each string in the list can be:
|
||||
- An exact model name (e.g., `"gpt-4o-mini"`)
|
||||
- A wildcard pattern (e.g., `"openai/*"`)
|
||||
- A model group name (e.g., `"my-model-group/*"`)
|
||||
@@ -135,6 +135,7 @@ guardrails:
|
||||
# application_id: "my-app"
|
||||
# monitor_mode: false
|
||||
# block_failures: true
|
||||
# anonymize_input: false
|
||||
```
|
||||
|
||||
### Required Parameters
|
||||
@@ -147,6 +148,7 @@ guardrails:
|
||||
- **`application_id`**: Your application identifier (defaults to `"litellm"`)
|
||||
- **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`)
|
||||
- **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`)
|
||||
- **`anonymize_input`**: If `true`, replaces sensitive content with anonymized version (defaults to `false`)
|
||||
|
||||
## Environment Variables
|
||||
|
||||
@@ -158,6 +160,7 @@ export NOMA_API_BASE="https://api.noma.security/" # Optional
|
||||
export NOMA_APPLICATION_ID="my-app" # Optional
|
||||
export NOMA_MONITOR_MODE="false" # Optional
|
||||
export NOMA_BLOCK_FAILURES="true" # Optional
|
||||
export NOMA_ANONYMIZE_INPUT="false" # Optional
|
||||
```
|
||||
|
||||
## Advanced Configuration
|
||||
@@ -190,6 +193,20 @@ guardrails:
|
||||
block_failures: false # Allow requests to proceed if guardrail API fails
|
||||
```
|
||||
|
||||
### Content Anonymization
|
||||
|
||||
Enable anonymization to replace sensitive content instead of blocking:
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "noma-anonymize"
|
||||
litellm_params:
|
||||
guardrail: noma
|
||||
mode: "pre_call"
|
||||
api_key: os.environ/NOMA_API_KEY
|
||||
anonymize_input: true # Replace sensitive data with anonymized version
|
||||
```
|
||||
|
||||
### Multiple Guardrails
|
||||
|
||||
Apply different configurations for input and output:
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Tool Permission Guardrail
|
||||
|
||||
LiteLLM provides a Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools).
|
||||
|
||||
## Quick Start
|
||||
### 1. Define Guardrails on your LiteLLM config.yaml
|
||||
|
||||
Define your guardrails under the `guardrails` section
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "tool-permission-guardrail"
|
||||
litellm_params:
|
||||
guardrail: tool_permission
|
||||
mode: "post_call"
|
||||
rules:
|
||||
- id: "allow_bash"
|
||||
tool_name: "Bash"
|
||||
decision: "allow"
|
||||
- id: "allow_github_mcp"
|
||||
tool_name: "mcp__github_*"
|
||||
decision: "allow"
|
||||
- id: "allow_aws_documentation"
|
||||
tool_name: "mcp__aws-documentation_*_documentation"
|
||||
decision: "allow"
|
||||
- id: "deny_read_commands"
|
||||
tool_name: "Read"
|
||||
decision: "Deny"
|
||||
default_action: "deny" # Fallback when no rule matches: "allow" or "deny"
|
||||
on_disallowed_action: "block" # How to handle disallowed tools: "block" or "rewrite"
|
||||
```
|
||||
|
||||
#### Rule Structure
|
||||
|
||||
```yaml
|
||||
- id: "unique_rule_id" # Unique identifier for the rule
|
||||
tool_name: "pattern" # Tool name or pattern to match
|
||||
decision: "allow" # "allow" or "deny"
|
||||
```
|
||||
|
||||
#### Supported values for `mode`
|
||||
|
||||
- `pre_call` Run **before** LLM call, on **input**
|
||||
- `post_call` Run **after** LLM call, on **input & output**
|
||||
|
||||
### 2. Start the Proxy
|
||||
|
||||
```shell
|
||||
litellm --config config.yaml --port 4000
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="block" label="Block Request">
|
||||
|
||||
**Block requset**
|
||||
|
||||
```bash
|
||||
# Test
|
||||
curl -X POST "http://localhost:4000/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer your-master-key-here" \
|
||||
-d '{
|
||||
"model": "gpt-5-mini",
|
||||
"messages": [{"role": "user","content": "What is the weather like in Tokyo today?"}],
|
||||
"tools": [
|
||||
{
|
||||
"type":"function",
|
||||
"function": {
|
||||
"name":"get_current_weather",
|
||||
"description": "Get the current weather in a given location"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
**Expected response (Denied):**
|
||||
|
||||
```json
|
||||
{
|
||||
"error":
|
||||
{
|
||||
"message": "Guardrail raised an exception, Guardrail: tool-permission-guardrail, Message: Tool 'get_current_weather' denied by default action",
|
||||
"type": "None",
|
||||
"param": "None",
|
||||
"code": "500"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="rewrite" label="Rewrite Request">
|
||||
|
||||
**Rewrite requset**
|
||||
|
||||
```bash
|
||||
# Test
|
||||
curl -X POST "http://localhost:4000/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer your-master-key-here" \
|
||||
-d '{
|
||||
"model": "gpt-5-mini",
|
||||
"messages": [{"role": "user","content": "What is the weather like in Tokyo today?"}],
|
||||
"tools": [
|
||||
{
|
||||
"type":"function",
|
||||
"function": {
|
||||
"name":"get_current_weather",
|
||||
"description": "Get the current weather in a given location"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
**Expected response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "chatcmpl-xxxxxxxxxxxxxxx",
|
||||
"created": 1757716050,
|
||||
"model": "gpt-5-mini-2025-08-07",
|
||||
"object": "chat.completion",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "I can’t fetch live weather — I don’t have real‑time internet access.",
|
||||
"role": "assistant",
|
||||
"annotations": []
|
||||
},
|
||||
"provider_specific_fields": {}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 112,
|
||||
"total_tokens": 735,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 384,
|
||||
},
|
||||
},
|
||||
"service_tier": "default"
|
||||
}
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
@@ -2,6 +2,10 @@
|
||||
|
||||
Special headers that are supported by LiteLLM.
|
||||
|
||||
## Header Forwarding
|
||||
|
||||
By default, LiteLLM does not forward client headers to LLM provider APIs. However, you can selectively enable header forwarding for specific model groups. [Learn more about configuring header forwarding](./forward_client_headers.md).
|
||||
|
||||
## LiteLLM Headers
|
||||
|
||||
`x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
|
||||
@@ -21,11 +25,15 @@ Special headers that are supported by LiteLLM.
|
||||
`anthropic-version` Optional[str]: The version of the Anthropic API to use.
|
||||
`anthropic-beta` Optional[str]: The beta version of the Anthropic API to use.
|
||||
- For `/v1/messages` endpoint, this will always be forward the header to the underlying model.
|
||||
- For `/chat/completions` endpoint, this will only be forwarded if `forward_client_headers_to_llm_api` is true.
|
||||
- For `/chat/completions` endpoint, this will only be forwarded if the model is configured in `forward_client_headers_to_llm_api`. [Learn more](./forward_client_headers.md)
|
||||
|
||||
## OpenAI Headers
|
||||
|
||||
`openai-organization` Optional[str]: The organization to use for the OpenAI API. (currently needs to be enabled via `general_settings::forward_openai_org_id: true`)
|
||||
|
||||
## Custom Headers
|
||||
|
||||
Custom headers starting with `x-` can be forwarded to LLM provider APIs when the model is configured in `forward_client_headers_to_llm_api`. [Learn more about header forwarding configuration](./forward_client_headers.md).
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -89,16 +89,20 @@ To track spend and usage for each Open WebUI user, configure both Open WebUI and
|
||||
|
||||
2. **Configure LiteLLM to Parse User Headers**
|
||||
|
||||
Add the following to your LiteLLM `config.yaml` to specify a header to use for user tracking:
|
||||
Add the following to your LiteLLM `config.yaml` to specify the request header mapping for user tracking:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
user_header_name: X-OpenWebUI-User-Id
|
||||
user_header_mappings:
|
||||
- header_name: X-OpenWebUI-User-Id
|
||||
litellm_user_role: internal_user
|
||||
- header_name: X-OpenWebUI-User-Email
|
||||
litellm_user_role: customer
|
||||
```
|
||||
|
||||
ⓘ Available tracking options
|
||||
|
||||
You can use any of the following headers for `user_header_name`:
|
||||
You can use any of the following headers in `header_name` in `user_header_mappings` :
|
||||
- `X-OpenWebUI-User-Id`
|
||||
- `X-OpenWebUI-User-Email`
|
||||
- `X-OpenWebUI-User-Name`
|
||||
@@ -109,6 +113,12 @@ To track spend and usage for each Open WebUI user, configure both Open WebUI and
|
||||
- Users can modify their own usernames
|
||||
- Administrators can modify both usernames and emails of any account
|
||||
|
||||
This video walks through on how we can map the openweb ui headers to LiteLLM user roles
|
||||
|
||||
<iframe src="https://www.loom.com/embed/a1b6a4635fc0478ba4fd34cae16e2ffd?sid=791c2dcc-7e65-45be-bf7f-27d2601c123e" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen width="840" height="500"></iframe>
|
||||
|
||||
<br/>
|
||||
<br/>
|
||||
|
||||
|
||||
## Render `thinking` content on Open WebUI
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
@@ -0,0 +1,155 @@
|
||||
---
|
||||
title: "v1.77.2-stable - Bedrock Batches API"
|
||||
slug: "v1-77-2"
|
||||
date: 2025-09-13T10:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaffer
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
``` showLineNumbers title="docker run litellm"
|
||||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
ghcr.io/berriai/litellm:v1.77.2
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
``` showLineNumbers title="pip install litellm"
|
||||
pip install litellm==1.77.2
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Key Highlights
|
||||
|
||||
- **Bedrock Batches API** - Support for creating Batch Inference Jobs on Bedrock using LiteLLM's unified batch API (OpenAI compatible)
|
||||
- **Qwen API Tiered Pricing** - Cost tracking support for Dashscope (Qwen) models with multiple pricing tiers
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Pricing ($/1M tokens) | Features |
|
||||
| ----------- | ------------------------------- | -------------- | --------------------- | -------- |
|
||||
| DeepInfra | `deepinfra/deepseek-ai/DeepSeek-R1` | 164K | **Input:** $0.70<br/>**Output:** $2.40 | Chat completions, tool calling |
|
||||
| Heroku | `heroku/claude-4-sonnet` | 8K | Contact provider for pricing | Function calling, tool choice |
|
||||
| Heroku | `heroku/claude-3-7-sonnet` | 8K | Contact provider for pricing | Function calling, tool choice |
|
||||
| Heroku | `heroku/claude-3-5-sonnet-latest` | 8K | Contact provider for pricing | Function calling, tool choice |
|
||||
| Heroku | `heroku/claude-3-5-haiku` | 4K | Contact provider for pricing | Function calling, tool choice |
|
||||
| Dashscope | `dashscope/qwen-plus-latest` | 1M | **Tiered Pricing:**<br/>• 0-256K tokens: $0.40 / $1.20<br/>• 256K-1M tokens: $1.20 / $3.60 | Function calling, reasoning |
|
||||
| Dashscope | `dashscope/qwen3-max-preview` | 262K | **Tiered Pricing:**<br/>• 0-32K tokens: $1.20 / $6.00<br/>• 32K-128K tokens: $2.40 / $12.00<br/>• 128K-252K tokens: $3.00 / $15.00 | Function calling, reasoning |
|
||||
| Dashscope | `dashscope/qwen-flash` | 1M | **Tiered Pricing:**<br/>• 0-256K tokens: $0.05 / $0.40<br/>• 256K-1M tokens: $0.25 / $2.00 | Function calling, reasoning |
|
||||
| Dashscope | `dashscope/qwen3-coder-plus` | 1M | **Tiered Pricing:**<br/>• 0-32K tokens: $1.00 / $5.00<br/>• 32K-128K tokens: $1.80 / $9.00<br/>• 128K-256K tokens: $3.00 / $15.00<br/>• 256K-1M tokens: $6.00 / $60.00 | Function calling, reasoning, caching |
|
||||
| Dashscope | `dashscope/qwen3-coder-flash` | 1M | **Tiered Pricing:**<br/>• 0-32K tokens: $0.30 / $1.50<br/>• 32K-128K tokens: $0.50 / $2.50<br/>• 128K-256K tokens: $0.80 / $4.00<br/>• 256K-1M tokens: $1.60 / $9.60 | Function calling, reasoning, caching |
|
||||
|
||||
---
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Bedrock](../../docs/providers/bedrock_batches)**
|
||||
- Bedrock Batches API - batch processing support with file upload and request transformation - [PR #14518](https://github.com/BerriAI/litellm/pull/14518), [PR #14522](https://github.com/BerriAI/litellm/pull/14522)
|
||||
- **[VLLM](../../docs/providers/vllm)**
|
||||
- Added transcription endpoint support - [PR #14523](https://github.com/BerriAI/litellm/pull/14523)
|
||||
- **[Ollama](../../docs/providers/ollama)**
|
||||
- `ollama_chat/` - images, thinking, and content as list handling - [PR #14523](https://github.com/BerriAI/litellm/pull/14523)
|
||||
- **General**
|
||||
- New debug flag for detailed request/response logging [PR #14482](https://github.com/BerriAI/litellm/pull/14482)
|
||||
|
||||
#### Bug Fixes
|
||||
|
||||
- **[Azure OpenAI](../../docs/providers/azure)**
|
||||
- Fixed extra_body injection causing payload rejection in image generation - [PR #14475](https://github.com/BerriAI/litellm/pull/14475)
|
||||
- **[LM Studio](../../docs/providers/lm-studio)**
|
||||
- Resolved illegal Bearer header value issue - [PR #14512](https://github.com/BerriAI/litellm/pull/14512)
|
||||
|
||||
---
|
||||
|
||||
## LLM API Endpoints
|
||||
|
||||
#### Bug Fixes
|
||||
|
||||
- **[/messages](../../docs/anthropic_unified)**
|
||||
- Don't send content block after message w/ finish reason + usage block - [PR #14477](https://github.com/BerriAI/litellm/pull/14477)
|
||||
- **[/generateContent](../../docs/generateContent)**
|
||||
- Gemini CLI Integration - Fixed token count errors - [PR #14451](https://github.com/BerriAI/litellm/pull/14451), [PR #14417](https://github.com/BerriAI/litellm/pull/14417)
|
||||
|
||||
---
|
||||
|
||||
## Spend Tracking, Budgets and Rate Limiting
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Qwen API Tiered Pricing](../../docs/providers/dashscope)** - Added comprehensive tiered cost tracking for Dashscope/Qwen models - [PR #14471](https://github.com/BerriAI/litellm/pull/14471), [PR #14479](https://github.com/BerriAI/litellm/pull/14479)
|
||||
|
||||
#### Bug Fixes
|
||||
|
||||
- **Provider Budgets** - Fixed provider budget calculations - [PR #14459](https://github.com/BerriAI/litellm/pull/14459)
|
||||
|
||||
---
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
#### Features
|
||||
|
||||
- **User Headers Mapping** - New X-LiteLLM Users mapping feature for enhanced user tracking - [PR #14485](https://github.com/BerriAI/litellm/pull/14485)
|
||||
- **Key Unblocking** - Support for hashed tokens in `/key/unblock` endpoint - [PR #14477](https://github.com/BerriAI/litellm/pull/14477)
|
||||
- **Model Group Header Forwarding** - Enhanced wildcard model support with documentation - [PR #14528](https://github.com/BerriAI/litellm/pull/14528)
|
||||
|
||||
#### Bug Fixes
|
||||
|
||||
- **Log Tab Key Alias** - Fixed filtering inaccuracies for failed logs - [PR #14469](https://github.com/BerriAI/litellm/pull/14469), [PR #14529](https://github.com/BerriAI/litellm/pull/14529)
|
||||
|
||||
---
|
||||
|
||||
## Logging / Guardrail Integrations
|
||||
|
||||
#### Features
|
||||
|
||||
- **Noma Integration** - Added non-blocking monitor mode with anonymize input support - [PR #14401](https://github.com/BerriAI/litellm/pull/14401)
|
||||
|
||||
---
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
#### Performance
|
||||
- Removed dynamic creation of static values - [PR #14538](https://github.com/BerriAI/litellm/pull/14538)
|
||||
- Using `_PROXY_MaxParallelRequestsHandler_v3` by default for optimal throughput - [PR #14450](https://github.com/BerriAI/litellm/pull/14450)
|
||||
- Improved execution context propagation into logging tasks - [PR #14455](https://github.com/BerriAI/litellm/pull/14455)
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
## New Contributors
|
||||
* @Sameerlite made their first contribution in [PR #14460](https://github.com/BerriAI/litellm/pull/14460)
|
||||
* @holzman made their first contribution in [PR #14459](https://github.com/BerriAI/litellm/pull/14459)
|
||||
* @sashank5644 made their first contribution in [PR #14469](https://github.com/BerriAI/litellm/pull/14469)
|
||||
* @TomAlon made their first contribution in [PR #14401](https://github.com/BerriAI/litellm/pull/14401)
|
||||
* @AlexsanderHamir made their first contribution in [PR #14538](https://github.com/BerriAI/litellm/pull/14538)
|
||||
|
||||
---
|
||||
|
||||
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.1.dev.2...v1.77.2.dev)**
|
||||
@@ -49,6 +49,7 @@ const sidebars = {
|
||||
"proxy/guardrails/secret_detection",
|
||||
"proxy/guardrails/custom_guardrail",
|
||||
"proxy/guardrails/prompt_injection",
|
||||
"proxy/guardrails/tool_permission",
|
||||
].sort(),
|
||||
],
|
||||
},
|
||||
@@ -141,6 +142,7 @@ const sidebars = {
|
||||
"proxy/clientside_auth",
|
||||
"proxy/request_headers",
|
||||
"proxy/response_headers",
|
||||
"proxy/forward_client_headers",
|
||||
"proxy/model_discovery",
|
||||
],
|
||||
},
|
||||
@@ -410,6 +412,7 @@ const sidebars = {
|
||||
items: [
|
||||
"providers/bedrock",
|
||||
"providers/bedrock_agents",
|
||||
"providers/bedrock_batches",
|
||||
"providers/bedrock_vector_store",
|
||||
]
|
||||
},
|
||||
@@ -485,7 +488,8 @@ const sidebars = {
|
||||
"providers/bytez",
|
||||
"providers/heroku",
|
||||
"providers/oci",
|
||||
"providers/datarobot",
|
||||
"providers/datarobot",
|
||||
"providers/ovhcloud",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
from typing import Literal, TypedDict
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
||||
class CustomAuthSettings(TypedDict):
|
||||
|
||||
Generated
+5
-4
@@ -6,7 +6,7 @@
|
||||
"": {
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.10.1",
|
||||
"hono": "^4.6.5"
|
||||
"hono": "^4.9.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.17",
|
||||
@@ -463,9 +463,10 @@
|
||||
}
|
||||
},
|
||||
"node_modules/hono": {
|
||||
"version": "4.6.5",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.6.5.tgz",
|
||||
"integrity": "sha512-qsmN3V5fgtwdKARGLgwwHvcdLKursMd+YOt69eGpl1dUCJb8mCd7hZfyZnBYjxCegBG7qkJRQRUy2oO25yHcyQ==",
|
||||
"version": "4.9.7",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.9.7.tgz",
|
||||
"integrity": "sha512-t4Te6ERzIaC48W3x4hJmBwgNlLhmiEdEE5ViYb02ffw4ignHNHa5IBtPjmbKstmtKa8X6C35iWwK4HaqvrzG9w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=16.9.0"
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.10.1",
|
||||
"hono": "^4.6.5"
|
||||
"hono": "^4.9.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.17",
|
||||
|
||||
@@ -243,6 +243,7 @@ nebius_key: Optional[str] = None
|
||||
wandb_key: Optional[str] = None
|
||||
heroku_key: Optional[str] = None
|
||||
cometapi_key: Optional[str] = None
|
||||
ovhcloud_key: Optional[str] = None
|
||||
common_cloud_provider_auth_params: dict = {
|
||||
"params": ["project", "region_name", "token"],
|
||||
"providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
|
||||
@@ -523,6 +524,8 @@ oci_models: Set = set()
|
||||
vercel_ai_gateway_models: Set = set()
|
||||
volcengine_models: Set = set()
|
||||
wandb_models: Set = set(WANDB_MODELS)
|
||||
ovhcloud_models: Set = set()
|
||||
ovhcloud_embedding_models: Set = set()
|
||||
|
||||
|
||||
def is_bedrock_pricing_only_model(key: str) -> bool:
|
||||
@@ -739,6 +742,10 @@ def add_known_models():
|
||||
volcengine_models.add(key)
|
||||
elif value.get("litellm_provider") == "wandb":
|
||||
wandb_models.add(key)
|
||||
elif value.get("litellm_provider") == "ovhcloud":
|
||||
ovhcloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "ovhcloud-embedding-models":
|
||||
ovhcloud_embedding_models.add(key)
|
||||
|
||||
|
||||
add_known_models()
|
||||
@@ -834,6 +841,7 @@ model_list = list(
|
||||
| vercel_ai_gateway_models
|
||||
| volcengine_models
|
||||
| wandb_models
|
||||
| ovhcloud_models
|
||||
)
|
||||
|
||||
model_list_set = set(model_list)
|
||||
@@ -916,6 +924,7 @@ models_by_provider: dict = {
|
||||
"oci": oci_models,
|
||||
"volcengine": volcengine_models,
|
||||
"wandb": wandb_models,
|
||||
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
|
||||
}
|
||||
|
||||
# mapping for those models which have larger equivalents
|
||||
@@ -950,6 +959,7 @@ all_embedding_models = (
|
||||
| fireworks_ai_embedding_models
|
||||
| nebius_embedding_models
|
||||
| sambanova_embedding_models
|
||||
| ovhcloud_embedding_models
|
||||
)
|
||||
|
||||
####### IMAGE GENERATION MODELS ###################
|
||||
@@ -1262,6 +1272,8 @@ from .llms.morph.chat.transformation import MorphChatConfig
|
||||
from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
|
||||
from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
|
||||
from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
|
||||
from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
|
||||
from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig
|
||||
from .main import * # type: ignore
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
|
||||
+61
-18
@@ -19,6 +19,7 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.azure.batches.handler import AzureBatchesAPI
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
@@ -38,6 +39,7 @@ from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
get_litellm_params,
|
||||
get_llm_provider,
|
||||
supports_httpx_timeout,
|
||||
)
|
||||
|
||||
@@ -49,6 +51,45 @@ base_llm_http_handler = BaseLLMHTTPHandler()
|
||||
#################################################
|
||||
|
||||
|
||||
def _resolve_timeout(
|
||||
optional_params: GenericLiteLLMParams,
|
||||
kwargs: Dict[str, Any],
|
||||
custom_llm_provider: str,
|
||||
default_timeout: float = 600.0,
|
||||
) -> float:
|
||||
"""
|
||||
Resolve timeout value from various sources and handle httpx.Timeout objects.
|
||||
|
||||
Args:
|
||||
optional_params: GenericLiteLLMParams object containing timeout
|
||||
kwargs: Additional kwargs that may contain request_timeout
|
||||
custom_llm_provider: Provider name for httpx timeout support check
|
||||
default_timeout: Default timeout value to use
|
||||
|
||||
Returns:
|
||||
Resolved timeout as float
|
||||
"""
|
||||
timeout = optional_params.timeout or kwargs.get("request_timeout", default_timeout) or default_timeout
|
||||
|
||||
# Handle httpx.Timeout objects
|
||||
if isinstance(timeout, httpx.Timeout):
|
||||
if supports_httpx_timeout(custom_llm_provider) is False:
|
||||
# Extract read timeout for providers that don't support httpx.Timeout
|
||||
read_timeout = timeout.read or default_timeout
|
||||
return float(read_timeout)
|
||||
else:
|
||||
# For providers that support httpx.Timeout, we still need to return a float
|
||||
# This case might need to be handled differently based on the actual use case
|
||||
return float(timeout.read or default_timeout)
|
||||
|
||||
# Handle None case
|
||||
if timeout is None:
|
||||
return float(default_timeout)
|
||||
|
||||
# Handle numeric values (int, float, string representations)
|
||||
return float(timeout)
|
||||
|
||||
|
||||
@client
|
||||
async def acreate_batch(
|
||||
completion_window: Literal["24h"],
|
||||
@@ -118,13 +159,23 @@ def create_batch(
|
||||
litellm_call_id = kwargs.get("litellm_call_id", None)
|
||||
proxy_server_request = kwargs.get("proxy_server_request", None)
|
||||
model_info = kwargs.get("model_info", None)
|
||||
model: Optional[str] = kwargs.get("model", None)
|
||||
try:
|
||||
if model is not None:
|
||||
model, _, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=None,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {str(e)}")
|
||||
|
||||
_is_async = kwargs.pop("acreate_batch", False) is True
|
||||
litellm_params = dict(GenericLiteLLMParams(**kwargs))
|
||||
litellm_logging_obj: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None))
|
||||
### TIMEOUT LOGIC ###
|
||||
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
|
||||
timeout = _resolve_timeout(optional_params, kwargs, custom_llm_provider)
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=None,
|
||||
model=model,
|
||||
user=None,
|
||||
optional_params=optional_params.model_dump(),
|
||||
litellm_params={
|
||||
@@ -138,18 +189,6 @@ def create_batch(
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
if (
|
||||
timeout is not None
|
||||
and isinstance(timeout, httpx.Timeout)
|
||||
and supports_httpx_timeout(custom_llm_provider) is False
|
||||
):
|
||||
read_timeout = timeout.read or 600
|
||||
timeout = read_timeout # default 10 min timeout
|
||||
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
|
||||
timeout = float(timeout) # type: ignore
|
||||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
|
||||
_create_batch_request = CreateBatchRequest(
|
||||
@@ -160,10 +199,13 @@ def create_batch(
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
provider_config = ProviderConfigManager.get_provider_batches_config(
|
||||
model="",
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
)
|
||||
if model is not None:
|
||||
provider_config = ProviderConfigManager.get_provider_batches_config(
|
||||
model=model,
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
)
|
||||
else:
|
||||
provider_config = None
|
||||
if provider_config is not None:
|
||||
response = base_llm_http_handler.create_batch(
|
||||
provider_config=provider_config,
|
||||
@@ -179,6 +221,7 @@ def create_batch(
|
||||
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
|
||||
else None,
|
||||
timeout=timeout,
|
||||
model=model,
|
||||
)
|
||||
return response
|
||||
api_base: Optional[str] = None
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
Handler for transforming /chat/completions api requests to litellm.responses requests
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Coroutine, TypedDict, Union
|
||||
from typing import TYPE_CHECKING, Any, Coroutine, Union
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
|
||||
|
||||
@@ -15,7 +15,7 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
|
||||
os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
|
||||
)
|
||||
DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(
|
||||
os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4)
|
||||
os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
|
||||
)
|
||||
DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
|
||||
SQS_SEND_MESSAGE_ACTION = "SendMessage"
|
||||
@@ -60,7 +60,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int(
|
||||
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128)
|
||||
)
|
||||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
|
||||
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512)
|
||||
os.getenv(
|
||||
"DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512
|
||||
)
|
||||
)
|
||||
|
||||
# Generic fallback for unknown models
|
||||
@@ -312,6 +314,7 @@ LITELLM_CHAT_PROVIDERS = [
|
||||
"lambda_ai",
|
||||
"vercel_ai_gateway",
|
||||
"wandb",
|
||||
"ovhcloud",
|
||||
]
|
||||
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
|
||||
@@ -985,7 +988,9 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
|
||||
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
|
||||
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
|
||||
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
|
||||
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000))
|
||||
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
|
||||
os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
|
||||
)
|
||||
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
|
||||
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
|
||||
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
|
||||
@@ -1055,6 +1060,7 @@ SENTRY_DENYLIST = [
|
||||
"FIREWORKS_API_KEY",
|
||||
"FIREWORKS_AI_API_KEY",
|
||||
"FIREWORKSAI_API_KEY",
|
||||
"OVHCLOUD_API_KEY",
|
||||
# Database and Connection Strings
|
||||
"database_url",
|
||||
"redis_url",
|
||||
|
||||
@@ -344,6 +344,11 @@ def cost_per_token( # noqa: PLR0915
|
||||
return perplexity_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "xai":
|
||||
return xai_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "dashscope":
|
||||
from litellm.llms.dashscope.cost_calculator import (
|
||||
cost_per_token as dashscope_cost_per_token,
|
||||
)
|
||||
return dashscope_cost_per_token(model=model, usage=usage_block)
|
||||
else:
|
||||
model_info = _cached_get_model_info_helper(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
Handler for transforming /chat/completions api requests to litellm.responses requests
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Optional, TypedDict, Union
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm import LiteLLMLoggingObj
|
||||
|
||||
@@ -17,22 +17,60 @@ from litellm.types.utils import ChatCompletionMessageToolCall
|
||||
########################################################
|
||||
def transform_mcp_tool_to_openai_tool(mcp_tool: MCPTool) -> ChatCompletionToolParam:
|
||||
"""Convert an MCP tool to an OpenAI tool."""
|
||||
normalized_parameters = _normalize_mcp_input_schema(mcp_tool.inputSchema)
|
||||
|
||||
return ChatCompletionToolParam(
|
||||
type="function",
|
||||
function=FunctionDefinition(
|
||||
name=mcp_tool.name,
|
||||
description=mcp_tool.description or "",
|
||||
parameters=mcp_tool.inputSchema,
|
||||
parameters=normalized_parameters,
|
||||
strict=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_mcp_input_schema(input_schema: dict) -> dict:
|
||||
"""
|
||||
Normalize MCP input schema to ensure it's valid for OpenAI function calling.
|
||||
|
||||
OpenAI requires that function parameters have:
|
||||
- type: 'object'
|
||||
- properties: dict (can be empty)
|
||||
- additionalProperties: false (recommended)
|
||||
"""
|
||||
if not input_schema:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": False
|
||||
}
|
||||
|
||||
# Make a copy to avoid modifying the original
|
||||
normalized_schema = dict(input_schema)
|
||||
|
||||
# Ensure type is 'object'
|
||||
if "type" not in normalized_schema:
|
||||
normalized_schema["type"] = "object"
|
||||
|
||||
# Ensure properties exists (can be empty)
|
||||
if "properties" not in normalized_schema:
|
||||
normalized_schema["properties"] = {}
|
||||
|
||||
# Add additionalProperties if not present (recommended by OpenAI)
|
||||
if "additionalProperties" not in normalized_schema:
|
||||
normalized_schema["additionalProperties"] = False
|
||||
|
||||
return normalized_schema
|
||||
|
||||
|
||||
def transform_mcp_tool_to_openai_responses_api_tool(mcp_tool: MCPTool) -> FunctionToolParam:
|
||||
"""Convert an MCP tool to an OpenAI Responses API tool."""
|
||||
normalized_parameters = _normalize_mcp_input_schema(mcp_tool.inputSchema)
|
||||
|
||||
return FunctionToolParam(
|
||||
name=mcp_tool.name,
|
||||
parameters=mcp_tool.inputSchema,
|
||||
parameters=normalized_parameters,
|
||||
strict=False,
|
||||
type="function",
|
||||
description=mcp_tool.description or "",
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from typing import Optional
|
||||
|
||||
from litellm.types.llms.openai import CreateFileRequest
|
||||
from litellm.types.utils import ExtractedFileData
|
||||
|
||||
|
||||
class FilesAPIUtils:
|
||||
"""
|
||||
Utils for files API interface on litellm
|
||||
"""
|
||||
@staticmethod
|
||||
def is_batch_jsonl_file(create_file_data: CreateFileRequest, extracted_file_data: ExtractedFileData) -> bool:
|
||||
"""
|
||||
Check if the file is a batch jsonl file
|
||||
"""
|
||||
return (
|
||||
create_file_data.get("purpose") == "batch"
|
||||
and FilesAPIUtils.valid_content_type(extracted_file_data.get("content_type"))
|
||||
and extracted_file_data.get("content") is not None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def valid_content_type(content_type: Optional[str]) -> bool:
|
||||
"""
|
||||
Check if the content type is valid
|
||||
"""
|
||||
return content_type in set(["application/jsonl", "application/octet-stream"])
|
||||
@@ -224,6 +224,9 @@ async def agenerate_content(
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["agenerate_content"] = True
|
||||
|
||||
# Handle generationConfig parameter from kwargs for backward compatibility
|
||||
if "generationConfig" in kwargs and config is None:
|
||||
config = kwargs.pop("generationConfig")
|
||||
# get custom llm provider so we can use this for mapping exceptions
|
||||
if custom_llm_provider is None:
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
@@ -288,6 +291,9 @@ def generate_content(
|
||||
try:
|
||||
_is_async = kwargs.pop("agenerate_content", False) is True
|
||||
|
||||
# Handle generationConfig parameter from kwargs for backward compatibility
|
||||
if "generationConfig" in kwargs and config is None:
|
||||
config = kwargs.pop("generationConfig")
|
||||
# Check for mock response first
|
||||
litellm_params = GenericLiteLLMParams(**kwargs)
|
||||
if litellm_params.mock_response and isinstance(
|
||||
@@ -374,6 +380,9 @@ async def agenerate_content_stream(
|
||||
try:
|
||||
kwargs["agenerate_content_stream"] = True
|
||||
|
||||
# Handle generationConfig parameter from kwargs for backward compatibility
|
||||
if "generationConfig" in kwargs and config is None:
|
||||
config = kwargs.pop("generationConfig")
|
||||
# get custom llm provider so we can use this for mapping exceptions
|
||||
if custom_llm_provider is None:
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
@@ -461,6 +470,9 @@ def generate_content_stream(
|
||||
# Remove any async-related flags since this is the sync function
|
||||
_is_async = kwargs.pop("agenerate_content_stream", False)
|
||||
|
||||
# Handle generationConfig parameter from kwargs for backward compatibility
|
||||
if "generationConfig" in kwargs and config is None:
|
||||
config = kwargs.pop("generationConfig")
|
||||
# Setup the call
|
||||
setup_result = GenerateContentHelper.setup_generate_content_call(
|
||||
model=model,
|
||||
|
||||
@@ -31,7 +31,7 @@ class SoftBudgetAlert(BaseBudgetAlertType):
|
||||
return "Soft Budget Crossed: "
|
||||
|
||||
def get_id(self, user_info: CallInfo) -> str:
|
||||
return "default_id"
|
||||
return user_info.token or "default_id"
|
||||
|
||||
|
||||
class UserBudgetAlert(BaseBudgetAlertType):
|
||||
|
||||
@@ -64,7 +64,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
self.flush_lock = asyncio.Lock()
|
||||
self.log_queue: List[LLMObsPayload] = []
|
||||
|
||||
|
||||
#########################################################
|
||||
# Handle datadog_llm_observability_params set as litellm.datadog_llm_observability_params
|
||||
#########################################################
|
||||
@@ -83,22 +83,25 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
"""
|
||||
dict_datadog_llm_obs_params: Dict = {}
|
||||
if litellm.datadog_llm_observability_params is not None:
|
||||
if isinstance(litellm.datadog_llm_observability_params, DatadogLLMObsInitParams):
|
||||
dict_datadog_llm_obs_params = litellm.datadog_llm_observability_params.model_dump()
|
||||
if isinstance(
|
||||
litellm.datadog_llm_observability_params, DatadogLLMObsInitParams
|
||||
):
|
||||
dict_datadog_llm_obs_params = (
|
||||
litellm.datadog_llm_observability_params.model_dump()
|
||||
)
|
||||
elif isinstance(litellm.datadog_llm_observability_params, Dict):
|
||||
# only allow params that are of DatadogLLMObsInitParams
|
||||
dict_datadog_llm_obs_params = DatadogLLMObsInitParams(**litellm.datadog_llm_observability_params).model_dump()
|
||||
dict_datadog_llm_obs_params = DatadogLLMObsInitParams(
|
||||
**litellm.datadog_llm_observability_params
|
||||
).model_dump()
|
||||
return dict_datadog_llm_obs_params
|
||||
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
verbose_logger.debug(
|
||||
f"DataDogLLMObs: Logging success event for model {kwargs.get('model', 'unknown')}"
|
||||
)
|
||||
payload = self.create_llm_obs_payload(
|
||||
kwargs, start_time, end_time
|
||||
)
|
||||
payload = self.create_llm_obs_payload(kwargs, start_time, end_time)
|
||||
verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
|
||||
self.log_queue.append(payload)
|
||||
|
||||
@@ -108,15 +111,13 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
verbose_logger.exception(
|
||||
f"DataDogLLMObs: Error logging success event - {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
verbose_logger.debug(
|
||||
f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}"
|
||||
)
|
||||
payload = self.create_llm_obs_payload(
|
||||
kwargs, start_time, end_time
|
||||
)
|
||||
payload = self.create_llm_obs_payload(kwargs, start_time, end_time)
|
||||
verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
|
||||
self.log_queue.append(payload)
|
||||
|
||||
@@ -184,7 +185,6 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
|
||||
messages = standard_logging_payload["messages"]
|
||||
messages = self._ensure_string_content(messages=messages)
|
||||
response_obj = standard_logging_payload.get("response")
|
||||
|
||||
metadata = kwargs.get("litellm_params", {}).get("metadata", {})
|
||||
|
||||
@@ -193,10 +193,12 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
messages
|
||||
)
|
||||
)
|
||||
output_meta = OutputMeta(messages=self._get_response_messages(
|
||||
response_obj=response_obj,
|
||||
call_type=standard_logging_payload.get("call_type")
|
||||
))
|
||||
output_meta = OutputMeta(
|
||||
messages=self._get_response_messages(
|
||||
standard_logging_payload=standard_logging_payload,
|
||||
call_type=standard_logging_payload.get("call_type"),
|
||||
)
|
||||
)
|
||||
|
||||
error_info = self._assemble_error_info(standard_logging_payload)
|
||||
|
||||
@@ -214,7 +216,9 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
output_tokens=float(standard_logging_payload.get("completion_tokens", 0)),
|
||||
total_tokens=float(standard_logging_payload.get("total_tokens", 0)),
|
||||
total_cost=float(standard_logging_payload.get("response_cost", 0)),
|
||||
time_to_first_token=self._get_time_to_first_token_seconds(standard_logging_payload),
|
||||
time_to_first_token=self._get_time_to_first_token_seconds(
|
||||
standard_logging_payload
|
||||
),
|
||||
)
|
||||
|
||||
payload: LLMObsPayload = LLMObsPayload(
|
||||
@@ -251,27 +255,35 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
|
||||
|
||||
def _assemble_error_info(
|
||||
self, standard_logging_payload: StandardLoggingPayload
|
||||
) -> Optional[DDLLMObsError]:
|
||||
"""
|
||||
Assemble error information for failure cases according to DD LLM Obs API spec
|
||||
"""
|
||||
# Handle error information for failure cases according to DD LLM Obs API spec
|
||||
error_info: Optional[DDLLMObsError] = None
|
||||
|
||||
|
||||
if standard_logging_payload.get("status") == "failure":
|
||||
# Try to get structured error information first
|
||||
error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information")
|
||||
|
||||
error_information: Optional[
|
||||
StandardLoggingPayloadErrorInformation
|
||||
] = standard_logging_payload.get("error_information")
|
||||
|
||||
if error_information:
|
||||
error_info = DDLLMObsError(
|
||||
message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error",
|
||||
message=error_information.get("error_message")
|
||||
or standard_logging_payload.get("error_str")
|
||||
or "Unknown error",
|
||||
type=error_information.get("error_class"),
|
||||
stack=error_information.get("traceback")
|
||||
stack=error_information.get("traceback"),
|
||||
)
|
||||
return error_info
|
||||
|
||||
def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
|
||||
def _get_time_to_first_token_seconds(
|
||||
self, standard_logging_payload: StandardLoggingPayload
|
||||
) -> float:
|
||||
"""
|
||||
Get the time to first token in seconds
|
||||
|
||||
@@ -280,7 +292,9 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
For non streaming calls, CompletionStartTime is time we get the response back
|
||||
"""
|
||||
start_time: Optional[float] = standard_logging_payload.get("startTime")
|
||||
completion_start_time: Optional[float] = standard_logging_payload.get("completionStartTime")
|
||||
completion_start_time: Optional[float] = standard_logging_payload.get(
|
||||
"completionStartTime"
|
||||
)
|
||||
end_time: Optional[float] = standard_logging_payload.get("endTime")
|
||||
|
||||
if completion_start_time is not None and start_time is not None:
|
||||
@@ -290,19 +304,43 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
else:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _get_response_messages(
|
||||
self, response_obj: Any, call_type: Optional[str]
|
||||
self, standard_logging_payload: StandardLoggingPayload, call_type: Optional[str]
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Get the messages from the response object
|
||||
|
||||
for now this handles logging /chat/completions responses
|
||||
"""
|
||||
|
||||
response_obj = standard_logging_payload.get("response")
|
||||
if response_obj is None:
|
||||
return []
|
||||
|
||||
if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]:
|
||||
|
||||
# edge case: handle response_obj is a string representation of a dict
|
||||
if isinstance(response_obj, str):
|
||||
try:
|
||||
import ast
|
||||
|
||||
response_obj = ast.literal_eval(response_obj)
|
||||
except (ValueError, SyntaxError):
|
||||
try:
|
||||
# fallback to json parsing
|
||||
response_obj = json.loads(str(response_obj))
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
if call_type in [
|
||||
CallTypes.completion.value,
|
||||
CallTypes.acompletion.value,
|
||||
CallTypes.text_completion.value,
|
||||
CallTypes.atext_completion.value,
|
||||
CallTypes.generate_content.value,
|
||||
CallTypes.agenerate_content.value,
|
||||
CallTypes.generate_content_stream.value,
|
||||
CallTypes.agenerate_content_stream.value,
|
||||
CallTypes.anthropic_messages.value,
|
||||
]:
|
||||
try:
|
||||
# Safely extract message from response_obj, handle failure cases
|
||||
if isinstance(response_obj, dict) and "choices" in response_obj:
|
||||
@@ -315,102 +353,104 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
return []
|
||||
return []
|
||||
|
||||
def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
|
||||
def _get_datadog_span_kind(
|
||||
self, call_type: Optional[str]
|
||||
) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
|
||||
"""
|
||||
Map liteLLM call_type to appropriate DataDog LLM Observability span kind.
|
||||
|
||||
|
||||
Available DataDog span kinds: "llm", "tool", "task", "embedding", "retrieval"
|
||||
"""
|
||||
if call_type is None:
|
||||
return "llm"
|
||||
|
||||
|
||||
# Embedding operations
|
||||
if call_type in [CallTypes.embedding.value, CallTypes.aembedding.value]:
|
||||
return "embedding"
|
||||
|
||||
# LLM completion operations
|
||||
|
||||
# LLM completion operations
|
||||
if call_type in [
|
||||
CallTypes.completion.value,
|
||||
CallTypes.completion.value,
|
||||
CallTypes.acompletion.value,
|
||||
CallTypes.text_completion.value,
|
||||
CallTypes.text_completion.value,
|
||||
CallTypes.atext_completion.value,
|
||||
CallTypes.generate_content.value,
|
||||
CallTypes.generate_content.value,
|
||||
CallTypes.agenerate_content.value,
|
||||
CallTypes.generate_content_stream.value,
|
||||
CallTypes.generate_content_stream.value,
|
||||
CallTypes.agenerate_content_stream.value,
|
||||
CallTypes.anthropic_messages.value
|
||||
CallTypes.anthropic_messages.value,
|
||||
]:
|
||||
return "llm"
|
||||
|
||||
|
||||
# Tool operations
|
||||
if call_type in [CallTypes.call_mcp_tool.value]:
|
||||
return "tool"
|
||||
|
||||
|
||||
# Retrieval operations
|
||||
if call_type in [
|
||||
CallTypes.get_assistants.value,
|
||||
CallTypes.get_assistants.value,
|
||||
CallTypes.aget_assistants.value,
|
||||
CallTypes.get_thread.value,
|
||||
CallTypes.get_thread.value,
|
||||
CallTypes.aget_thread.value,
|
||||
CallTypes.get_messages.value,
|
||||
CallTypes.get_messages.value,
|
||||
CallTypes.aget_messages.value,
|
||||
CallTypes.afile_retrieve.value,
|
||||
CallTypes.afile_retrieve.value,
|
||||
CallTypes.file_retrieve.value,
|
||||
CallTypes.afile_list.value,
|
||||
CallTypes.afile_list.value,
|
||||
CallTypes.file_list.value,
|
||||
CallTypes.afile_content.value,
|
||||
CallTypes.afile_content.value,
|
||||
CallTypes.file_content.value,
|
||||
CallTypes.retrieve_batch.value,
|
||||
CallTypes.retrieve_batch.value,
|
||||
CallTypes.aretrieve_batch.value,
|
||||
CallTypes.retrieve_fine_tuning_job.value,
|
||||
CallTypes.retrieve_fine_tuning_job.value,
|
||||
CallTypes.aretrieve_fine_tuning_job.value,
|
||||
CallTypes.responses.value,
|
||||
CallTypes.responses.value,
|
||||
CallTypes.aresponses.value,
|
||||
CallTypes.alist_input_items.value
|
||||
CallTypes.alist_input_items.value,
|
||||
]:
|
||||
return "retrieval"
|
||||
|
||||
|
||||
# Task operations (batch, fine-tuning, file operations, etc.)
|
||||
if call_type in [
|
||||
CallTypes.create_batch.value,
|
||||
CallTypes.create_batch.value,
|
||||
CallTypes.acreate_batch.value,
|
||||
CallTypes.create_fine_tuning_job.value,
|
||||
CallTypes.create_fine_tuning_job.value,
|
||||
CallTypes.acreate_fine_tuning_job.value,
|
||||
CallTypes.cancel_fine_tuning_job.value,
|
||||
CallTypes.cancel_fine_tuning_job.value,
|
||||
CallTypes.acancel_fine_tuning_job.value,
|
||||
CallTypes.list_fine_tuning_jobs.value,
|
||||
CallTypes.list_fine_tuning_jobs.value,
|
||||
CallTypes.alist_fine_tuning_jobs.value,
|
||||
CallTypes.create_assistants.value,
|
||||
CallTypes.create_assistants.value,
|
||||
CallTypes.acreate_assistants.value,
|
||||
CallTypes.delete_assistant.value,
|
||||
CallTypes.delete_assistant.value,
|
||||
CallTypes.adelete_assistant.value,
|
||||
CallTypes.create_thread.value,
|
||||
CallTypes.create_thread.value,
|
||||
CallTypes.acreate_thread.value,
|
||||
CallTypes.add_message.value,
|
||||
CallTypes.add_message.value,
|
||||
CallTypes.a_add_message.value,
|
||||
CallTypes.run_thread.value,
|
||||
CallTypes.run_thread.value,
|
||||
CallTypes.arun_thread.value,
|
||||
CallTypes.run_thread_stream.value,
|
||||
CallTypes.run_thread_stream.value,
|
||||
CallTypes.arun_thread_stream.value,
|
||||
CallTypes.file_delete.value,
|
||||
CallTypes.file_delete.value,
|
||||
CallTypes.afile_delete.value,
|
||||
CallTypes.create_file.value,
|
||||
CallTypes.create_file.value,
|
||||
CallTypes.acreate_file.value,
|
||||
CallTypes.image_generation.value,
|
||||
CallTypes.image_generation.value,
|
||||
CallTypes.aimage_generation.value,
|
||||
CallTypes.image_edit.value,
|
||||
CallTypes.image_edit.value,
|
||||
CallTypes.aimage_edit.value,
|
||||
CallTypes.moderation.value,
|
||||
CallTypes.moderation.value,
|
||||
CallTypes.amoderation.value,
|
||||
CallTypes.transcription.value,
|
||||
CallTypes.transcription.value,
|
||||
CallTypes.atranscription.value,
|
||||
CallTypes.speech.value,
|
||||
CallTypes.speech.value,
|
||||
CallTypes.aspeech.value,
|
||||
CallTypes.rerank.value,
|
||||
CallTypes.arerank.value
|
||||
CallTypes.rerank.value,
|
||||
CallTypes.arerank.value,
|
||||
]:
|
||||
return "task"
|
||||
|
||||
|
||||
# Default fallback for unknown or passthrough operations
|
||||
return "llm"
|
||||
|
||||
@@ -443,7 +483,9 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
"cache_hit": standard_logging_payload.get("cache_hit", "unknown"),
|
||||
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
|
||||
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
|
||||
"guardrail_information": standard_logging_payload.get("guardrail_information", None),
|
||||
"guardrail_information": standard_logging_payload.get(
|
||||
"guardrail_information", None
|
||||
),
|
||||
}
|
||||
|
||||
#########################################################
|
||||
@@ -452,22 +494,32 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
latency_metrics = self._get_latency_metrics(standard_logging_payload)
|
||||
_metadata.update({"latency_metrics": dict(latency_metrics)})
|
||||
|
||||
## extract tool calls and add to metadata
|
||||
tool_call_metadata = self._extract_tool_call_metadata(standard_logging_payload)
|
||||
_metadata.update(tool_call_metadata)
|
||||
|
||||
_standard_logging_metadata: dict = (
|
||||
dict(standard_logging_payload.get("metadata", {})) or {}
|
||||
)
|
||||
_metadata.update(_standard_logging_metadata)
|
||||
return _metadata
|
||||
|
||||
def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics:
|
||||
def _get_latency_metrics(
|
||||
self, standard_logging_payload: StandardLoggingPayload
|
||||
) -> DDLLMObsLatencyMetrics:
|
||||
"""
|
||||
Get the latency metrics from the standard logging payload
|
||||
"""
|
||||
latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics()
|
||||
# Add latency metrics to metadata
|
||||
# Time to first token (convert from seconds to milliseconds for consistency)
|
||||
time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload)
|
||||
time_to_first_token_seconds = self._get_time_to_first_token_seconds(
|
||||
standard_logging_payload
|
||||
)
|
||||
if time_to_first_token_seconds > 0:
|
||||
latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000
|
||||
latency_metrics["time_to_first_token_ms"] = (
|
||||
time_to_first_token_seconds * 1000
|
||||
)
|
||||
|
||||
# LiteLLM overhead time
|
||||
hidden_params = standard_logging_payload.get("hidden_params", {})
|
||||
@@ -476,11 +528,143 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
|
||||
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
|
||||
|
||||
# Guardrail overhead latency
|
||||
guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information")
|
||||
guardrail_info: Optional[
|
||||
StandardLoggingGuardrailInformation
|
||||
] = standard_logging_payload.get("guardrail_information")
|
||||
if guardrail_info is not None:
|
||||
_guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration")
|
||||
_guardrail_duration_seconds: Optional[float] = guardrail_info.get(
|
||||
"duration"
|
||||
)
|
||||
if _guardrail_duration_seconds is not None:
|
||||
# Convert from seconds to milliseconds for consistency
|
||||
latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000
|
||||
|
||||
return latency_metrics
|
||||
latency_metrics["guardrail_overhead_time_ms"] = (
|
||||
_guardrail_duration_seconds * 1000
|
||||
)
|
||||
|
||||
return latency_metrics
|
||||
|
||||
def _process_input_messages_preserving_tool_calls(
|
||||
self, messages: List[Any]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Process input messages while preserving tool_calls and tool message types.
|
||||
|
||||
This bypasses the lossy string conversion when tool calls are present,
|
||||
allowing complex nested tool_calls objects to be preserved for Datadog.
|
||||
"""
|
||||
processed = []
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict):
|
||||
# Preserve messages with tool_calls or tool role as-is
|
||||
if "tool_calls" in msg or msg.get("role") == "tool":
|
||||
processed.append(msg)
|
||||
else:
|
||||
# For regular messages, still apply string conversion
|
||||
converted = (
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion(
|
||||
[msg]
|
||||
)
|
||||
)
|
||||
processed.extend(converted)
|
||||
else:
|
||||
# For non-dict messages, apply string conversion
|
||||
converted = (
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion(
|
||||
[msg]
|
||||
)
|
||||
)
|
||||
processed.extend(converted)
|
||||
return processed
|
||||
|
||||
@staticmethod
|
||||
def _tool_calls_kv_pair(tool_calls: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract tool call information into key-value pairs for Datadog metadata.
|
||||
|
||||
Similar to OpenTelemetry's implementation but adapted for Datadog's format.
|
||||
"""
|
||||
kv_pairs: Dict[str, Any] = {}
|
||||
for idx, tool_call in enumerate(tool_calls):
|
||||
try:
|
||||
# Extract tool call ID
|
||||
tool_id = tool_call.get("id")
|
||||
if tool_id:
|
||||
kv_pairs[f"tool_calls.{idx}.id"] = tool_id
|
||||
|
||||
# Extract tool call type
|
||||
tool_type = tool_call.get("type")
|
||||
if tool_type:
|
||||
kv_pairs[f"tool_calls.{idx}.type"] = tool_type
|
||||
|
||||
# Extract function information
|
||||
function = tool_call.get("function")
|
||||
if function:
|
||||
function_name = function.get("name")
|
||||
if function_name:
|
||||
kv_pairs[f"tool_calls.{idx}.function.name"] = function_name
|
||||
|
||||
function_arguments = function.get("arguments")
|
||||
if function_arguments:
|
||||
# Store arguments as JSON string for Datadog
|
||||
if isinstance(function_arguments, str):
|
||||
kv_pairs[
|
||||
f"tool_calls.{idx}.function.arguments"
|
||||
] = function_arguments
|
||||
else:
|
||||
import json
|
||||
|
||||
kv_pairs[
|
||||
f"tool_calls.{idx}.function.arguments"
|
||||
] = json.dumps(function_arguments)
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
verbose_logger.debug(
|
||||
f"DataDogLLMObs: Error processing tool call {idx}: {str(e)}"
|
||||
)
|
||||
continue
|
||||
|
||||
return kv_pairs
|
||||
|
||||
def _extract_tool_call_metadata(
|
||||
self, standard_logging_payload: StandardLoggingPayload
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract tool call information from both input messages and response for Datadog metadata.
|
||||
"""
|
||||
tool_call_metadata: Dict[str, Any] = {}
|
||||
|
||||
try:
|
||||
# Extract tool calls from input messages
|
||||
messages = standard_logging_payload.get("messages", [])
|
||||
if messages and isinstance(messages, list):
|
||||
for message in messages:
|
||||
if isinstance(message, dict) and "tool_calls" in message:
|
||||
tool_calls = message.get("tool_calls")
|
||||
if tool_calls:
|
||||
input_tool_calls_kv = self._tool_calls_kv_pair(tool_calls)
|
||||
# Prefix with "input_" to distinguish from response tool calls
|
||||
for key, value in input_tool_calls_kv.items():
|
||||
tool_call_metadata[f"input_{key}"] = value
|
||||
|
||||
# Extract tool calls from response
|
||||
response_obj = standard_logging_payload.get("response")
|
||||
if response_obj and isinstance(response_obj, dict):
|
||||
choices = response_obj.get("choices", [])
|
||||
for choice in choices:
|
||||
if isinstance(choice, dict):
|
||||
message = choice.get("message")
|
||||
if message and isinstance(message, dict):
|
||||
tool_calls = message.get("tool_calls")
|
||||
if tool_calls:
|
||||
response_tool_calls_kv = self._tool_calls_kv_pair(
|
||||
tool_calls
|
||||
)
|
||||
# Prefix with "output_" to distinguish from input tool calls
|
||||
for key, value in response_tool_calls_kv.items():
|
||||
tool_call_metadata[f"output_{key}"] = value
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
f"DataDogLLMObs: Error extracting tool call metadata: {str(e)}"
|
||||
)
|
||||
|
||||
return tool_call_metadata
|
||||
|
||||
@@ -4,9 +4,10 @@ Humanloop integration
|
||||
https://humanloop.com/
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
import httpx
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.caching import DualCache
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
@@ -62,6 +62,7 @@ def get_litellm_params(
|
||||
use_litellm_proxy: Optional[bool] = None,
|
||||
api_version: Optional[str] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
litellm_request_debug: Optional[bool] = None,
|
||||
**kwargs,
|
||||
) -> dict:
|
||||
litellm_params = {
|
||||
@@ -118,5 +119,6 @@ def get_litellm_params(
|
||||
"vertex_credentials": kwargs.get("vertex_credentials"),
|
||||
"vertex_project": kwargs.get("vertex_project"),
|
||||
"use_litellm_proxy": use_litellm_proxy,
|
||||
"litellm_request_debug": litellm_request_debug,
|
||||
}
|
||||
return litellm_params
|
||||
|
||||
@@ -375,6 +375,8 @@ def get_llm_provider( # noqa: PLR0915
|
||||
custom_llm_provider = "cometapi"
|
||||
elif model.startswith("oci/"):
|
||||
custom_llm_provider = "oci"
|
||||
elif model.startswith("ovhcloud/"):
|
||||
custom_llm_provider = "ovhcloud"
|
||||
if not custom_llm_provider:
|
||||
if litellm.suppress_debug_info is False:
|
||||
print() # noqa
|
||||
|
||||
@@ -245,6 +245,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
global supabaseClient, promptLayerLogger, weightsBiasesLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger, logfireLogger, prometheusLogger, slack_app
|
||||
custom_pricing: bool = False
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -470,6 +471,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
**self.litellm_params,
|
||||
**scrub_sensitive_keys_in_metadata(litellm_params),
|
||||
}
|
||||
self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
|
||||
self.logger_fn = litellm_params.get("logger_fn", None)
|
||||
verbose_logger.debug(f"self.optional_params: {self.optional_params}")
|
||||
|
||||
@@ -907,13 +909,19 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
|
||||
Prints the RAW curl command sent from LiteLLM
|
||||
"""
|
||||
if _is_debugging_on():
|
||||
if _is_debugging_on() or self.litellm_request_debug:
|
||||
if json_logs:
|
||||
masked_headers = self._get_masked_headers(headers)
|
||||
verbose_logger.debug(
|
||||
"POST Request Sent from LiteLLM",
|
||||
extra={"api_base": {api_base}, **masked_headers},
|
||||
)
|
||||
if self.litellm_request_debug:
|
||||
verbose_logger.warning( # .warning ensures this shows up in all environments
|
||||
"POST Request Sent from LiteLLM",
|
||||
extra={"api_base": {api_base}, **masked_headers},
|
||||
)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"POST Request Sent from LiteLLM",
|
||||
extra={"api_base": {api_base}, **masked_headers},
|
||||
)
|
||||
else:
|
||||
headers = additional_args.get("headers", {})
|
||||
if headers is None:
|
||||
@@ -926,7 +934,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
additional_args=additional_args,
|
||||
data=data,
|
||||
)
|
||||
verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
|
||||
if self.litellm_request_debug:
|
||||
verbose_logger.warning(
|
||||
f"\033[92m{curl_command}\033[0m\n"
|
||||
) # .warning ensures this shows up in all environments
|
||||
else:
|
||||
verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
|
||||
|
||||
def _get_request_body(self, data: dict) -> str:
|
||||
return str(data)
|
||||
@@ -983,8 +996,14 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
self.model_call_details["additional_args"] = additional_args
|
||||
self.model_call_details["log_event_type"] = "post_api_call"
|
||||
|
||||
if self.litellm_request_debug:
|
||||
attr = "warning"
|
||||
else:
|
||||
attr = "debug"
|
||||
|
||||
if json_logs:
|
||||
verbose_logger.debug(
|
||||
callattr = getattr(verbose_logger, attr)
|
||||
callattr(
|
||||
"RAW RESPONSE:\n{}\n\n".format(
|
||||
self.model_call_details.get(
|
||||
"original_response", self.model_call_details
|
||||
@@ -992,7 +1011,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
),
|
||||
)
|
||||
else:
|
||||
print_verbose(
|
||||
callattr = getattr(verbose_logger, attr)
|
||||
callattr(
|
||||
"RAW RESPONSE:\n{}\n\n".format(
|
||||
self.model_call_details.get(
|
||||
"original_response", self.model_call_details
|
||||
@@ -1714,12 +1734,16 @@ class Logging(LiteLLMLoggingBaseClass):
|
||||
response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
litellm_call_id=current_call_id
|
||||
if (
|
||||
current_call_id := litellm_params.get("litellm_call_id")
|
||||
)
|
||||
is not None
|
||||
else str(uuid.uuid4()),
|
||||
litellm_call_id=(
|
||||
current_call_id
|
||||
if (
|
||||
current_call_id := litellm_params.get(
|
||||
"litellm_call_id"
|
||||
)
|
||||
)
|
||||
is not None
|
||||
else str(uuid.uuid4())
|
||||
),
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
if callback == "wandb" and weightsBiasesLogger is not None:
|
||||
@@ -3367,6 +3391,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
||||
return galileo_logger # type: ignore
|
||||
elif logging_integration == "cloudzero":
|
||||
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, CloudZeroLogger):
|
||||
return callback # type: ignore
|
||||
@@ -3594,6 +3619,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
|
||||
return callback
|
||||
elif logging_integration == "cloudzero":
|
||||
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, CloudZeroLogger):
|
||||
return callback
|
||||
@@ -4504,7 +4530,7 @@ def get_standard_logging_object_payload(
|
||||
|
||||
def emit_standard_logging_payload(payload: StandardLoggingPayload):
|
||||
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
|
||||
print(json.dumps(payload, indent=4)) # noqa
|
||||
print(json.dumps(payload, indent=4)) # noqa
|
||||
|
||||
|
||||
def get_standard_logging_metadata(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
@@ -9,6 +8,9 @@ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_extract_reasoning_content,
|
||||
)
|
||||
from litellm.types.llms.databricks import DatabricksTool
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionThinkingBlock,
|
||||
@@ -274,49 +276,6 @@ def _handle_invalid_parallel_tool_calls(
|
||||
return tool_calls
|
||||
|
||||
|
||||
def _parse_content_for_reasoning(
|
||||
message_text: Optional[str],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Parse the content for reasoning
|
||||
|
||||
Returns:
|
||||
- reasoning_content: The content of the reasoning
|
||||
- content: The content of the message
|
||||
"""
|
||||
if not message_text:
|
||||
return None, message_text
|
||||
|
||||
reasoning_match = re.match(
|
||||
r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
|
||||
)
|
||||
|
||||
if reasoning_match:
|
||||
return reasoning_match.group(1), reasoning_match.group(2)
|
||||
|
||||
return None, message_text
|
||||
|
||||
|
||||
def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Extract reasoning content and main content from a message.
|
||||
|
||||
Args:
|
||||
message (dict): The message dictionary that may contain reasoning_content
|
||||
|
||||
Returns:
|
||||
tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
|
||||
"""
|
||||
message_content = message.get("content")
|
||||
if "reasoning_content" in message:
|
||||
return message["reasoning_content"], message["content"]
|
||||
elif "reasoning" in message:
|
||||
return message["reasoning"], message["content"]
|
||||
elif isinstance(message_content, str):
|
||||
return _parse_content_for_reasoning(message_content)
|
||||
return None, message_content
|
||||
|
||||
|
||||
class LiteLLMResponseObjectHandler:
|
||||
@staticmethod
|
||||
def convert_to_image_response(
|
||||
|
||||
@@ -1,10 +1,23 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
import contextvars
|
||||
from typing import Coroutine, Optional
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
||||
class LoggingTask(TypedDict):
|
||||
"""
|
||||
A logging task with its associated context to ensure logging is executed in
|
||||
the original task's context.
|
||||
"""
|
||||
|
||||
coroutine: Coroutine
|
||||
context: contextvars.Context
|
||||
|
||||
|
||||
class LoggingWorker:
|
||||
"""
|
||||
A simple, async logging worker that processes log coroutines in the background.
|
||||
@@ -13,77 +26,84 @@ class LoggingWorker:
|
||||
This leads to a +200 RPS performance improvement when using LiteLLM Python SDK or Proxy Server.
|
||||
- Use this to queue coroutine tasks that are not critical to the main flow of the application. e.g Success/Error callbacks, logging, etc.
|
||||
"""
|
||||
|
||||
LOGGING_WORKER_MAX_QUEUE_SIZE = 50_000
|
||||
LOGGING_WORKER_MAX_TIME_PER_COROUTINE = 20.0
|
||||
|
||||
MAX_ITERATIONS_TO_CLEAR_QUEUE = 200
|
||||
MAX_TIME_TO_CLEAR_QUEUE = 5.0
|
||||
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
|
||||
self,
|
||||
timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
|
||||
max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
|
||||
):
|
||||
self.timeout = timeout
|
||||
self.max_queue_size = max_queue_size
|
||||
self._queue: Optional[asyncio.Queue] = None
|
||||
self._queue: Optional[asyncio.Queue[LoggingTask]] = None
|
||||
self._worker_task: Optional[asyncio.Task] = None
|
||||
|
||||
|
||||
def _ensure_queue(self) -> None:
|
||||
"""Initialize the queue if it doesn't exist."""
|
||||
if self._queue is None:
|
||||
self._queue = asyncio.Queue(maxsize=self.max_queue_size)
|
||||
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the logging worker. Idempotent - safe to call multiple times."""
|
||||
self._ensure_queue()
|
||||
if self._worker_task is None or self._worker_task.done():
|
||||
self._worker_task = asyncio.create_task(self._worker_loop())
|
||||
|
||||
|
||||
async def _worker_loop(self) -> None:
|
||||
"""Main worker loop that processes log coroutines sequentially."""
|
||||
try:
|
||||
if self._queue is None:
|
||||
return
|
||||
|
||||
|
||||
while True:
|
||||
# Process one coroutine at a time to keep event loop load predictable
|
||||
coroutine = await self._queue.get()
|
||||
task = await self._queue.get()
|
||||
try:
|
||||
await asyncio.wait_for(coroutine, timeout=self.timeout)
|
||||
# Run the coroutine in its original context
|
||||
await asyncio.wait_for(
|
||||
task["context"].run(asyncio.create_task, task["coroutine"]),
|
||||
timeout=self.timeout,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"LoggingWorker error: {e}")
|
||||
pass
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
|
||||
except asyncio.CancelledError:
|
||||
verbose_logger.debug("LoggingWorker cancelled during shutdown")
|
||||
# Attempt to clear remaining items to prevent "never awaited" warnings
|
||||
await self.clear_queue()
|
||||
|
||||
|
||||
def enqueue(self, coroutine: Coroutine) -> None:
|
||||
"""
|
||||
Add a coroutine to the logging queue.
|
||||
Add a coroutine to the logging queue.
|
||||
Hot path: never blocks, drops logs if queue is full.
|
||||
"""
|
||||
if self._queue is None:
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
self._queue.put_nowait(coroutine)
|
||||
# Capture the current context when enqueueing
|
||||
task = LoggingTask(coroutine=coroutine, context=contextvars.copy_context())
|
||||
self._queue.put_nowait(task)
|
||||
except asyncio.QueueFull as e:
|
||||
verbose_logger.exception(f"LoggingWorker queue is full: {e}")
|
||||
# Drop logs on overload to protect request throughput
|
||||
pass
|
||||
|
||||
|
||||
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
|
||||
"""
|
||||
Ensure the logging worker is initialized and enqueue the coroutine.
|
||||
"""
|
||||
self.start()
|
||||
self.enqueue(async_coroutine)
|
||||
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the logging worker and clean up resources."""
|
||||
if self._worker_task:
|
||||
@@ -91,34 +111,42 @@ class LoggingWorker:
|
||||
with contextlib.suppress(Exception):
|
||||
await self._worker_task
|
||||
self._worker_task = None
|
||||
|
||||
|
||||
async def flush(self) -> None:
|
||||
"""Flush the logging queue."""
|
||||
if self._queue is None:
|
||||
return
|
||||
while not self._queue.empty():
|
||||
await self._queue.join()
|
||||
|
||||
|
||||
async def clear_queue(self):
|
||||
"""
|
||||
Clear the queue with a maximum time limit.
|
||||
"""
|
||||
if self._queue is None:
|
||||
return
|
||||
|
||||
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
|
||||
|
||||
for _ in range(self.MAX_ITERATIONS_TO_CLEAR_QUEUE):
|
||||
# Check if we've exceeded the maximum time
|
||||
if asyncio.get_event_loop().time() - start_time >= self.MAX_TIME_TO_CLEAR_QUEUE:
|
||||
verbose_logger.warning(f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early")
|
||||
if (
|
||||
asyncio.get_event_loop().time() - start_time
|
||||
>= self.MAX_TIME_TO_CLEAR_QUEUE
|
||||
):
|
||||
verbose_logger.warning(
|
||||
f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early"
|
||||
)
|
||||
break
|
||||
|
||||
|
||||
try:
|
||||
coroutine = self._queue.get_nowait()
|
||||
task = self._queue.get_nowait()
|
||||
# Await the coroutine to properly execute and avoid "never awaited" warnings
|
||||
try:
|
||||
await asyncio.wait_for(coroutine, timeout=self.timeout)
|
||||
await asyncio.wait_for(
|
||||
task["context"].run(asyncio.create_task, task["coroutine"]),
|
||||
timeout=self.timeout,
|
||||
)
|
||||
except Exception:
|
||||
# Suppress errors during cleanup
|
||||
pass
|
||||
@@ -129,4 +157,3 @@ class LoggingWorker:
|
||||
|
||||
# Global instance for backward compatibility
|
||||
GLOBAL_LOGGING_WORKER = LoggingWorker()
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from typing import (
|
||||
Literal,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
@@ -869,3 +870,63 @@ def convert_prefix_message_to_non_prefix_messages(
|
||||
else:
|
||||
new_messages.append(message)
|
||||
return new_messages
|
||||
|
||||
|
||||
def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Extract reasoning content and main content from a message.
|
||||
|
||||
Args:
|
||||
message (dict): The message dictionary that may contain reasoning_content
|
||||
|
||||
Returns:
|
||||
tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
|
||||
"""
|
||||
message_content = message.get("content")
|
||||
if "reasoning_content" in message:
|
||||
return message["reasoning_content"], message["content"]
|
||||
elif "reasoning" in message:
|
||||
return message["reasoning"], message["content"]
|
||||
elif isinstance(message_content, str):
|
||||
return _parse_content_for_reasoning(message_content)
|
||||
return None, message_content
|
||||
|
||||
|
||||
def _parse_content_for_reasoning(
|
||||
message_text: Optional[str],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Parse the content for reasoning
|
||||
|
||||
Returns:
|
||||
- reasoning_content: The content of the reasoning
|
||||
- content: The content of the message
|
||||
"""
|
||||
if not message_text:
|
||||
return None, message_text
|
||||
|
||||
reasoning_match = re.match(
|
||||
r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
|
||||
)
|
||||
|
||||
if reasoning_match:
|
||||
return reasoning_match.group(1), reasoning_match.group(2)
|
||||
|
||||
return None, message_text
|
||||
|
||||
|
||||
def extract_images_from_message(message: AllMessageValues) -> List[str]:
|
||||
"""
|
||||
Extract images from a message
|
||||
"""
|
||||
images = []
|
||||
message_content = message.get("content")
|
||||
if isinstance(message_content, list):
|
||||
for m in message_content:
|
||||
image_url = m.get("image_url")
|
||||
if image_url:
|
||||
if isinstance(image_url, str):
|
||||
images.append(image_url)
|
||||
elif isinstance(image_url, dict) and "url" in image_url:
|
||||
images.append(image_url["url"])
|
||||
return images
|
||||
|
||||
@@ -1024,6 +1024,8 @@ class CustomStreamWrapper:
|
||||
return
|
||||
|
||||
def chunk_creator(self, chunk: Any): # type: ignore # noqa: PLR0915
|
||||
if hasattr(chunk, 'id'):
|
||||
self.response_id = chunk.id
|
||||
model_response = self.model_response_creator()
|
||||
response_obj: Dict[str, Any] = {}
|
||||
try:
|
||||
|
||||
@@ -107,10 +107,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
||||
user_anthropic_beta_headers: Optional[List[str]] = None,
|
||||
) -> dict:
|
||||
betas = set()
|
||||
# Note: prompt-caching-2024-07-31 header is no longer required for prompt caching
|
||||
# as per current Anthropic documentation. It's now generally available.
|
||||
# if prompt_caching_set:
|
||||
# betas.add("prompt-caching-2024-07-31")
|
||||
if prompt_caching_set:
|
||||
betas.add("prompt-caching-2024-07-31")
|
||||
if computer_tool_used:
|
||||
betas.add("computer-use-2024-10-22")
|
||||
# if pdf_used:
|
||||
@@ -178,11 +176,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
||||
mcp_server_used=mcp_server_used,
|
||||
)
|
||||
|
||||
# For Vertex AI requests, remove any user-provided anthropic-beta headers
|
||||
# since Vertex AI rejects them and they're no longer required for prompt caching
|
||||
if optional_params.get("is_vertex_request", False):
|
||||
headers = {k: v for k, v in headers.items() if k != "anthropic-beta"}
|
||||
|
||||
headers = {**headers, **anthropic_headers}
|
||||
|
||||
return headers
|
||||
|
||||
@@ -28,10 +28,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
||||
TextBlock,
|
||||
)
|
||||
|
||||
def __init__(self, completion_stream: Any, model: str):
|
||||
super().__init__(completion_stream)
|
||||
self.model = model
|
||||
|
||||
sent_first_chunk: bool = False
|
||||
sent_content_block_start: bool = False
|
||||
sent_content_block_finish: bool = False
|
||||
@@ -39,6 +35,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
||||
sent_last_message: bool = False
|
||||
holding_chunk: Optional[Any] = None
|
||||
holding_stop_reason_chunk: Optional[Any] = None
|
||||
queued_usage_chunk: bool = False
|
||||
current_content_block_index: int = 0
|
||||
current_content_block_start: ContentBlockContentBlockDict = TextBlock(
|
||||
type="text",
|
||||
@@ -47,6 +44,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
||||
pending_new_content_block: bool = False
|
||||
chunk_queue: deque = deque() # Queue for buffering multiple chunks
|
||||
|
||||
def __init__(self, completion_stream: Any, model: str):
|
||||
super().__init__(completion_stream)
|
||||
self.model = model
|
||||
|
||||
def __next__(self):
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
@@ -217,77 +218,83 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
||||
|
||||
# Queue the merged chunk and reset
|
||||
self.chunk_queue.append(merged_chunk)
|
||||
self.queued_usage_chunk = True
|
||||
self.holding_stop_reason_chunk = None
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
# Check if this processed chunk has a stop_reason - hold it for next chunk
|
||||
|
||||
if should_start_new_block and not self.sent_content_block_finish:
|
||||
# Queue the sequence: content_block_stop -> content_block_start -> current_chunk
|
||||
if not self.queued_usage_chunk:
|
||||
if should_start_new_block and not self.sent_content_block_finish:
|
||||
# Queue the sequence: content_block_stop -> content_block_start -> current_chunk
|
||||
|
||||
# 1. Stop current content block
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_stop",
|
||||
"index": max(self.current_content_block_index - 1, 0),
|
||||
}
|
||||
)
|
||||
# 1. Stop current content block
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_stop",
|
||||
"index": max(self.current_content_block_index - 1, 0),
|
||||
}
|
||||
)
|
||||
|
||||
# 2. Start new content block
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": self.current_content_block_index,
|
||||
"content_block": self.current_content_block_start,
|
||||
}
|
||||
)
|
||||
# 2. Start new content block
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": self.current_content_block_index,
|
||||
"content_block": self.current_content_block_start,
|
||||
}
|
||||
)
|
||||
|
||||
# 3. Queue the current chunk (don't lose it!)
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
|
||||
# Reset state for new block
|
||||
self.sent_content_block_finish = False
|
||||
|
||||
# Return the first queued item
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
if (
|
||||
processed_chunk["type"] == "message_delta"
|
||||
and self.sent_content_block_finish is False
|
||||
):
|
||||
# Queue both the content_block_stop and the holding chunk
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_stop",
|
||||
"index": self.current_content_block_index,
|
||||
}
|
||||
)
|
||||
self.sent_content_block_finish = True
|
||||
if processed_chunk.get("delta", {}).get("stop_reason") is not None:
|
||||
|
||||
self.holding_stop_reason_chunk = processed_chunk
|
||||
else:
|
||||
# 3. Queue the current chunk (don't lose it!)
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
return self.chunk_queue.popleft()
|
||||
elif self.holding_chunk is not None:
|
||||
# Queue both chunks
|
||||
self.chunk_queue.append(self.holding_chunk)
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
self.holding_chunk = None
|
||||
return self.chunk_queue.popleft()
|
||||
else:
|
||||
# Queue the current chunk
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
# Reset state for new block
|
||||
self.sent_content_block_finish = False
|
||||
|
||||
# Return the first queued item
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
if (
|
||||
processed_chunk["type"] == "message_delta"
|
||||
and self.sent_content_block_finish is False
|
||||
):
|
||||
# Queue both the content_block_stop and the holding chunk
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_stop",
|
||||
"index": self.current_content_block_index,
|
||||
}
|
||||
)
|
||||
self.sent_content_block_finish = True
|
||||
if (
|
||||
processed_chunk.get("delta", {}).get("stop_reason")
|
||||
is not None
|
||||
):
|
||||
|
||||
self.holding_stop_reason_chunk = processed_chunk
|
||||
else:
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
return self.chunk_queue.popleft()
|
||||
elif self.holding_chunk is not None:
|
||||
# Queue both chunks
|
||||
self.chunk_queue.append(self.holding_chunk)
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
self.holding_chunk = None
|
||||
return self.chunk_queue.popleft()
|
||||
else:
|
||||
# Queue the current chunk
|
||||
self.chunk_queue.append(processed_chunk)
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
# Handle any remaining held chunks after stream ends
|
||||
if self.holding_stop_reason_chunk is not None:
|
||||
self.chunk_queue.append(self.holding_stop_reason_chunk)
|
||||
self.holding_stop_reason_chunk = None
|
||||
if not self.queued_usage_chunk:
|
||||
if self.holding_stop_reason_chunk is not None:
|
||||
self.chunk_queue.append(self.holding_stop_reason_chunk)
|
||||
self.holding_stop_reason_chunk = None
|
||||
|
||||
if self.holding_chunk is not None:
|
||||
self.chunk_queue.append(self.holding_chunk)
|
||||
self.holding_chunk = None
|
||||
if self.holding_chunk is not None:
|
||||
self.chunk_queue.append(self.holding_chunk)
|
||||
self.holding_chunk = None
|
||||
|
||||
if not self.sent_last_message:
|
||||
self.sent_last_message = True
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -23,12 +23,13 @@ else:
|
||||
class AudioTranscriptionRequestData:
|
||||
"""
|
||||
Structured data for audio transcription requests.
|
||||
|
||||
|
||||
Attributes:
|
||||
data: The request data (form data for multipart, json data for regular requests)
|
||||
files: Optional files dict for multipart form data
|
||||
content_type: Optional content type override
|
||||
"""
|
||||
|
||||
data: Union[dict, bytes]
|
||||
files: Optional[dict] = None
|
||||
content_type: Optional[str] = None
|
||||
@@ -66,13 +67,11 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
|
||||
audio_file: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> Union[AudioTranscriptionRequestData, Dict]:
|
||||
) -> AudioTranscriptionRequestData:
|
||||
raise NotImplementedError(
|
||||
"AudioTranscriptionConfig needs a request transformation for audio transcription models"
|
||||
)
|
||||
|
||||
|
||||
|
||||
def transform_audio_transcription_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
@@ -110,7 +109,6 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
|
||||
raise NotImplementedError(
|
||||
"AudioTranscriptionConfig does not need a response transformation for audio transcription models"
|
||||
)
|
||||
|
||||
|
||||
def get_provider_specific_params(
|
||||
self,
|
||||
@@ -141,7 +139,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
|
||||
provider_specific_params[key] = value
|
||||
|
||||
return provider_specific_params
|
||||
|
||||
|
||||
def _should_exclude_param(
|
||||
self,
|
||||
param_name: str,
|
||||
|
||||
@@ -124,15 +124,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
|
||||
"AWS IAM role ARN is required for Bedrock batch jobs. "
|
||||
"Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var"
|
||||
)
|
||||
|
||||
|
||||
# Get the actual Bedrock model ID using common utility
|
||||
bedrock_model_id = self.common_utils.extract_model_from_s3_file_path(input_file_id, optional_params)
|
||||
|
||||
if not bedrock_model_id:
|
||||
raise ValueError("Could not determine Bedrock model ID. Ensure the model is specified in the input file or passed as a parameter.")
|
||||
if not model:
|
||||
raise ValueError("Could not determine Bedrock model ID. Please pass `model` in your request body.")
|
||||
|
||||
# Generate job name with the correct model ID using common utility
|
||||
job_name = self.common_utils.generate_unique_job_name(bedrock_model_id, prefix="litellm")
|
||||
job_name = self.common_utils.generate_unique_job_name(model, prefix="litellm")
|
||||
output_key = f"litellm-batch-outputs/{job_name}/"
|
||||
|
||||
# Build input data config
|
||||
@@ -151,7 +149,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
|
||||
|
||||
# Create Bedrock batch request with proper typing
|
||||
bedrock_request: BedrockCreateBatchRequest = {
|
||||
"modelId": bedrock_model_id,
|
||||
"modelId": model,
|
||||
"jobName": job_name,
|
||||
"inputDataConfig": input_data_config,
|
||||
"outputDataConfig": output_data_config,
|
||||
|
||||
@@ -14,7 +14,7 @@ from litellm._logging import verbose_logger
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
@@ -397,7 +397,11 @@ class AmazonConverseConfig(BaseConfig):
|
||||
for param, value in non_default_params.items():
|
||||
if param == "response_format" and isinstance(value, dict):
|
||||
optional_params = self._translate_response_format_param(
|
||||
value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled
|
||||
value=value,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
non_default_params=non_default_params,
|
||||
is_thinking_enabled=is_thinking_enabled,
|
||||
)
|
||||
if param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["maxTokens"] = value
|
||||
@@ -446,11 +450,11 @@ class AmazonConverseConfig(BaseConfig):
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
|
||||
def _translate_response_format_param(
|
||||
self,
|
||||
value: dict,
|
||||
model: str,
|
||||
self,
|
||||
value: dict,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
non_default_params: dict,
|
||||
is_thinking_enabled: bool,
|
||||
@@ -504,7 +508,7 @@ class AmazonConverseConfig(BaseConfig):
|
||||
optional_params["json_mode"] = True
|
||||
if non_default_params.get("stream", False) is True:
|
||||
optional_params["fake_stream"] = True
|
||||
|
||||
|
||||
return optional_params
|
||||
|
||||
def update_optional_params_with_thinking_tokens(
|
||||
|
||||
@@ -3,7 +3,7 @@ from typing import Any, List, Optional, cast
|
||||
from httpx import Response
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
|
||||
@@ -6,6 +6,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from httpx import Headers, Response
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.files.utils import FilesAPIUtils
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
@@ -21,6 +23,7 @@ from litellm.types.llms.openai import (
|
||||
PathLike,
|
||||
)
|
||||
from litellm.types.utils import ExtractedFileData, LlmProviders
|
||||
from litellm.utils import get_llm_provider
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import BedrockError
|
||||
@@ -111,6 +114,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
# Remove bedrock/ prefix if present
|
||||
if _model.startswith("bedrock/"):
|
||||
_model = _model[8:]
|
||||
|
||||
# Replace colons with hyphens for Bedrock S3 URI compliance
|
||||
_model = _model.replace(":", "-")
|
||||
|
||||
object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
|
||||
return object_name
|
||||
|
||||
@@ -191,24 +198,6 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
) -> dict:
|
||||
return optional_params
|
||||
|
||||
def _get_bedrock_provider_from_model(self, model: str) -> Optional[str]:
|
||||
"""
|
||||
Extract provider from Bedrock model name
|
||||
"""
|
||||
if model.startswith("anthropic."):
|
||||
return "anthropic"
|
||||
elif model.startswith("cohere."):
|
||||
return "cohere"
|
||||
elif model.startswith("meta.") or model.startswith("llama"):
|
||||
return "meta"
|
||||
elif model.startswith("mistral."):
|
||||
return "mistral"
|
||||
elif model.startswith("ai21."):
|
||||
return "ai21"
|
||||
elif model.startswith("amazon."):
|
||||
return "amazon"
|
||||
else:
|
||||
return None
|
||||
|
||||
def _map_openai_to_bedrock_params(
|
||||
self,
|
||||
@@ -218,11 +207,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
"""
|
||||
Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic
|
||||
"""
|
||||
from litellm.types.utils import LlmProviders
|
||||
_model = openai_request_body.get("model", "")
|
||||
messages = openai_request_body.get("messages", [])
|
||||
|
||||
# Use existing Anthropic transformation logic for Anthropic models
|
||||
if provider == "anthropic":
|
||||
if provider == LlmProviders.ANTHROPIC:
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
@@ -231,16 +221,22 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
|
||||
# Extract optional params (everything except model and messages)
|
||||
optional_params = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
|
||||
mapped_params = anthropic_config.map_openai_params(
|
||||
non_default_params={},
|
||||
optional_params=optional_params,
|
||||
model=_model,
|
||||
drop_params=False
|
||||
)
|
||||
|
||||
# Transform using existing Anthropic logic
|
||||
bedrock_params = anthropic_config.transform_request(
|
||||
model=_model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
optional_params=mapped_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
|
||||
|
||||
return bedrock_params
|
||||
else:
|
||||
# For other providers, use basic mapping
|
||||
@@ -278,9 +274,17 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
# Extract the request body from OpenAI format
|
||||
openai_body = _openai_jsonl_content.get("body", {})
|
||||
model = openai_body.get("model", "")
|
||||
|
||||
try:
|
||||
model, _, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=None,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {str(e)}")
|
||||
|
||||
# Determine provider from model name
|
||||
provider = self._get_bedrock_provider_from_model(model)
|
||||
provider = self.get_bedrock_invoke_provider(model)
|
||||
|
||||
# Transform to Bedrock modelInput format
|
||||
model_input = self._map_openai_to_bedrock_params(
|
||||
@@ -315,11 +319,13 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
extracted_file_data = extract_file_data(file_data)
|
||||
extracted_file_data_content = extracted_file_data.get("content")
|
||||
|
||||
if extracted_file_data_content is None:
|
||||
raise ValueError("file content is required")
|
||||
|
||||
# Get and transform the file content
|
||||
if (
|
||||
create_file_data.get("purpose") == "batch"
|
||||
and extracted_file_data.get("content_type") == "application/jsonl"
|
||||
and extracted_file_data_content is not None
|
||||
if FilesAPIUtils.is_batch_jsonl_file(
|
||||
create_file_data=create_file_data,
|
||||
extracted_file_data=extracted_file_data,
|
||||
):
|
||||
## Transform JSONL content to Bedrock format
|
||||
original_file_content = self._get_content_from_openai_file(
|
||||
@@ -357,6 +363,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
api_base=api_base,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
litellm_params["upload_url"] = api_base
|
||||
|
||||
# Return a dict that tells the HTTP handler exactly what to do
|
||||
return {
|
||||
@@ -440,6 +448,56 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
|
||||
return dict(aws_request.headers), signed_body
|
||||
|
||||
def _convert_https_url_to_s3_uri(self, https_url: str) -> tuple[str, str]:
|
||||
"""
|
||||
Convert HTTPS S3 URL to s3:// URI format.
|
||||
|
||||
Args:
|
||||
https_url: HTTPS S3 URL (e.g., "https://s3.us-west-2.amazonaws.com/bucket/key")
|
||||
|
||||
Returns:
|
||||
Tuple of (s3_uri, filename)
|
||||
|
||||
Example:
|
||||
Input: "https://s3.us-west-2.amazonaws.com/litellm-proxy/file.jsonl"
|
||||
Output: ("s3://litellm-proxy/file.jsonl", "file.jsonl")
|
||||
"""
|
||||
import re
|
||||
|
||||
# Match HTTPS S3 URL patterns
|
||||
# Pattern 1: https://s3.region.amazonaws.com/bucket/key
|
||||
# Pattern 2: https://bucket.s3.region.amazonaws.com/key
|
||||
|
||||
pattern1 = r"https://s3\.([^.]+)\.amazonaws\.com/([^/]+)/(.+)"
|
||||
pattern2 = r"https://([^.]+)\.s3\.([^.]+)\.amazonaws\.com/(.+)"
|
||||
|
||||
match1 = re.match(pattern1, https_url)
|
||||
match2 = re.match(pattern2, https_url)
|
||||
|
||||
if match1:
|
||||
# Pattern: https://s3.region.amazonaws.com/bucket/key
|
||||
region, bucket, key = match1.groups()
|
||||
s3_uri = f"s3://{bucket}/{key}"
|
||||
elif match2:
|
||||
# Pattern: https://bucket.s3.region.amazonaws.com/key
|
||||
bucket, region, key = match2.groups()
|
||||
s3_uri = f"s3://{bucket}/{key}"
|
||||
else:
|
||||
# Fallback: try to extract bucket and key from URL path
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(https_url)
|
||||
path_parts = parsed.path.lstrip('/').split('/', 1)
|
||||
if len(path_parts) >= 2:
|
||||
bucket, key = path_parts[0], path_parts[1]
|
||||
s3_uri = f"s3://{bucket}/{key}"
|
||||
else:
|
||||
raise ValueError(f"Unable to parse S3 URL: {https_url}")
|
||||
|
||||
# Extract filename from key
|
||||
filename = key.split("/")[-1] if "/" in key else key
|
||||
|
||||
return s3_uri, filename
|
||||
|
||||
def transform_create_file_response(
|
||||
self,
|
||||
model: Optional[str],
|
||||
@@ -452,21 +510,18 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||
"""
|
||||
# For S3 uploads, we typically get an ETag and other metadata
|
||||
response_headers = raw_response.headers
|
||||
|
||||
# Extract S3 object information from the response
|
||||
# S3 PUT object returns ETag and other metadata in headers
|
||||
content_length = response_headers.get("Content-Length", "0")
|
||||
|
||||
# Extract bucket and key from the request URL or litellm_params
|
||||
bucket_name = litellm_params.get("s3_bucket_name") or os.getenv("AWS_S3_BUCKET_NAME")
|
||||
|
||||
# Generate file ID in S3 format
|
||||
object_key = getattr(logging_obj, 'object_key', None) or f"file-{int(time.time())}"
|
||||
file_id = f"s3://{bucket_name}/{object_key}"
|
||||
|
||||
# Extract filename from object key
|
||||
filename = object_key.split("/")[-1] if "/" in object_key else object_key
|
||||
|
||||
# Use the actual upload URL that was used for the S3 upload
|
||||
upload_url = litellm_params.get("upload_url")
|
||||
file_id: str = ""
|
||||
filename: str = ""
|
||||
if upload_url:
|
||||
# Convert HTTPS S3 URL to s3:// URI format
|
||||
file_id, filename = self._convert_https_url_to_s3_uri(upload_url)
|
||||
|
||||
return OpenAIFileObject(
|
||||
purpose="batch", # Default purpose for Bedrock files
|
||||
id=file_id,
|
||||
|
||||
@@ -118,7 +118,6 @@ class BaseLLMHTTPHandler:
|
||||
response: Optional[httpx.Response] = None
|
||||
for i in range(max(max_retry_on_unprocessable_entity_error, 1)):
|
||||
try:
|
||||
|
||||
response = await async_httpx_client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
@@ -2201,7 +2200,6 @@ class BaseLLMHTTPHandler:
|
||||
litellm_params=litellm_params,
|
||||
optional_params={},
|
||||
)
|
||||
|
||||
if _is_async:
|
||||
return self.async_create_file(
|
||||
transformed_request=transformed_request,
|
||||
@@ -2218,10 +2216,13 @@ class BaseLLMHTTPHandler:
|
||||
sync_httpx_client = _get_httpx_client()
|
||||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
|
||||
if isinstance(transformed_request, dict) and "method" in transformed_request:
|
||||
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
|
||||
upload_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
|
||||
upload_response = getattr(
|
||||
sync_httpx_client, transformed_request["method"].lower()
|
||||
)(
|
||||
url=transformed_request["url"],
|
||||
headers=transformed_request["headers"],
|
||||
data=transformed_request["data"],
|
||||
@@ -2233,8 +2234,8 @@ class BaseLLMHTTPHandler:
|
||||
# Handle traditional file uploads
|
||||
# Ensure transformed_request is a string for httpx compatibility
|
||||
if isinstance(transformed_request, bytes):
|
||||
transformed_request = transformed_request.decode('utf-8')
|
||||
|
||||
transformed_request = transformed_request.decode("utf-8")
|
||||
|
||||
# Use the HTTP method specified by the provider config
|
||||
http_method = provider_config.file_upload_http_method.upper()
|
||||
if http_method == "PUT":
|
||||
@@ -2283,11 +2284,15 @@ class BaseLLMHTTPHandler:
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
# Store the upload URL in litellm_params for the transformation method
|
||||
litellm_params_with_url = dict(litellm_params)
|
||||
litellm_params_with_url["upload_url"] = api_base
|
||||
|
||||
return provider_config.transform_create_file_response(
|
||||
model=None,
|
||||
raw_response=upload_response,
|
||||
logging_obj=logging_obj,
|
||||
litellm_params=litellm_params,
|
||||
litellm_params=litellm_params_with_url,
|
||||
)
|
||||
|
||||
async def async_create_file(
|
||||
@@ -2310,7 +2315,7 @@ class BaseLLMHTTPHandler:
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
|
||||
#########################################################
|
||||
# Debug Logging
|
||||
#########################################################
|
||||
@@ -2326,7 +2331,9 @@ class BaseLLMHTTPHandler:
|
||||
|
||||
if isinstance(transformed_request, dict) and "method" in transformed_request:
|
||||
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
|
||||
upload_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
|
||||
upload_response = await getattr(
|
||||
async_httpx_client, transformed_request["method"].lower()
|
||||
)(
|
||||
url=transformed_request["url"],
|
||||
headers=transformed_request["headers"],
|
||||
data=transformed_request["data"],
|
||||
@@ -2338,8 +2345,8 @@ class BaseLLMHTTPHandler:
|
||||
# Handle traditional file uploads
|
||||
# Ensure transformed_request is a string for httpx compatibility
|
||||
if isinstance(transformed_request, bytes):
|
||||
transformed_request = transformed_request.decode('utf-8')
|
||||
|
||||
transformed_request = transformed_request.decode("utf-8")
|
||||
|
||||
# Use the HTTP method specified by the provider config
|
||||
http_method = provider_config.file_upload_http_method.upper()
|
||||
if http_method == "PUT":
|
||||
@@ -2408,15 +2415,19 @@ class BaseLLMHTTPHandler:
|
||||
_is_async: bool = False,
|
||||
client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]:
|
||||
"""
|
||||
Creates a batch using provider-specific batch creation process
|
||||
"""
|
||||
# get config from model, custom llm provider
|
||||
if model is None:
|
||||
raise ValueError("model is required for create_batch")
|
||||
|
||||
headers = provider_config.validate_environment(
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
model="",
|
||||
model=model,
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
@@ -2425,7 +2436,7 @@ class BaseLLMHTTPHandler:
|
||||
api_base = provider_config.get_complete_batch_url(
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model="",
|
||||
model=model,
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
data=create_batch_data,
|
||||
@@ -2435,7 +2446,7 @@ class BaseLLMHTTPHandler:
|
||||
|
||||
# Get the transformed request data
|
||||
transformed_request = provider_config.transform_create_batch_request(
|
||||
model="",
|
||||
model=model,
|
||||
create_batch_data=create_batch_data,
|
||||
litellm_params=litellm_params,
|
||||
optional_params={},
|
||||
@@ -2460,9 +2471,14 @@ class BaseLLMHTTPHandler:
|
||||
sync_httpx_client = client
|
||||
|
||||
try:
|
||||
if isinstance(transformed_request, dict) and "method" in transformed_request:
|
||||
if (
|
||||
isinstance(transformed_request, dict)
|
||||
and "method" in transformed_request
|
||||
):
|
||||
# Handle pre-signed requests (e.g., from Bedrock with AWS auth)
|
||||
batch_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
|
||||
batch_response = getattr(
|
||||
sync_httpx_client, transformed_request["method"].lower()
|
||||
)(
|
||||
url=transformed_request["url"],
|
||||
headers=transformed_request["headers"],
|
||||
data=transformed_request["data"],
|
||||
@@ -2492,10 +2508,13 @@ class BaseLLMHTTPHandler:
|
||||
)
|
||||
|
||||
# Store original request for response transformation
|
||||
litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data}
|
||||
|
||||
litellm_params_with_request = {
|
||||
**litellm_params,
|
||||
"original_batch_request": create_batch_data,
|
||||
}
|
||||
|
||||
return provider_config.transform_create_batch_response(
|
||||
model=None,
|
||||
model=model,
|
||||
raw_response=batch_response,
|
||||
logging_obj=logging_obj,
|
||||
litellm_params=litellm_params_with_request,
|
||||
@@ -2512,6 +2531,7 @@ class BaseLLMHTTPHandler:
|
||||
client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
create_batch_data: Optional["CreateBatchRequest"] = None,
|
||||
model: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Async version of create_batch
|
||||
@@ -2522,7 +2542,7 @@ class BaseLLMHTTPHandler:
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
|
||||
#########################################################
|
||||
# Debug Logging
|
||||
#########################################################
|
||||
@@ -2537,9 +2557,14 @@ class BaseLLMHTTPHandler:
|
||||
)
|
||||
|
||||
try:
|
||||
if isinstance(transformed_request, dict) and "method" in transformed_request:
|
||||
if (
|
||||
isinstance(transformed_request, dict)
|
||||
and "method" in transformed_request
|
||||
):
|
||||
# Handle pre-signed requests (e.g., from Bedrock with AWS auth)
|
||||
batch_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
|
||||
batch_response = await getattr(
|
||||
async_httpx_client, transformed_request["method"].lower()
|
||||
)(
|
||||
url=transformed_request["url"],
|
||||
headers=transformed_request["headers"],
|
||||
data=transformed_request["data"],
|
||||
@@ -2569,10 +2594,13 @@ class BaseLLMHTTPHandler:
|
||||
)
|
||||
|
||||
# Store original request for response transformation (for async version)
|
||||
litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}}
|
||||
|
||||
litellm_params_with_request = {
|
||||
**litellm_params,
|
||||
"original_batch_request": create_batch_data or {},
|
||||
}
|
||||
|
||||
return provider_config.transform_create_batch_response(
|
||||
model=None,
|
||||
model=model,
|
||||
raw_response=batch_response,
|
||||
logging_obj=logging_obj,
|
||||
litellm_params=litellm_params_with_request,
|
||||
|
||||
@@ -1,21 +1,155 @@
|
||||
"""
|
||||
Cost calculator for DeepSeek Chat models.
|
||||
Cost calculator for Dashscope Chat models.
|
||||
|
||||
Handles prompt caching scenario.
|
||||
Handles tiered pricing and prompt caching scenarios.
|
||||
"""
|
||||
|
||||
from typing import Tuple
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenBreakdown:
|
||||
"""Token breakdown for cost calculation."""
|
||||
text_tokens: int
|
||||
cached_tokens: int
|
||||
completion_tokens: int
|
||||
reasoning_tokens: int
|
||||
|
||||
|
||||
def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
|
||||
"""Extract token counts from usage, handling cached and reasoning tokens."""
|
||||
cached_tokens = 0
|
||||
if usage.prompt_tokens_details and hasattr(usage.prompt_tokens_details, "cached_tokens"):
|
||||
cached_tokens = usage.prompt_tokens_details.cached_tokens or 0
|
||||
|
||||
text_tokens = usage.prompt_tokens - cached_tokens
|
||||
|
||||
reasoning_tokens = 0
|
||||
if (hasattr(usage, "completion_tokens_details") and
|
||||
usage.completion_tokens_details and
|
||||
hasattr(usage.completion_tokens_details, "reasoning_tokens")):
|
||||
reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0
|
||||
|
||||
completion_tokens = (usage.completion_tokens or 0) - reasoning_tokens
|
||||
|
||||
return TokenBreakdown(text_tokens, cached_tokens, completion_tokens, reasoning_tokens)
|
||||
|
||||
|
||||
def _calculate_tiered_cost(
|
||||
tokens: int,
|
||||
tiered_pricing: List[dict],
|
||||
cost_key: str,
|
||||
fallback_cost_key: Optional[str] = None
|
||||
) -> float:
|
||||
"""Calculate cost using tiered pricing structure.
|
||||
|
||||
Finds the appropriate tier based on token count and applies that tier's rate to all tokens.
|
||||
"""
|
||||
if not tiered_pricing or tokens <= 0:
|
||||
return 0.0
|
||||
|
||||
# Find the appropriate tier for the token count
|
||||
for tier in tiered_pricing:
|
||||
tier_range = tier.get("range", [])
|
||||
if len(tier_range) != 2:
|
||||
continue
|
||||
|
||||
range_start, range_end = tier_range
|
||||
|
||||
# Check if tokens fall within this tier's range
|
||||
if range_start <= tokens <= range_end:
|
||||
cost_per_token = tier.get(cost_key) or tier.get(fallback_cost_key, 0)
|
||||
return tokens * cost_per_token
|
||||
|
||||
# If no tier matches, use the last tier (highest tier)
|
||||
if tiered_pricing:
|
||||
last_tier = tiered_pricing[-1]
|
||||
cost_per_token = last_tier.get(cost_key) or last_tier.get(fallback_cost_key, 0)
|
||||
return tokens * cost_per_token
|
||||
|
||||
return 0.0
|
||||
|
||||
|
||||
def _calculate_flat_cost(tokens: int, cost_per_token: float) -> float:
|
||||
"""Calculate cost using flat pricing."""
|
||||
return tokens * cost_per_token
|
||||
|
||||
|
||||
def _calculate_prompt_cost(breakdown: TokenBreakdown, model_info: ModelInfo, tiered_pricing: Optional[List[dict]]) -> float:
|
||||
"""Calculate total prompt cost including cached tokens."""
|
||||
if tiered_pricing:
|
||||
text_cost = _calculate_tiered_cost(
|
||||
tokens=breakdown.text_tokens,
|
||||
tiered_pricing=tiered_pricing,
|
||||
cost_key="input_cost_per_token"
|
||||
)
|
||||
cache_cost = _calculate_tiered_cost(
|
||||
tokens=breakdown.cached_tokens,
|
||||
tiered_pricing=tiered_pricing,
|
||||
cost_key="cache_read_input_token_cost"
|
||||
)
|
||||
return text_cost + cache_cost
|
||||
|
||||
input_cost = model_info.get("input_cost_per_token", 0.0)
|
||||
cache_cost = model_info.get("cache_read_input_token_cost", input_cost) or input_cost
|
||||
|
||||
return (_calculate_flat_cost(tokens=breakdown.text_tokens, cost_per_token=input_cost) +
|
||||
_calculate_flat_cost(tokens=breakdown.cached_tokens, cost_per_token=cache_cost))
|
||||
|
||||
|
||||
def _calculate_completion_cost(breakdown: TokenBreakdown, model_info: ModelInfo, tiered_pricing: Optional[List[dict]]) -> float:
|
||||
"""Calculate total completion cost including reasoning tokens."""
|
||||
if tiered_pricing:
|
||||
completion_cost = _calculate_tiered_cost(
|
||||
tokens=breakdown.completion_tokens,
|
||||
tiered_pricing=tiered_pricing,
|
||||
cost_key="output_cost_per_token"
|
||||
)
|
||||
reasoning_cost = _calculate_tiered_cost(
|
||||
tokens=breakdown.reasoning_tokens,
|
||||
tiered_pricing=tiered_pricing,
|
||||
cost_key="output_cost_per_reasoning_token",
|
||||
fallback_cost_key="output_cost_per_token"
|
||||
)
|
||||
return completion_cost + reasoning_cost
|
||||
|
||||
output_cost = model_info.get("output_cost_per_token", 0.0)
|
||||
reasoning_cost = model_info.get("output_cost_per_reasoning_token", output_cost) or output_cost
|
||||
|
||||
return (_calculate_flat_cost(tokens=breakdown.completion_tokens, cost_per_token=output_cost) +
|
||||
_calculate_flat_cost(tokens=breakdown.reasoning_tokens, cost_per_token=reasoning_cost))
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
||||
|
||||
Follows the same logic as Anthropic's cost per token calculation.
|
||||
Calculate cost per token for Dashscope models.
|
||||
|
||||
Supports both tiered and flat pricing with cached and reasoning tokens.
|
||||
|
||||
Args:
|
||||
model: Model name without provider prefix
|
||||
usage: LiteLLM Usage block
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
|
||||
"""
|
||||
return generic_cost_per_token(
|
||||
model=model, usage=usage, custom_llm_provider="deepseek"
|
||||
model_info = get_model_info(model=model, custom_llm_provider="dashscope")
|
||||
breakdown = _extract_token_breakdown(usage)
|
||||
tiered_pricing = model_info.get("tiered_pricing") if isinstance(model_info.get("tiered_pricing"), list) else None
|
||||
|
||||
prompt_cost = _calculate_prompt_cost(
|
||||
breakdown=breakdown,
|
||||
model_info=model_info,
|
||||
tiered_pricing=tiered_pricing
|
||||
)
|
||||
completion_cost = _calculate_completion_cost(
|
||||
breakdown=breakdown,
|
||||
model_info=model_info,
|
||||
tiered_pricing=tiered_pricing
|
||||
)
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
||||
@@ -169,17 +169,20 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
||||
if tool is None:
|
||||
return None
|
||||
|
||||
kwags = {
|
||||
"name":tool["name"],
|
||||
"parameters":cast(dict, tool.get("input_schema") or {})
|
||||
# Build DatabricksFunction explicitly to avoid parameter conflicts
|
||||
function_params: DatabricksFunction = {
|
||||
"name": tool["name"],
|
||||
"parameters": cast(dict, tool.get("input_schema") or {})
|
||||
}
|
||||
|
||||
if tool.get("description"):
|
||||
kwags["description"] = tool.get("description")
|
||||
|
||||
# Only add description if it exists
|
||||
description = tool.get("description")
|
||||
if description is not None:
|
||||
function_params["description"] = cast(Union[dict, str], description)
|
||||
|
||||
return DatabricksTool(
|
||||
type="function",
|
||||
function=DatabricksFunction(**kwags),
|
||||
function=function_params,
|
||||
)
|
||||
|
||||
def _map_openai_to_dbrx_tool(self, model: str, tools: List) -> List[DatabricksTool]:
|
||||
@@ -336,8 +339,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
||||
elif isinstance(content, list):
|
||||
content_str = ""
|
||||
for item in content:
|
||||
if item["type"] == "text":
|
||||
content_str += item["text"]
|
||||
if item.get("type") == "text":
|
||||
text_value = item.get("text", "")
|
||||
content_str += str(text_value) if text_value is not None else ""
|
||||
return content_str
|
||||
else:
|
||||
raise Exception(f"Unsupported content type: {type(content)}")
|
||||
@@ -366,19 +370,21 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
||||
reasoning_content: Optional[str] = None
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if item["type"] == "reasoning":
|
||||
for sum in item["summary"]:
|
||||
if reasoning_content is None:
|
||||
reasoning_content = ""
|
||||
reasoning_content += sum["text"]
|
||||
thinking_block = ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
thinking=sum.get("text", ""),
|
||||
signature=sum.get("signature", ""),
|
||||
)
|
||||
if thinking_blocks is None:
|
||||
thinking_blocks = []
|
||||
thinking_blocks.append(thinking_block)
|
||||
if item.get("type") == "reasoning":
|
||||
summary_list = item.get("summary", [])
|
||||
if isinstance(summary_list, list):
|
||||
for sum in summary_list:
|
||||
if reasoning_content is None:
|
||||
reasoning_content = ""
|
||||
reasoning_content += sum["text"]
|
||||
thinking_block = ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
thinking=sum.get("text", ""),
|
||||
signature=sum.get("signature", ""),
|
||||
)
|
||||
if thinking_blocks is None:
|
||||
thinking_blocks = []
|
||||
thinking_blocks.append(thinking_block)
|
||||
return reasoning_content, thinking_blocks
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, cast
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_generic_image_chunk_to_openai_image_obj,
|
||||
convert_to_anthropic_image_obj,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
convert_url_to_base64,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionFileObject
|
||||
from litellm.types.llms.vertex_ai import ContentType, PartType
|
||||
from litellm.utils import supports_reasoning
|
||||
|
||||
@@ -99,7 +102,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
||||
self, messages: List[AllMessageValues]
|
||||
) -> List[ContentType]:
|
||||
"""
|
||||
Google AI Studio Gemini does not support image urls in messages.
|
||||
Google AI Studio Gemini does not support HTTP/HTTPS URLs for files.
|
||||
Convert them to base64 data instead.
|
||||
"""
|
||||
for message in messages:
|
||||
_message_content = message.get("content")
|
||||
@@ -124,4 +128,16 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
||||
image_obj
|
||||
)
|
||||
)
|
||||
elif element.get("type") == "file":
|
||||
file_element = cast(ChatCompletionFileObject, element)
|
||||
file_id = file_element["file"].get("file_id")
|
||||
if file_id and ("http://" in file_id or "https://" in file_id):
|
||||
# Convert HTTP/HTTPS file URL to base64 data
|
||||
try:
|
||||
base64_data = convert_url_to_base64(file_id)
|
||||
file_element["file"]["file_data"] = base64_data # type: ignore
|
||||
file_element["file"].pop("file_id", None) # type: ignore
|
||||
except Exception:
|
||||
# If conversion fails, leave as is and let the API handle it
|
||||
pass
|
||||
return _gemini_convert_messages_with_history(messages=messages)
|
||||
|
||||
@@ -11,7 +11,39 @@ if TYPE_CHECKING:
|
||||
else:
|
||||
GenerateContentContentListUnionDict = Any
|
||||
|
||||
|
||||
class GoogleAIStudioTokenCounter:
|
||||
def _clean_contents_for_gemini_api(self, contents: Any) -> Any:
|
||||
"""
|
||||
Clean up contents to remove unsupported fields for the Gemini API.
|
||||
|
||||
The Google Gemini API doesn't recognize the 'id' field in function responses,
|
||||
so we need to remove it to prevent 400 Bad Request errors.
|
||||
|
||||
Args:
|
||||
contents: The contents to clean up
|
||||
|
||||
Returns:
|
||||
Cleaned contents with unsupported fields removed
|
||||
"""
|
||||
import copy
|
||||
|
||||
from google.genai.types import FunctionResponse
|
||||
|
||||
cleaned_contents = copy.deepcopy(contents)
|
||||
|
||||
for content in cleaned_contents:
|
||||
parts = content["parts"]
|
||||
for part in parts:
|
||||
if "functionResponse" in part:
|
||||
function_response_data = part["functionResponse"]
|
||||
function_response_part = FunctionResponse(**function_response_data)
|
||||
function_response_part.id = None
|
||||
part["functionResponse"] = function_response_part.model_dump(
|
||||
exclude_none=True
|
||||
)
|
||||
|
||||
return cleaned_contents
|
||||
|
||||
def _construct_url(self, model: str, api_base: Optional[str] = None) -> str:
|
||||
"""
|
||||
@@ -20,7 +52,6 @@ class GoogleAIStudioTokenCounter:
|
||||
base_url = api_base or "https://generativelanguage.googleapis.com"
|
||||
return f"{base_url}/v1beta/models/{model}:countTokens"
|
||||
|
||||
|
||||
async def validate_environment(
|
||||
self,
|
||||
api_base: Optional[str] = None,
|
||||
@@ -33,7 +64,8 @@ class GoogleAIStudioTokenCounter:
|
||||
Returns a Tuple of headers and url for the Google Gen AI Studio countTokens endpoint.
|
||||
"""
|
||||
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
|
||||
headers = GoogleGenAIConfig().validate_environment(
|
||||
|
||||
headers = GoogleGenAIConfig().validate_environment(
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
model=model,
|
||||
@@ -54,7 +86,7 @@ class GoogleAIStudioTokenCounter:
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Count tokens using Google Gen AI Studio countTokens endpoint.
|
||||
|
||||
|
||||
Args:
|
||||
contents: The content to count tokens for (Google Gen AI format)
|
||||
Example: [{"parts": [{"text": "Hello world"}]}]
|
||||
@@ -63,7 +95,7 @@ class GoogleAIStudioTokenCounter:
|
||||
api_base: Optional API base URL (defaults to Google Gen AI Studio)
|
||||
timeout: Optional timeout for the request
|
||||
**kwargs: Additional parameters
|
||||
|
||||
|
||||
Returns:
|
||||
Dict containing token count information from Google Gen AI Studio API.
|
||||
Example response:
|
||||
@@ -77,14 +109,13 @@ class GoogleAIStudioTokenCounter:
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
Raises:
|
||||
ValueError: If API key is missing
|
||||
litellm.APIError: If the API call fails
|
||||
litellm.APIConnectionError: If the connection fails
|
||||
Exception: For any other unexpected errors
|
||||
"""
|
||||
# Set up API base URL
|
||||
|
||||
# Prepare headers
|
||||
headers, url = await self.validate_environment(
|
||||
@@ -94,46 +125,40 @@ class GoogleAIStudioTokenCounter:
|
||||
model=model,
|
||||
litellm_params=kwargs,
|
||||
)
|
||||
|
||||
# Prepare request body
|
||||
request_body = {
|
||||
"contents": contents
|
||||
}
|
||||
|
||||
|
||||
# Prepare request body - clean up contents to remove unsupported fields
|
||||
cleaned_contents = self._clean_contents_for_gemini_api(contents)
|
||||
request_body = {"contents": cleaned_contents}
|
||||
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=LlmProviders.GEMINI,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.post(
|
||||
url=url,
|
||||
headers=headers,
|
||||
json=request_body
|
||||
url=url, headers=headers, json=request_body
|
||||
)
|
||||
|
||||
|
||||
# Check for HTTP errors
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
# Parse response
|
||||
result = response.json()
|
||||
return result
|
||||
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
error_msg = f"Google Gen AI Studio API error: {e.response.status_code} - {e.response.text}"
|
||||
raise litellm.APIError(
|
||||
message=error_msg,
|
||||
llm_provider="gemini",
|
||||
model=model,
|
||||
status_code=e.response.status_code
|
||||
status_code=e.response.status_code,
|
||||
) from e
|
||||
except httpx.RequestError as e:
|
||||
error_msg = f"Request to Google Gen AI Studio failed: {str(e)}"
|
||||
raise litellm.APIConnectionError(
|
||||
message=error_msg,
|
||||
llm_provider="gemini",
|
||||
model=model
|
||||
message=error_msg, llm_provider="gemini", model=model
|
||||
) from e
|
||||
except Exception as e:
|
||||
error_msg = f"Unexpected error during token counting: {str(e)}"
|
||||
raise Exception(error_msg) from e
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Transformation logic for Hosted VLLM rerank
|
||||
"""
|
||||
|
||||
from typing import Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import (
|
||||
AudioTranscriptionRequestData,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.openai.transcriptions.whisper_transformation import (
|
||||
OpenAIWhisperAudioTranscriptionConfig,
|
||||
)
|
||||
from litellm.types.utils import FileTypes
|
||||
|
||||
|
||||
class HostedVLLMAudioTranscriptionError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
message: str,
|
||||
headers: Optional[Union[dict, httpx.Headers]] = None,
|
||||
):
|
||||
super().__init__(status_code=status_code, message=message, headers=headers)
|
||||
|
||||
|
||||
class HostedVLLMAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
if api_base:
|
||||
# Remove trailing slashes and ensure clean base URL
|
||||
api_base = api_base.rstrip("/")
|
||||
if not api_base.endswith("/v1/audio/transcriptions"):
|
||||
api_base = f"{api_base}/v1/audio/transcriptions"
|
||||
return api_base
|
||||
raise ValueError("api_base must be provided for Hosted VLLM rerank")
|
||||
|
||||
def transform_audio_transcription_request(
|
||||
self,
|
||||
model: str,
|
||||
audio_file: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> AudioTranscriptionRequestData:
|
||||
"""
|
||||
Transform the audio transcription request
|
||||
"""
|
||||
|
||||
data = {"model": model, "file": audio_file, **optional_params}
|
||||
|
||||
if "response_format" not in data or (
|
||||
data["response_format"] == "text" or data["response_format"] == "json"
|
||||
):
|
||||
data["response_format"] = (
|
||||
"verbose_json" # ensures 'duration' is received - used for cost calculation
|
||||
)
|
||||
|
||||
return AudioTranscriptionRequestData(
|
||||
data=data,
|
||||
)
|
||||
@@ -1,8 +1,9 @@
|
||||
import os
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, TypedDict, Union
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
@@ -15,8 +15,8 @@ class LMStudioChatConfig(OpenAIGPTConfig):
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
api_base = api_base or get_secret_str("LM_STUDIO_API_BASE") # type: ignore
|
||||
dynamic_api_key = (
|
||||
api_key or get_secret_str("LM_STUDIO_API_KEY") or " "
|
||||
) # vllm does not require an api key
|
||||
api_key or get_secret_str("LM_STUDIO_API_KEY") or "fake-api-key"
|
||||
) # LM Studio does not require an api key, but OpenAI client requires non-None value
|
||||
return api_base, dynamic_api_key
|
||||
|
||||
def map_openai_params(
|
||||
|
||||
@@ -16,9 +16,18 @@ from httpx._models import Headers, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_extract_reasoning_content,
|
||||
convert_content_list_to_str,
|
||||
extract_images_from_message,
|
||||
)
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.types.llms.ollama import OllamaToolCall, OllamaToolCallFunction
|
||||
from litellm.types.llms.ollama import (
|
||||
OllamaChatCompletionMessage,
|
||||
OllamaToolCall,
|
||||
OllamaToolCallFunction,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantToolCall,
|
||||
@@ -299,7 +308,23 @@ class OllamaChatConfig(BaseConfig):
|
||||
)
|
||||
new_tools.append(ollama_tool_call)
|
||||
cast(dict, m)["tool_calls"] = new_tools
|
||||
new_messages.append(m)
|
||||
reasoning_content, parsed_content = _extract_reasoning_content(
|
||||
cast(dict, m)
|
||||
)
|
||||
content_str = convert_content_list_to_str(cast(AllMessageValues, m))
|
||||
images = extract_images_from_message(cast(AllMessageValues, m))
|
||||
|
||||
ollama_message = OllamaChatCompletionMessage(
|
||||
role=cast(str, m.get("role")),
|
||||
)
|
||||
if reasoning_content is not None:
|
||||
ollama_message["thinking"] = reasoning_content
|
||||
if content_str is not None:
|
||||
ollama_message["content"] = content_str
|
||||
if images is not None:
|
||||
ollama_message["images"] = images
|
||||
|
||||
new_messages.append(ollama_message)
|
||||
|
||||
# Load Config
|
||||
config = self.get_config()
|
||||
@@ -361,7 +386,7 @@ class OllamaChatConfig(BaseConfig):
|
||||
del response_json_message["thinking"]
|
||||
elif response_json_message.get("content") is not None:
|
||||
# parse reasoning content from content
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
|
||||
@@ -229,7 +229,7 @@ class OllamaConfig(BaseConfig):
|
||||
model = model.split("/", 1)[1]
|
||||
api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
|
||||
api_key = self.get_api_key()
|
||||
headers = { "Authorization": f"Bearer {api_key}" } if api_key else {}
|
||||
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
||||
|
||||
try:
|
||||
response = litellm.module_level_client.post(
|
||||
@@ -279,7 +279,7 @@ class OllamaConfig(BaseConfig):
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
|
||||
@@ -272,6 +272,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS: WebSearchCallInProgressEvent,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING: WebSearchCallSearchingEvent,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED: WebSearchCallCompletedEvent,
|
||||
ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS: MCPListToolsInProgressEvent,
|
||||
ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED: MCPListToolsCompletedEvent,
|
||||
ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED: MCPListToolsFailedEvent,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS: MCPCallInProgressEvent,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA: MCPCallArgumentsDeltaEvent,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE: MCPCallArgumentsDoneEvent,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_COMPLETED: MCPCallCompletedEvent,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent,
|
||||
ResponsesAPIStreamEvents.ERROR: ErrorEvent,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from typing import List
|
||||
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import (
|
||||
AudioTranscriptionRequestData,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
|
||||
from litellm.types.utils import FileTypes
|
||||
|
||||
@@ -27,8 +30,12 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
|
||||
audio_file: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> dict:
|
||||
) -> AudioTranscriptionRequestData:
|
||||
"""
|
||||
Transform the audio transcription request
|
||||
"""
|
||||
return {"model": model, "file": audio_file, **optional_params}
|
||||
data = {"model": model, "file": audio_file, **optional_params}
|
||||
|
||||
return AudioTranscriptionRequestData(
|
||||
data=data,
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Optional, Union
|
||||
from typing import Optional, Union, cast
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
@@ -34,6 +34,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
||||
- call openai_aclient.audio.transcriptions.create by default
|
||||
"""
|
||||
try:
|
||||
|
||||
raw_response = (
|
||||
await openai_aclient.audio.transcriptions.with_raw_response.create(
|
||||
**data, timeout=timeout
|
||||
@@ -93,15 +94,14 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
||||
Handle audio transcription request
|
||||
"""
|
||||
if provider_config is not None:
|
||||
data = provider_config.transform_audio_transcription_request(
|
||||
transformed_data = provider_config.transform_audio_transcription_request(
|
||||
model=model,
|
||||
audio_file=audio_file,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("OpenAI transformation route requires a dict")
|
||||
data = cast(dict, transformed_data.data)
|
||||
else:
|
||||
data = {"model": model, "file": audio_file, **optional_params}
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from httpx import Headers
|
||||
from httpx import Headers, Response
|
||||
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import (
|
||||
AudioTranscriptionRequestData,
|
||||
BaseAudioTranscriptionConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
@@ -11,12 +12,40 @@ from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
OpenAIAudioTranscriptionOptionalParams,
|
||||
)
|
||||
from litellm.types.utils import FileTypes
|
||||
from litellm.types.utils import FileTypes, TranscriptionResponse
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
|
||||
|
||||
class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
"""
|
||||
OPTIONAL
|
||||
|
||||
Get the complete url for the request
|
||||
|
||||
Some providers need `model` in `api_base`
|
||||
"""
|
||||
## get the api base, attach the endpoint - v1/audio/transcriptions
|
||||
# strip trailing slash if present
|
||||
api_base = api_base.rstrip("/") if api_base else ""
|
||||
|
||||
# if endswith "/v1"
|
||||
if api_base and api_base.endswith("/v1"):
|
||||
api_base = f"{api_base}/audio/transcriptions"
|
||||
else:
|
||||
api_base = f"{api_base}/v1/audio/transcriptions"
|
||||
|
||||
return api_base or ""
|
||||
|
||||
def get_supported_openai_params(
|
||||
self, model: str
|
||||
) -> List[OpenAIAudioTranscriptionOptionalParams]:
|
||||
@@ -72,21 +101,22 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
||||
audio_file: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> dict:
|
||||
) -> AudioTranscriptionRequestData:
|
||||
"""
|
||||
Transform the audio transcription request
|
||||
"""
|
||||
|
||||
data = {"model": model, "file": audio_file, **optional_params}
|
||||
|
||||
if "response_format" not in data or (
|
||||
data["response_format"] == "text" or data["response_format"] == "json"
|
||||
):
|
||||
data[
|
||||
"response_format"
|
||||
] = "verbose_json" # ensures 'duration' is received - used for cost calculation
|
||||
data["response_format"] = (
|
||||
"verbose_json" # ensures 'duration' is received - used for cost calculation
|
||||
)
|
||||
|
||||
return data
|
||||
return AudioTranscriptionRequestData(
|
||||
data=data,
|
||||
)
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, Headers]
|
||||
@@ -96,3 +126,25 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def transform_audio_transcription_response(
|
||||
self,
|
||||
raw_response: Response,
|
||||
) -> TranscriptionResponse:
|
||||
try:
|
||||
raw_response_json = raw_response.json()
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}"
|
||||
)
|
||||
|
||||
if any(
|
||||
key in raw_response_json
|
||||
for key in TranscriptionResponse.model_fields.keys()
|
||||
):
|
||||
return TranscriptionResponse(**raw_response_json)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid response format. Received response does not match the expected format. Got: ",
|
||||
raw_response_json,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Support for OVHCloud AI Endpoints `/v1/chat/completions` endpoint.
|
||||
|
||||
Our unified API follows the OpenAI standard.
|
||||
More information on our website: https://endpoints.ai.cloud.ovh.net
|
||||
"""
|
||||
from typing import Optional, Union, List
|
||||
|
||||
import httpx
|
||||
from litellm import ModelResponseStream, OpenAIGPTConfig, get_model_info, verbose_logger
|
||||
from litellm.llms.ovhcloud.utils import OVHCloudException
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
class OVHCloudChatConfig(OpenAIGPTConfig):
|
||||
@property
|
||||
def custom_llm_provider(self) -> Optional[str]:
|
||||
return "ovhcloud"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Details about function calling support can be found here:
|
||||
https://help.ovhcloud.com/csm/en-gb-public-cloud-ai-endpoints-function-calling?id=kb_article_view&sysparm_article=KB0071907
|
||||
"""
|
||||
supports_function_calling: Optional[bool] = None
|
||||
try:
|
||||
model_info = get_model_info(model, custom_llm_provider="ovhcloud")
|
||||
supports_function_calling = model_info.get(
|
||||
"supports_function_calling", False
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error getting supported OpenAI params: {e}")
|
||||
pass
|
||||
|
||||
optional_params = super().get_supported_openai_params(model)
|
||||
if supports_function_calling is not True:
|
||||
verbose_logger.debug(
|
||||
"You can see our models supporting function_calling in our catalog: https://endpoints.ai.cloud.ovh.net/catalog "
|
||||
)
|
||||
optional_params.remove("tools")
|
||||
optional_params.remove("tool_choice")
|
||||
optional_params.remove("function_call")
|
||||
optional_params.remove("response_format")
|
||||
return optional_params
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
api_base = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base.rstrip("/")
|
||||
complete_url = f"{api_base}/chat/completions"
|
||||
return complete_url
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
return OVHCloudException(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
mapped_openai_params = super().map_openai_params(
|
||||
non_default_params, optional_params, model, drop_params
|
||||
)
|
||||
return mapped_openai_params
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
extra_body = optional_params.pop("extra_body", {})
|
||||
response = super().transform_request(
|
||||
model, messages, optional_params, litellm_params, headers
|
||||
)
|
||||
response.update(extra_body)
|
||||
return response
|
||||
|
||||
class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator):
|
||||
"""
|
||||
Handler for OVHCloud AI Endpoints streaming chat completion responses
|
||||
"""
|
||||
|
||||
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
|
||||
"""
|
||||
Parse individual chunks from streaming response
|
||||
"""
|
||||
try:
|
||||
if "error" in chunk:
|
||||
error_chunk = chunk["error"]
|
||||
error_message = "OVHCloud Error: {}".format(
|
||||
error_chunk.get("message", "Unknown error")
|
||||
)
|
||||
raise OVHCloudException(
|
||||
message=error_message,
|
||||
status_code=error_chunk.get("code", 400),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
new_choices = []
|
||||
for choice in chunk["choices"]:
|
||||
if "delta" in choice and "reasoning" in choice["delta"]:
|
||||
choice["delta"]["reasoning_content"] = choice["delta"].get("reasoning")
|
||||
new_choices.append(choice)
|
||||
|
||||
return ModelResponseStream(
|
||||
id=chunk["id"],
|
||||
object="chat.completion.chunk",
|
||||
created=chunk["created"],
|
||||
usage=chunk.get("usage"),
|
||||
model=chunk["model"],
|
||||
choices=new_choices,
|
||||
)
|
||||
except KeyError as e:
|
||||
raise OVHCloudException(
|
||||
message=f"KeyError: {e}, Got unexpected response from CometAPI: {chunk}",
|
||||
status_code=400,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
This is OpenAI compatible - no transformation is applied
|
||||
|
||||
"""
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
|
||||
from litellm.types.utils import EmbeddingResponse, Usage
|
||||
|
||||
from ..utils import OVHCloudException
|
||||
|
||||
|
||||
class OVHCloudEmbeddingConfig(BaseEmbeddingConfig):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
api_base = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base.rstrip("/")
|
||||
complete_url = f"{api_base}/embeddings"
|
||||
return complete_url
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
if api_key is None:
|
||||
api_key = get_secret_str("OVHCLOUD_API_KEY")
|
||||
|
||||
default_headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
if "Authorization" in headers:
|
||||
default_headers["Authorization"] = headers["Authorization"]
|
||||
|
||||
return {**default_headers, **headers}
|
||||
|
||||
def get_supported_openai_params(self, model: str):
|
||||
return []
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
):
|
||||
supported_openai_params = self.get_supported_openai_params(model)
|
||||
for param, value in non_default_params.items():
|
||||
if param in supported_openai_params:
|
||||
optional_params[param] = value
|
||||
return optional_params
|
||||
|
||||
def transform_embedding_request(
|
||||
self,
|
||||
model: str,
|
||||
input: AllEmbeddingInputValues,
|
||||
optional_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
return {"input": input, "model": model, **optional_params}
|
||||
|
||||
def transform_embedding_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: EmbeddingResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
api_key: Optional[str],
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> EmbeddingResponse:
|
||||
try:
|
||||
raw_response_json = raw_response.json()
|
||||
except Exception:
|
||||
raise OVHCloudException(
|
||||
message=raw_response.text,
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
model_response.model = raw_response_json.get("model")
|
||||
model_response.data = raw_response_json.get("data")
|
||||
model_response.object = raw_response_json.get("object")
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=raw_response_json.get("usage", {}).get("prompt_tokens", 0),
|
||||
total_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0),
|
||||
)
|
||||
|
||||
model_response.usage = usage
|
||||
return model_response
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
return OVHCloudException(
|
||||
message=error_message, status_code=status_code, headers=headers
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
||||
class OVHCloudException(BaseLLMException):
|
||||
"""OVHCloud AI Endpoints exception handling class"""
|
||||
pass
|
||||
@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from httpx import Headers, Response
|
||||
|
||||
from litellm.files.utils import FilesAPIUtils
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
@@ -260,10 +261,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
||||
raise ValueError("file is required")
|
||||
extracted_file_data = extract_file_data(file_data)
|
||||
extracted_file_data_content = extracted_file_data.get("content")
|
||||
if (
|
||||
create_file_data.get("purpose") == "batch"
|
||||
and extracted_file_data.get("content_type") == "application/jsonl"
|
||||
and extracted_file_data_content is not None
|
||||
|
||||
if extracted_file_data_content is None:
|
||||
raise ValueError("file content is required")
|
||||
|
||||
if FilesAPIUtils.is_batch_jsonl_file(
|
||||
create_file_data=create_file_data,
|
||||
extracted_file_data=extracted_file_data,
|
||||
):
|
||||
## 1. If jsonl, check if there's a model name
|
||||
file_content = self._get_content_from_openai_file(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
Transformation for Calling Google models in their native format.
|
||||
"""
|
||||
from typing import Literal, Optional, Union
|
||||
from typing import Dict, Literal, Optional, Union
|
||||
|
||||
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
@@ -11,20 +11,20 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
|
||||
"""
|
||||
Configuration for calling Google models in their native format.
|
||||
"""
|
||||
|
||||
HEADER_NAME = "Authorization"
|
||||
BEARER_PREFIX = "Bearer"
|
||||
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> Literal["gemini", "vertex_ai"]:
|
||||
return "vertex_ai"
|
||||
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
headers: Optional[dict],
|
||||
model: str,
|
||||
litellm_params: Optional[Union[GenericLiteLLMParams, dict]]
|
||||
litellm_params: Optional[Union[GenericLiteLLMParams, dict]],
|
||||
) -> dict:
|
||||
default_headers = {
|
||||
"Content-Type": "application/json",
|
||||
@@ -36,4 +36,65 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
|
||||
default_headers.update(headers)
|
||||
|
||||
return default_headers
|
||||
|
||||
|
||||
def _camel_to_snake(self, camel_str: str) -> str:
|
||||
"""Convert camelCase to snake_case"""
|
||||
import re
|
||||
|
||||
return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
|
||||
|
||||
def map_generate_content_optional_params(
|
||||
self,
|
||||
generate_content_config_dict,
|
||||
model: str,
|
||||
):
|
||||
"""
|
||||
Map Google GenAI parameters to provider-specific format.
|
||||
|
||||
Args:
|
||||
generate_content_optional_params: Optional parameters for generate content
|
||||
model: The model name
|
||||
|
||||
Returns:
|
||||
Mapped parameters for the provider
|
||||
"""
|
||||
from litellm.types.google_genai.main import GenerateContentConfigDict
|
||||
|
||||
_generate_content_config_dict = GenerateContentConfigDict()
|
||||
|
||||
for param, value in generate_content_config_dict.items():
|
||||
camel_case_key = self._camel_to_snake(param)
|
||||
_generate_content_config_dict[camel_case_key] = value
|
||||
return dict(_generate_content_config_dict)
|
||||
|
||||
def transform_generate_content_request(
|
||||
self,
|
||||
model: str,
|
||||
contents: any,
|
||||
tools: Optional[any],
|
||||
generate_content_config_dict: Dict,
|
||||
system_instruction: Optional[any] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Transform the generate content request for Vertex AI.
|
||||
Since Vertex AI natively supports Google GenAI format, we can pass most fields directly.
|
||||
"""
|
||||
# Build the request in Google GenAI format that Vertex AI expects
|
||||
result = {
|
||||
"model": model,
|
||||
"contents": contents,
|
||||
}
|
||||
|
||||
# Add tools if provided
|
||||
if tools:
|
||||
result["tools"] = tools
|
||||
|
||||
# Add systemInstruction if provided
|
||||
if system_instruction:
|
||||
result["systemInstruction"] = system_instruction
|
||||
|
||||
# Handle generationConfig - Vertex AI expects it in the same format
|
||||
if generate_content_config_dict:
|
||||
result["generationConfig"] = generate_content_config_dict
|
||||
|
||||
return result
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from typing import Optional, TypedDict, Union
|
||||
from typing import Optional, Union
|
||||
|
||||
import httpx
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
||||
@@ -3,7 +3,9 @@ Types for Vertex Embeddings Requests
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import List, Optional, TypedDict, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
||||
class TaskType(str, Enum):
|
||||
|
||||
+99
-22
@@ -150,9 +150,9 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from .llms.custom_llm import CustomLLM, custom_chat_llm_router
|
||||
from .llms.databricks.embed.handler import DatabricksEmbeddingHandler
|
||||
from .llms.deprecated_providers import aleph_alpha, palm
|
||||
from .llms.gemini.common_utils import get_api_key_from_env
|
||||
from .llms.groq.chat.handler import GroqChatCompletion
|
||||
from .llms.heroku.chat.transformation import HerokuChatConfig
|
||||
from .llms.gemini.common_utils import get_api_key_from_env
|
||||
from .llms.huggingface.embedding.handler import HuggingFaceEmbedding
|
||||
from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
|
||||
from .llms.oci.chat.transformation import OCIChatConfig
|
||||
@@ -164,6 +164,7 @@ from .llms.openai.openai import OpenAIChatCompletion
|
||||
from .llms.openai.transcriptions.handler import OpenAIAudioTranscription
|
||||
from .llms.openai_like.chat.handler import OpenAILikeChatHandler
|
||||
from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler
|
||||
from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
|
||||
from .llms.petals.completion import handler as petals_handler
|
||||
from .llms.predibase.chat.handler import PredibaseChatCompletion
|
||||
from .llms.replicate.chat.handler import completion as replicate_chat_completion
|
||||
@@ -259,6 +260,7 @@ sagemaker_chat_completion = SagemakerChatHandler()
|
||||
bytez_transformation = BytezChatConfig()
|
||||
heroku_transformation = HerokuChatConfig()
|
||||
oci_transformation = OCIChatConfig()
|
||||
ovhcloud_transformation = OVHCloudChatConfig()
|
||||
####### COMPLETION ENDPOINTS ################
|
||||
|
||||
|
||||
@@ -358,7 +360,9 @@ async def acompletion(
|
||||
logprobs: Optional[bool] = None,
|
||||
top_logprobs: Optional[int] = None,
|
||||
deployment_id=None,
|
||||
reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
|
||||
reasoning_effort: Optional[
|
||||
Literal["none", "minimal", "low", "medium", "high", "default"]
|
||||
] = None,
|
||||
safety_identifier: Optional[str] = None,
|
||||
# set api_base, api_version, api_key
|
||||
base_url: Optional[str] = None,
|
||||
@@ -504,7 +508,9 @@ async def acompletion(
|
||||
}
|
||||
if custom_llm_provider is None:
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None)
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=completion_kwargs.get("base_url", None),
|
||||
)
|
||||
|
||||
fallbacks = fallbacks or litellm.model_fallbacks
|
||||
@@ -899,7 +905,9 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
logit_bias: Optional[dict] = None,
|
||||
user: Optional[str] = None,
|
||||
# openai v1.0+ new params
|
||||
reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
|
||||
reasoning_effort: Optional[
|
||||
Literal["none", "minimal", "low", "medium", "high", "default"]
|
||||
] = None,
|
||||
response_format: Optional[Union[dict, Type[BaseModel]]] = None,
|
||||
seed: Optional[int] = None,
|
||||
tools: Optional[List] = None,
|
||||
@@ -1116,10 +1124,12 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
)
|
||||
|
||||
if provider_specific_header is not None:
|
||||
headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers(
|
||||
provider_specific_header=provider_specific_header,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
))
|
||||
headers.update(
|
||||
ProviderSpecificHeaderUtils.get_provider_specific_headers(
|
||||
provider_specific_header=provider_specific_header,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
|
||||
if model_response is not None and hasattr(model_response, "_hidden_params"):
|
||||
model_response._hidden_params["custom_llm_provider"] = custom_llm_provider
|
||||
@@ -1325,6 +1335,7 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
azure_scope=kwargs.get("azure_scope"),
|
||||
max_retries=max_retries,
|
||||
timeout=timeout,
|
||||
litellm_request_debug=kwargs.get("litellm_request_debug", False),
|
||||
)
|
||||
cast(LiteLLMLoggingObj, logging).update_environment_variables(
|
||||
model=model,
|
||||
@@ -2713,9 +2724,7 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
)
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or get_secret("VERCEL_AI_GATEWAY_API_KEY")
|
||||
api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
|
||||
)
|
||||
|
||||
vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
|
||||
@@ -2731,7 +2740,7 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
vercel_headers.update(_headers)
|
||||
|
||||
headers = vercel_headers
|
||||
|
||||
|
||||
## Load Config
|
||||
config = litellm.VercelAIGatewayConfig.get_config()
|
||||
for k, v in config.items():
|
||||
@@ -3499,6 +3508,42 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
|
||||
pass
|
||||
|
||||
elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models:
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.ovhcloud_key
|
||||
or get_secret_str("OVHCLOUD_API_KEY")
|
||||
or litellm.api_key
|
||||
)
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret_str("OVHCLOUD_API_BASE")
|
||||
or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
|
||||
)
|
||||
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout, # type: ignore
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=encoding,
|
||||
stream=stream,
|
||||
provider_config=ovhcloud_transformation,
|
||||
)
|
||||
|
||||
pass
|
||||
|
||||
elif custom_llm_provider == "custom":
|
||||
url = litellm.api_base or api_base or ""
|
||||
if url is None or url == "":
|
||||
@@ -3713,7 +3758,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
|
||||
func_with_context = partial(ctx.run, func)
|
||||
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None)
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=kwargs.get("api_base", None),
|
||||
)
|
||||
|
||||
# Await normally
|
||||
@@ -4586,6 +4633,28 @@ def embedding( # noqa: PLR0915
|
||||
aembedding=aembedding,
|
||||
headers=headers,
|
||||
)
|
||||
elif custom_llm_provider == "ovhcloud":
|
||||
api_key = api_key or litellm.api_key or get_secret_str("OVHCLOUD_API_KEY")
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret_str("OVHCLOUD_API_BASE")
|
||||
or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
|
||||
)
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
timeout=timeout,
|
||||
model_response=EmbeddingResponse(),
|
||||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
litellm_params={},
|
||||
)
|
||||
elif custom_llm_provider in litellm._custom_providers:
|
||||
custom_handler: Optional[CustomLLM] = None
|
||||
for item in litellm.custom_provider_map:
|
||||
@@ -5280,7 +5349,10 @@ def transcription(
|
||||
model_response = litellm.utils.TranscriptionResponse()
|
||||
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
) # type: ignore
|
||||
|
||||
if dynamic_api_key is not None:
|
||||
@@ -5296,6 +5368,7 @@ def transcription(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**non_default_params,
|
||||
)
|
||||
|
||||
litellm_params_dict = get_litellm_params(**kwargs)
|
||||
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
@@ -5360,9 +5433,8 @@ def transcription(
|
||||
max_retries=max_retries,
|
||||
litellm_params=litellm_params_dict,
|
||||
)
|
||||
elif (
|
||||
custom_llm_provider == "openai"
|
||||
or custom_llm_provider in litellm.openai_compatible_providers
|
||||
elif custom_llm_provider == "openai" or (
|
||||
custom_llm_provider in litellm.openai_compatible_providers
|
||||
):
|
||||
api_base = (
|
||||
api_base
|
||||
@@ -5377,6 +5449,7 @@ def transcription(
|
||||
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
|
||||
)
|
||||
# set API KEY
|
||||
|
||||
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") # type: ignore
|
||||
response = openai_audio_transcriptions.audio_transcriptions(
|
||||
model=model,
|
||||
@@ -5393,10 +5466,7 @@ def transcription(
|
||||
provider_config=provider_config,
|
||||
litellm_params=litellm_params_dict,
|
||||
)
|
||||
elif custom_llm_provider in [
|
||||
LlmProviders.DEEPGRAM.value,
|
||||
LlmProviders.ELEVENLABS.value,
|
||||
]:
|
||||
elif provider_config is not None:
|
||||
response = base_llm_http_handler.audio_transcriptions(
|
||||
model=model,
|
||||
audio_file=file,
|
||||
@@ -5802,7 +5872,14 @@ async def ahealth_check(
|
||||
input=input or ["test"],
|
||||
),
|
||||
"audio_speech": lambda: litellm.aspeech(
|
||||
**{**_filter_model_params(model_params), **({"voice": "alloy"} if "voice" not in _filter_model_params(model_params) else {})},
|
||||
**{
|
||||
**_filter_model_params(model_params),
|
||||
**(
|
||||
{"voice": "alloy"}
|
||||
if "voice" not in _filter_model_params(model_params)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
input=prompt or "test",
|
||||
),
|
||||
"audio_transcription": lambda: litellm.atranscription(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -241,6 +241,9 @@ class MCPServerManager:
|
||||
transport=server_config.get("transport", MCPTransport.http),
|
||||
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
|
||||
auth_type=server_config.get("auth_type", None),
|
||||
authentication_token=server_config.get(
|
||||
"authentication_token", server_config.get("auth_value", None)
|
||||
),
|
||||
mcp_info=mcp_info,
|
||||
access_groups=server_config.get("access_groups", None),
|
||||
)
|
||||
@@ -716,8 +719,8 @@ class MCPServerManager:
|
||||
tasks = []
|
||||
if proxy_logging_obj:
|
||||
# Create synthetic LLM data for during hook processing
|
||||
from litellm.types.mcp import MCPDuringCallRequestObject
|
||||
from litellm.types.llms.base import HiddenParams
|
||||
from litellm.types.mcp import MCPDuringCallRequestObject
|
||||
|
||||
request_obj = MCPDuringCallRequestObject(
|
||||
tool_name=name,
|
||||
|
||||
@@ -215,9 +215,9 @@ if MCP_AVAILABLE:
|
||||
"""
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
from litellm.proxy.proxy_server import proxy_config
|
||||
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
||||
|
||||
# Validate arguments
|
||||
user_api_key_auth, mcp_auth_header, _, mcp_server_auth_headers, mcp_protocol_version = get_auth_context()
|
||||
@@ -279,33 +279,15 @@ if MCP_AVAILABLE:
|
||||
############ Helper Functions ##########################
|
||||
########################################################
|
||||
|
||||
async def _get_tools_from_mcp_servers(
|
||||
user_api_key_auth: Optional[UserAPIKeyAuth],
|
||||
mcp_auth_header: Optional[str],
|
||||
async def _get_allowed_mcp_servers_from_mcp_server_names(
|
||||
mcp_servers: Optional[List[str]],
|
||||
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
|
||||
mcp_protocol_version: Optional[str] = None,
|
||||
) -> List[MCPTool]:
|
||||
allowed_mcp_servers: List[str],
|
||||
) -> List[str]:
|
||||
"""
|
||||
Helper method to fetch tools from MCP servers based on server filtering criteria.
|
||||
|
||||
Args:
|
||||
user_api_key_auth: User authentication info for access control
|
||||
mcp_auth_header: Optional auth header for MCP server (deprecated)
|
||||
mcp_servers: Optional list of server names/aliases to filter by
|
||||
mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
|
||||
|
||||
Returns:
|
||||
List[MCPTool]: Combined list of tools from filtered servers
|
||||
Get the filtered MCP servers from the MCP server names
|
||||
"""
|
||||
if not MCP_AVAILABLE:
|
||||
return []
|
||||
|
||||
# Get allowed MCP servers based on user permissions
|
||||
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
|
||||
|
||||
filtered_server_ids = set()
|
||||
|
||||
from typing import Set
|
||||
filtered_server_ids: Set[str] = set()
|
||||
# Filter servers based on mcp_servers parameter if provided
|
||||
if mcp_servers is not None:
|
||||
for server_or_group in mcp_servers:
|
||||
@@ -336,6 +318,40 @@ if MCP_AVAILABLE:
|
||||
|
||||
if filtered_server_ids:
|
||||
allowed_mcp_servers = list(filtered_server_ids)
|
||||
|
||||
return allowed_mcp_servers
|
||||
|
||||
async def _get_tools_from_mcp_servers(
|
||||
user_api_key_auth: Optional[UserAPIKeyAuth],
|
||||
mcp_auth_header: Optional[str],
|
||||
mcp_servers: Optional[List[str]],
|
||||
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
|
||||
mcp_protocol_version: Optional[str] = None,
|
||||
) -> List[MCPTool]:
|
||||
"""
|
||||
Helper method to fetch tools from MCP servers based on server filtering criteria.
|
||||
|
||||
Args:
|
||||
user_api_key_auth: User authentication info for access control
|
||||
mcp_auth_header: Optional auth header for MCP server (deprecated)
|
||||
mcp_servers: Optional list of server names/aliases to filter by
|
||||
mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
|
||||
|
||||
Returns:
|
||||
List[MCPTool]: Combined list of tools from filtered servers
|
||||
"""
|
||||
if not MCP_AVAILABLE:
|
||||
return []
|
||||
|
||||
# Get allowed MCP servers based on user permissions
|
||||
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
|
||||
|
||||
if mcp_servers is not None:
|
||||
allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
|
||||
mcp_servers=mcp_servers,
|
||||
allowed_mcp_servers=allowed_mcp_servers,
|
||||
)
|
||||
|
||||
|
||||
# Get tools from each allowed server
|
||||
all_tools = []
|
||||
@@ -556,20 +572,25 @@ if MCP_AVAILABLE:
|
||||
except Exception as e:
|
||||
return [TextContent(text=f"Error: {str(e)}", type="text")]
|
||||
|
||||
async def extract_mcp_auth_context(scope, path):
|
||||
def _get_mcp_servers_in_path(path: str) -> Optional[List[str]]:
|
||||
"""
|
||||
Extracts mcp_servers from the path and processes the MCP request for auth context.
|
||||
Returns: (user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers)
|
||||
Get the MCP servers from the path
|
||||
"""
|
||||
import re
|
||||
|
||||
mcp_servers_from_path = None
|
||||
mcp_servers_from_path: Optional[List[str]] = None
|
||||
mcp_path_match = re.match(r"^/mcp/([^/]+)(/.*)?$", path)
|
||||
if mcp_path_match:
|
||||
mcp_servers_str = mcp_path_match.group(1)
|
||||
if mcp_servers_str:
|
||||
mcp_servers_from_path = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
|
||||
return mcp_servers_from_path
|
||||
|
||||
async def extract_mcp_auth_context(scope, path):
|
||||
"""
|
||||
Extracts mcp_servers from the path and processes the MCP request for auth context.
|
||||
Returns: (user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers)
|
||||
"""
|
||||
mcp_servers_from_path = _get_mcp_servers_in_path(path)
|
||||
if mcp_servers_from_path is not None:
|
||||
(
|
||||
user_api_key_auth,
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+4
-4
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{6580:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=6580)}),_N_E=n.O()}]);
|
||||
@@ -0,0 +1 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);
|
||||
+1
-1
@@ -1 +1 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
|
||||
+1
-1
@@ -1 +1 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{58538:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=58538)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{64563:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=64563)}),_N_E=e.O()}]);
|
||||
-1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user