mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-09 16:24:38 +00:00
LiteLLM Stable release notes (#10919)
* docs(index/v1.70.1-stable): style improvements * style: add style improvements to docs * docs: cleanup docs * docs: more style improvements * docs: style improvements * docs(gemini/realtime): add docs on realtime api via Google AI Studio * docs: add openai example to anthropic web search docs * docs: add missing doc links * docs: doc cleanup * docs: add more doc links * fix: cleanup * docs: add docker information * docs: update doc links * docs: add demo instance details to doc s
This commit is contained in:
Krish Dholakia
@@ -225,36 +225,6 @@ response = embedding(
|
||||
| text-embedding-3-large | `embedding('text-embedding-3-large', input)` | `os.environ['OPENAI_API_KEY']` |
|
||||
| text-embedding-ada-002 | `embedding('text-embedding-ada-002', input)` | `os.environ['OPENAI_API_KEY']` |
|
||||
|
||||
## Azure OpenAI Embedding Models
|
||||
|
||||
### API keys
|
||||
This can be set as env variables or passed as **params to litellm.embedding()**
|
||||
```python
|
||||
import os
|
||||
os.environ['AZURE_API_KEY'] =
|
||||
os.environ['AZURE_API_BASE'] =
|
||||
os.environ['AZURE_API_VERSION'] =
|
||||
```
|
||||
|
||||
### Usage
|
||||
```python
|
||||
from litellm import embedding
|
||||
response = embedding(
|
||||
model="azure/<your deployment name>",
|
||||
input=["good morning from litellm"],
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
| Model Name | Function Call |
|
||||
|----------------------|---------------------------------------------|
|
||||
| text-embedding-ada-002 | `embedding(model="azure/<your deployment name>", input=input)` |
|
||||
|
||||
h/t to [Mikko](https://www.linkedin.com/in/mikkolehtimaki/) for this integration
|
||||
|
||||
## OpenAI Compatible Embedding Models
|
||||
Use this for calling `/embedding` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Image from '@theme/IdealImage';
|
||||
|
||||
# Phoenix OSS
|
||||
# Arize Phoenix OSS
|
||||
|
||||
Open source tracing and evaluation platform
|
||||
|
||||
|
||||
@@ -847,13 +847,50 @@ curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
<TabItem value="web_search" label="Web Search">
|
||||
|
||||
:::info
|
||||
|
||||
Unified web search (same param across OpenAI + Anthropic) coming soon!
|
||||
Live from v1.70.1+
|
||||
:::
|
||||
|
||||
LiteLLM maps OpenAI's `search_context_size` param to Anthropic's `max_uses` param.
|
||||
|
||||
| OpenAI | Anthropic |
|
||||
| --- | --- |
|
||||
| Low | 1 |
|
||||
| Medium | 5 |
|
||||
| High | 10 |
|
||||
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai" label="OpenAI Format">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
model = "claude-3-5-sonnet-20241022"
|
||||
messages = [{"role": "user", "content": "What's the weather like today?"}]
|
||||
|
||||
resp = completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
web_search_options={
|
||||
"search_context_size": "medium",
|
||||
"user_location": {
|
||||
"type": "approximate",
|
||||
"approximate": {
|
||||
"city": "San Francisco",
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
print(resp)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="anthropic" label="Anthropic Format">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
@@ -873,8 +910,11 @@ resp = completion(
|
||||
|
||||
print(resp)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
@@ -894,22 +934,56 @@ litellm --config /path/to/config.yaml
|
||||
|
||||
3. Test it!
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai" label="OpenAI Format">
|
||||
|
||||
|
||||
```bash
|
||||
curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_KEY" \
|
||||
-d '{
|
||||
"model": "claude-3-5-sonnet-latest",
|
||||
"messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}],
|
||||
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}]
|
||||
"messages": [{"role": "user", "content": "What's the weather like today?"}],
|
||||
"web_search_options": {
|
||||
"search_context_size": "medium",
|
||||
"user_location": {
|
||||
"type": "approximate",
|
||||
"approximate": {
|
||||
"city": "San Francisco",
|
||||
},
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="anthropic" label="Anthropic Format">
|
||||
|
||||
```bash
|
||||
curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_KEY" \
|
||||
-d '{
|
||||
"model": "claude-3-5-sonnet-latest",
|
||||
"messages": [{"role": "user", "content": "What's the weather like today?"}],
|
||||
"tools": [{
|
||||
"type": "web_search_20250305",
|
||||
"name": "web_search",
|
||||
"max_uses": 5
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
|
||||
## Usage - Vision
|
||||
|
||||
```python
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
|
||||
|-------|-------|
|
||||
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
|
||||
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#azure-o-series-models) |
|
||||
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](../embedding/supported_embedding#azure-openai-embedding-models), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
|
||||
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
|
||||
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
|
||||
|
||||
## API Keys, Params
|
||||
@@ -0,0 +1,93 @@
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Azure OpenAI Embeddings
|
||||
|
||||
### API keys
|
||||
This can be set as env variables or passed as **params to litellm.embedding()**
|
||||
```python
|
||||
import os
|
||||
os.environ['AZURE_API_KEY'] =
|
||||
os.environ['AZURE_API_BASE'] =
|
||||
os.environ['AZURE_API_VERSION'] =
|
||||
```
|
||||
|
||||
### Usage
|
||||
```python
|
||||
from litellm import embedding
|
||||
response = embedding(
|
||||
model="azure/<your deployment name>",
|
||||
input=["good morning from litellm"],
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
| Model Name | Function Call |
|
||||
|----------------------|---------------------------------------------|
|
||||
| text-embedding-ada-002 | `embedding(model="azure/<your deployment name>", input=input)` |
|
||||
|
||||
h/t to [Mikko](https://www.linkedin.com/in/mikkolehtimaki/) for this integration
|
||||
|
||||
|
||||
## **Usage - LiteLLM Proxy Server**
|
||||
|
||||
Here's how to call Azure OpenAI models with the LiteLLM Proxy Server
|
||||
|
||||
### 1. Save key in your environment
|
||||
|
||||
```bash
|
||||
export AZURE_API_KEY=""
|
||||
```
|
||||
|
||||
### 2. Start the proxy
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: text-embedding-ada-002
|
||||
litellm_params:
|
||||
model: azure/my-deployment-name
|
||||
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
|
||||
api_version: "2023-05-15"
|
||||
api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env.
|
||||
```
|
||||
|
||||
### 3. Test it
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="Curl" label="Curl Request">
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/embeddings' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data ' {
|
||||
"model": "text-embedding-ada-002",
|
||||
"input": ["write a litellm poem"]
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="openai" label="OpenAI v1.0.0+">
|
||||
|
||||
```python
|
||||
import openai
|
||||
from openai import OpenAI
|
||||
|
||||
# set base_url to your proxy server
|
||||
# set api_key to send to proxy server
|
||||
client = OpenAI(api_key="<proxy-api-key>", base_url="http://0.0.0.0:4000")
|
||||
|
||||
response = client.embeddings.create(
|
||||
input=["hello from litellm"],
|
||||
model="text-embedding-ada-002"
|
||||
)
|
||||
|
||||
print(response)
|
||||
|
||||
```
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# Gemini Realtime API - Google AI Studio
|
||||
|
||||
| Feature | Description | Comments |
|
||||
| --- | --- | --- |
|
||||
| Proxy | ✅ | |
|
||||
| SDK | ⌛️ | Experimental access via `litellm._arealtime`. |
|
||||
|
||||
|
||||
## Proxy Usage
|
||||
|
||||
### Add model to config
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: "gemini-2.0-flash"
|
||||
litellm_params:
|
||||
model: gemini/gemini-2.0-flash-live-001
|
||||
model_info:
|
||||
mode: realtime
|
||||
```
|
||||
|
||||
### Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:8000
|
||||
```
|
||||
|
||||
### Test
|
||||
|
||||
Run this script using node - `node test.js`
|
||||
|
||||
```js
|
||||
// test.js
|
||||
const WebSocket = require("ws");
|
||||
|
||||
const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gemini-2.0-flash";
|
||||
|
||||
const ws = new WebSocket(url, {
|
||||
headers: {
|
||||
"api-key": `${LITELLM_API_KEY}`,
|
||||
"OpenAI-Beta": "realtime=v1",
|
||||
},
|
||||
});
|
||||
|
||||
ws.on("open", function open() {
|
||||
console.log("Connected to server.");
|
||||
ws.send(JSON.stringify({
|
||||
type: "response.create",
|
||||
response: {
|
||||
modalities: ["text"],
|
||||
instructions: "Please assist the user.",
|
||||
}
|
||||
}));
|
||||
});
|
||||
|
||||
ws.on("message", function incoming(message) {
|
||||
console.log(JSON.parse(message.toString()));
|
||||
});
|
||||
|
||||
ws.on("error", function handleError(error) {
|
||||
console.error("Error: ", error);
|
||||
});
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- Does not support audio transcription.
|
||||
- Does not support tool calling
|
||||
|
||||
## Supported OpenAI Realtime Events
|
||||
|
||||
- `session.created`
|
||||
- `response.created`
|
||||
- `response.output_item.added`
|
||||
- `conversation.item.created`
|
||||
- `response.content_part.added`
|
||||
- `response.text.delta`
|
||||
- `response.audio.delta`
|
||||
- `response.text.done`
|
||||
- `response.audio.done`
|
||||
- `response.content_part.done`
|
||||
- `response.output_item.done`
|
||||
- `response.done`
|
||||
|
||||
|
||||
|
||||
## [Supported Session Params](https://github.com/BerriAI/litellm/blob/e87b536d038f77c2a2206fd7433e275c487179ee/litellm/llms/gemini/realtime/transformation.py#L155)
|
||||
|
||||
## More Examples
|
||||
### [Gemini Realtime API with Audio Input/Output](../../../docs/tutorials/gemini_realtime_with_audio)
|
||||
@@ -163,9 +163,11 @@ LiteLLM Proxy works seamlessly with Langchain, LlamaIndex, OpenAI JS, Anthropic
|
||||
|
||||
[Learn how to use LiteLLM proxy with these libraries →](../proxy/user_keys)
|
||||
|
||||
## Flags to send requests to litellm proxy
|
||||
## Send all SDK requests to LiteLLM Proxy
|
||||
|
||||
Use the following options to route all requests through your LiteLLM proxy, regardless of the model specified.
|
||||
Use this when calling LiteLLM Proxy from any library / codebase already using the LiteLLM SDK.
|
||||
|
||||
These flags will route all requests through your LiteLLM proxy, regardless of the model specified.
|
||||
|
||||
When enabled, requests will use `LITELLM_PROXY_API_BASE` with `LITELLM_PROXY_API_KEY` as the authentication.
|
||||
|
||||
|
||||
@@ -10,10 +10,19 @@ https://docs.api.nvidia.com/nim/reference/
|
||||
|
||||
:::
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Nvidia NIM is a platform that provides a simple API for deploying and using AI models. LiteLLM supports all models from [Nvidia NIM](https://developer.nvidia.com/nim/) |
|
||||
| Provider Route on LiteLLM | `nvidia_nim/` |
|
||||
| Provider Doc | [Nvidia NIM Docs ↗](https://developer.nvidia.com/nim/) |
|
||||
| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ |
|
||||
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings` |
|
||||
|
||||
## API Key
|
||||
```python
|
||||
# env variable
|
||||
os.environ['NVIDIA_NIM_API_KEY']
|
||||
os.environ['NVIDIA_NIM_API_KEY'] = ""
|
||||
os.environ['NVIDIA_NIM_API_BASE'] = "" # [OPTIONAL] - default is https://integrate.api.nvidia.com/v1/
|
||||
```
|
||||
|
||||
## Sample Usage
|
||||
@@ -100,6 +109,7 @@ Here's how to call an Nvidia NIM Endpoint with the LiteLLM Proxy Server
|
||||
litellm_params:
|
||||
model: nvidia_nim/<your-model-name> # add nvidia_nim/ prefix to route as Nvidia NIM provider
|
||||
api_key: api-key # api key to send your model
|
||||
# api_base: "" # [OPTIONAL] - default is https://integrate.api.nvidia.com/v1/
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -60,6 +60,8 @@ Inherits from `StandardLoggingUserAPIKeyMetadata` and adds:
|
||||
| `requester_ip_address` | `Optional[str]` | Requester's IP address |
|
||||
| `requester_metadata` | `Optional[dict]` | Additional requester metadata |
|
||||
| `vector_store_request_metadata` | `Optional[List[StandardLoggingVectorStoreRequest]]` | Vector store request metadata |
|
||||
| `requester_custom_headers` | Dict[str, str] | Any custom (`x-`) headers sent by the client to the proxy. |
|
||||
| `guardrail_information` | `Optional[StandardLoggingGuardrailInformation]` | Guardrail information |
|
||||
|
||||
|
||||
## StandardLoggingVectorStoreRequest
|
||||
@@ -127,4 +129,20 @@ Inherits from `StandardLoggingUserAPIKeyMetadata` and adds:
|
||||
|
||||
A literal type with two possible values:
|
||||
- `"success"`
|
||||
- `"failure"`
|
||||
- `"failure"`
|
||||
|
||||
## StandardLoggingGuardrailInformation
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `guardrail_name` | `Optional[str]` | Guardrail name |
|
||||
| `guardrail_mode` | `Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks]]]` | Guardrail mode |
|
||||
| `guardrail_request` | `Optional[dict]` | Guardrail request |
|
||||
| `guardrail_response` | `Optional[Union[dict, str, List[dict]]]` | Guardrail response |
|
||||
| `guardrail_status` | `Literal["success", "failure"]` | Guardrail status |
|
||||
| `start_time` | `Optional[float]` | Start time of the guardrail |
|
||||
| `end_time` | `Optional[float]` | End time of the guardrail |
|
||||
| `duration` | `Optional[float]` | Duration of the guardrail in seconds |
|
||||
| `masked_entity_count` | `Optional[Dict[str, int]]` | Count of masked entities |
|
||||
|
||||
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
# LiteLLM Proxy Client
|
||||
|
||||
> **See also:** [LiteLLM Proxy CLI Management Tool](./management_cli.md)
|
||||
|
||||
A Python client library for interacting with the LiteLLM proxy server. This client provides a clean, typed interface for managing models, keys, credentials, and making chat completions.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install litellm
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python
|
||||
from litellm.proxy.client import Client
|
||||
|
||||
# Initialize the client
|
||||
client = Client(
|
||||
base_url="http://localhost:4000", # Your LiteLLM proxy server URL
|
||||
api_key="sk-api-key" # Optional: API key for authentication
|
||||
)
|
||||
|
||||
# Make a chat completion request
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
]
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
The client is organized into several resource clients for different functionality:
|
||||
|
||||
- `chat`: Chat completions
|
||||
- `models`: Model management
|
||||
- `model_groups`: Model group management
|
||||
- `keys`: API key management
|
||||
- `credentials`: Credential management
|
||||
- `http`: Low-level HTTP client
|
||||
|
||||
## Chat Completions
|
||||
|
||||
Make chat completion requests to your LiteLLM proxy:
|
||||
|
||||
```python
|
||||
# Basic chat completion
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What's the capital of France?"}
|
||||
]
|
||||
)
|
||||
|
||||
# Stream responses
|
||||
for chunk in client.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "Tell me a story"}],
|
||||
stream=True
|
||||
):
|
||||
print(chunk.choices[0].delta.content or "", end="")
|
||||
```
|
||||
|
||||
## Model Management
|
||||
|
||||
Manage available models on your proxy:
|
||||
|
||||
```python
|
||||
# List available models
|
||||
models = client.models.list()
|
||||
|
||||
# Add a new model
|
||||
client.models.add(
|
||||
model_name="gpt-4",
|
||||
litellm_params={
|
||||
"api_key": "your-openai-key",
|
||||
"api_base": "https://api.openai.com/v1"
|
||||
}
|
||||
)
|
||||
|
||||
# Delete a model
|
||||
client.models.delete(model_name="gpt-4")
|
||||
```
|
||||
|
||||
## API Key Management
|
||||
|
||||
Manage virtual API keys:
|
||||
|
||||
```python
|
||||
# Generate a new API key
|
||||
key = client.keys.generate(
|
||||
models=["gpt-4", "gpt-3.5-turbo"],
|
||||
aliases={"gpt4": "gpt-4"},
|
||||
duration="24h",
|
||||
key_alias="my-key",
|
||||
team_id="team123"
|
||||
)
|
||||
|
||||
# List all keys
|
||||
keys = client.keys.list(
|
||||
page=1,
|
||||
size=10,
|
||||
return_full_object=True
|
||||
)
|
||||
|
||||
# Delete keys
|
||||
client.keys.delete(
|
||||
keys=["sk-key1", "sk-key2"],
|
||||
key_aliases=["alias1", "alias2"]
|
||||
)
|
||||
```
|
||||
|
||||
## Credential Management
|
||||
|
||||
Manage model credentials:
|
||||
|
||||
```python
|
||||
# Create new credentials
|
||||
client.credentials.create(
|
||||
credential_name="azure1",
|
||||
credential_info={"api_type": "azure"},
|
||||
credential_values={
|
||||
"api_key": "your-azure-key",
|
||||
"api_base": "https://example.azure.openai.com"
|
||||
}
|
||||
)
|
||||
|
||||
# List all credentials
|
||||
credentials = client.credentials.list()
|
||||
|
||||
# Get a specific credential
|
||||
credential = client.credentials.get(credential_name="azure1")
|
||||
|
||||
# Delete credentials
|
||||
client.credentials.delete(credential_name="azure1")
|
||||
```
|
||||
|
||||
## Model Groups
|
||||
|
||||
Manage model groups for load balancing and fallbacks:
|
||||
|
||||
```python
|
||||
# Create a model group
|
||||
client.model_groups.create(
|
||||
name="gpt4-group",
|
||||
models=[
|
||||
{"model_name": "gpt-4", "litellm_params": {"api_key": "key1"}},
|
||||
{"model_name": "gpt-4-backup", "litellm_params": {"api_key": "key2"}}
|
||||
]
|
||||
)
|
||||
|
||||
# List model groups
|
||||
groups = client.model_groups.list()
|
||||
|
||||
# Delete a model group
|
||||
client.model_groups.delete(name="gpt4-group")
|
||||
```
|
||||
|
||||
## Low-Level HTTP Client
|
||||
|
||||
The client provides access to a low-level HTTP client for making direct requests
|
||||
to the LiteLLM proxy server. This is useful when you need more control or when
|
||||
working with endpoints that don't yet have a high-level interface.
|
||||
|
||||
```python
|
||||
# Access the HTTP client
|
||||
client = Client(
|
||||
base_url="http://localhost:4000",
|
||||
api_key="sk-api-key"
|
||||
)
|
||||
|
||||
# Make a custom request
|
||||
response = client.http.request(
|
||||
method="POST",
|
||||
uri="/health/test_connection",
|
||||
json={
|
||||
"litellm_params": {
|
||||
"model": "gpt-4",
|
||||
"api_key": "your-api-key",
|
||||
"api_base": "https://api.openai.com/v1"
|
||||
},
|
||||
"mode": "chat"
|
||||
}
|
||||
)
|
||||
|
||||
# The response is automatically parsed from JSON
|
||||
print(response)
|
||||
```
|
||||
|
||||
### HTTP Client Features
|
||||
|
||||
- Automatic URL handling (handles trailing/leading slashes)
|
||||
- Built-in authentication (adds Bearer token if `api_key` is provided)
|
||||
- JSON request/response handling
|
||||
- Configurable timeout (default: 30 seconds)
|
||||
- Comprehensive error handling
|
||||
- Support for custom headers and request parameters
|
||||
|
||||
### HTTP Client `request` method parameters
|
||||
|
||||
- `method`: HTTP method (GET, POST, PUT, DELETE, etc.)
|
||||
- `uri`: URI path (will be appended to base_url)
|
||||
- `data`: (optional) Data to send in the request body
|
||||
- `json`: (optional) JSON data to send in the request body
|
||||
- `headers`: (optional) Custom HTTP headers
|
||||
- Additional keyword arguments are passed to the underlying requests library
|
||||
|
||||
## Error Handling
|
||||
|
||||
The client provides clear error handling with custom exceptions:
|
||||
|
||||
```python
|
||||
from litellm.proxy.client.exceptions import UnauthorizedError
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "Hello"}]
|
||||
)
|
||||
except UnauthorizedError as e:
|
||||
print("Authentication failed:", e)
|
||||
except Exception as e:
|
||||
print("Request failed:", e)
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Request Customization
|
||||
|
||||
All methods support returning the raw request object for inspection or modification:
|
||||
|
||||
```python
|
||||
# Get the prepared request without sending it
|
||||
request = client.models.list(return_request=True)
|
||||
print(request.method) # GET
|
||||
print(request.url) # http://localhost:8000/models
|
||||
print(request.headers) # {'Content-Type': 'application/json', ...}
|
||||
```
|
||||
|
||||
### Pagination
|
||||
|
||||
Methods that return lists support pagination:
|
||||
|
||||
```python
|
||||
# Get the first page of keys
|
||||
page1 = client.keys.list(page=1, size=10)
|
||||
|
||||
# Get the second page
|
||||
page2 = client.keys.list(page=2, size=10)
|
||||
```
|
||||
|
||||
### Filtering
|
||||
|
||||
Many list methods support filtering:
|
||||
|
||||
```python
|
||||
# Filter keys by user and team
|
||||
keys = client.keys.list(
|
||||
user_id="user123",
|
||||
team_id="team456",
|
||||
include_team_keys=True
|
||||
)
|
||||
```
|
||||
@@ -1,21 +1,14 @@
|
||||
---
|
||||
id: spend_logs_deletion
|
||||
title: Spend Logs Deletion
|
||||
---
|
||||
# Maximum Retention Period for Spend Logs
|
||||
|
||||
# Spend Log Cleanup
|
||||
|
||||
LiteLLM stores a log for every request. Over time, these logs can grow large and slow down your database. The Spend Log Cleanup feature helps manage database size by deleting old logs automatically.
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
This walks through how to set the maximum retention period for spend logs. This helps manage database size by deleting old logs automatically.
|
||||
|
||||
### Requirements
|
||||
|
||||
- **Postgres** (for log storage)
|
||||
- **Redis** *(optional)* — required only if you're running multiple proxy instances and want to enable distributed locking
|
||||
|
||||
## Usage
|
||||
|
||||
### Setup
|
||||
|
||||
Add this to your `proxy_config.yaml` under `general_settings`:
|
||||
@@ -24,8 +17,8 @@ Add this to your `proxy_config.yaml` under `general_settings`:
|
||||
general_settings:
|
||||
maximum_spend_logs_retention_period: "7d" # Keep logs for 7 days
|
||||
|
||||
# Optional: set how frequently cleanup should run
|
||||
maximum_spend_logs_retention_interval: "1d" # Run cleanup every day
|
||||
# Optional: set how frequently cleanup should run - default is daily
|
||||
maximum_spend_logs_retention_interval: "1d" # Run cleanup daily
|
||||
|
||||
litellm_settings:
|
||||
cache: true
|
||||
@@ -48,8 +41,6 @@ How long logs should be kept before deletion. Supported formats:
|
||||
|
||||
How often the cleanup job should run. Uses the same format as above. If not set, cleanup will run every 24 hours if and only if `maximum_spend_logs_retention_period` is set.
|
||||
|
||||
---
|
||||
|
||||
## How it works
|
||||
|
||||
### Step 1. Lock Acquisition (Optional with Redis)
|
||||
@@ -65,8 +56,6 @@ If Redis is enabled, LiteLLM uses it to make sure only one instance runs the cle
|
||||
|
||||
*Working of spend log deletions*
|
||||
|
||||
---
|
||||
|
||||
### Step 2. Batch Deletion
|
||||
|
||||
Once cleanup starts:
|
||||
@@ -90,9 +79,3 @@ This would allow up to 200,000 logs to be deleted in one run.
|
||||
|
||||
|
||||
*Batch deletion of old logs*
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Spend Log Cleanup helps keep your database fast by regularly deleting old logs. It’s safe, customizable, and works well for both single-node and multi-node deployments.
|
||||
@@ -25,7 +25,7 @@ If you want a server to load balance across different LLM APIs, use our [LiteLLM
|
||||
|
||||
### Quick Start
|
||||
|
||||
Loadbalance across multiple [azure](./providers/azure.md)/[bedrock](./providers/bedrock.md)/[provider](./providers/) deployments. LiteLLM will handle retrying in different regions if a call fails.
|
||||
Loadbalance across multiple [azure](./providers/azure)/[bedrock](./providers/bedrock.md)/[provider](./providers/) deployments. LiteLLM will handle retrying in different regions if a call fails.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Call Gemini Realtime API with Audio Input/Output
|
||||
|
||||
:::info
|
||||
Requires LiteLLM Proxy v1.70.1+
|
||||
:::
|
||||
|
||||
1. Setup config.yaml for LiteLLM Proxy
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: "gemini-2.0-flash"
|
||||
litellm_params:
|
||||
model: gemini/gemini-2.0-flash-live-001
|
||||
model_info:
|
||||
mode: realtime
|
||||
```
|
||||
|
||||
2. Start LiteLLM Proxy
|
||||
|
||||
```bash
|
||||
litellm-proxy start
|
||||
```
|
||||
|
||||
3. Run test script
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
import websockets
|
||||
import json
|
||||
import base64
|
||||
from dotenv import load_dotenv
|
||||
import wave
|
||||
import base64
|
||||
import soundfile as sf
|
||||
import sounddevice as sd
|
||||
import io
|
||||
import numpy as np
|
||||
|
||||
# Load environment variables
|
||||
|
||||
OPENAI_API_KEY = "sk-1234" # Replace with your LiteLLM API key
|
||||
OPENAI_API_URL = 'ws://{PROXY_URL}/v1/realtime?model=gemini-2.0-flash' # REPLACE WITH `wss://{PROXY_URL}/v1/realtime?model=gemini-2.0-flash` for secure connection
|
||||
WAV_FILE_PATH = "/path/to/audio.wav" # Replace with your .wav file path
|
||||
|
||||
async def send_session_update(ws):
|
||||
session_update = {
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"conversation_id": "123456",
|
||||
"language": "en-US",
|
||||
"transcription_mode": "fast",
|
||||
"modalities": ["text"]
|
||||
}
|
||||
}
|
||||
await ws.send(json.dumps(session_update))
|
||||
|
||||
async def send_audio_file(ws, file_path):
|
||||
with wave.open(file_path, 'rb') as wav_file:
|
||||
chunk_size = 1024 # Adjust as needed
|
||||
while True:
|
||||
chunk = wav_file.readframes(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
base64_audio = base64.b64encode(chunk).decode('utf-8')
|
||||
audio_message = {
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64_audio
|
||||
}
|
||||
await ws.send(json.dumps(audio_message))
|
||||
await asyncio.sleep(0.1) # Add a small delay to simulate real-time streaming
|
||||
|
||||
# Send end of audio stream message
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.end"}))
|
||||
|
||||
def play_base64_audio(base64_string, sample_rate=24000, channels=1):
|
||||
# Decode the base64 string
|
||||
audio_data = base64.b64decode(base64_string)
|
||||
|
||||
# Convert to numpy array
|
||||
audio_np = np.frombuffer(audio_data, dtype=np.int16)
|
||||
|
||||
# Reshape if stereo
|
||||
if channels == 2:
|
||||
audio_np = audio_np.reshape(-1, 2)
|
||||
|
||||
# Normalize
|
||||
audio_float = audio_np.astype(np.float32) / 32768.0
|
||||
|
||||
# Play the audio
|
||||
sd.play(audio_float, sample_rate)
|
||||
sd.wait()
|
||||
|
||||
|
||||
def combine_base64_audio(base64_strings):
|
||||
# Step 1: Decode base64 strings to binary
|
||||
binary_data = [base64.b64decode(s) for s in base64_strings]
|
||||
|
||||
# Step 2: Concatenate binary data
|
||||
combined_binary = b''.join(binary_data)
|
||||
|
||||
# Step 3: Encode combined binary back to base64
|
||||
combined_base64 = base64.b64encode(combined_binary).decode('utf-8')
|
||||
|
||||
return combined_base64
|
||||
|
||||
async def listen_in_background(ws):
|
||||
combined_b64_audio_str = []
|
||||
try:
|
||||
while True:
|
||||
response = await ws.recv()
|
||||
message_json = json.loads(response)
|
||||
print(f"message_json: {message_json}")
|
||||
|
||||
if message_json['type'] == 'response.audio.delta' and message_json.get('delta'):
|
||||
play_base64_audio(message_json["delta"])
|
||||
except Exception:
|
||||
print("END OF STREAM")
|
||||
|
||||
async def main():
|
||||
async with websockets.connect(
|
||||
OPENAI_API_URL,
|
||||
additional_headers={
|
||||
"Authorization": f"Bearer {OPENAI_API_KEY}",
|
||||
"OpenAI-Beta": "realtime=v1"
|
||||
}
|
||||
) as ws:
|
||||
asyncio.create_task(listen_in_background(ws=ws))
|
||||
await send_session_update(ws)
|
||||
await send_audio_file(ws, WAV_FILE_PATH)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
@@ -175,7 +175,7 @@ export LITELLM_RATE_LIMIT_ACCURACY=true
|
||||
- **Auth**
|
||||
- Support [`x-litellm-api-key` header param by default](../../docs/pass_through/vertex_ai#use-with-virtual-keys), this fixes an issue from the prior release where `x-litellm-api-key` was not being used on vertex ai passthrough requests - [PR](https://github.com/BerriAI/litellm/pull/10392)
|
||||
- Allow key at max budget to call non-llm api endpoints - [PR](https://github.com/BerriAI/litellm/pull/10392)
|
||||
- 🆕 **[Python Client Library](../../docs/proxy/management_client) for LiteLLM Proxy management endpoints**
|
||||
- 🆕 **[Python Client Library](../../docs/proxy/management_cli) for LiteLLM Proxy management endpoints**
|
||||
- Initial PR - [PR](https://github.com/BerriAI/litellm/pull/10445)
|
||||
- Support for doing HTTP requests - [PR](https://github.com/BerriAI/litellm/pull/10452)
|
||||
- **Dependencies**
|
||||
|
||||
@@ -20,60 +20,92 @@ import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
``` showLineNumbers title="docker run litellm"
|
||||
docker run
|
||||
-e STORE_MODEL_IN_DB=True
|
||||
-p 4000:4000
|
||||
ghcr.io/berriai/litellm:main-v1.70.1-stable
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
``` showLineNumbers title="pip install litellm"
|
||||
pip install litellm==1.70.1
|
||||
```
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
- **Gemini** (VertexAI + Google AI Studio)
|
||||
- /chat/completion - Handle audio input - https://github.com/BerriAI/litellm/pull/10739
|
||||
- Fixes maximum recursion depth issue when using deeply nested response schemas with Vertex AI by Increasing DEFAULT_MAX_RECURSE_DEPTH from 10 to 100 in constants. https://github.com/BerriAI/litellm/pull/10798
|
||||
- Capture reasoning tokens in streaming mode - https://github.com/BerriAI/litellm/pull/10789
|
||||
- **VertexAI**
|
||||
- Fix llama streaming error - where model response was nested in returned streaming chunk - https://github.com/BerriAI/litellm/pull/10878
|
||||
- **Ollama**
|
||||
- structure responses fix - https://github.com/BerriAI/litellm/pull/10617
|
||||
- **Bedrock**
|
||||
- `/chat/completion` - Handle thinking_blocks when assistant.content is None - https://github.com/BerriAI/litellm/pull/10688
|
||||
- `/messages` - allow using dynamic AWS Params
|
||||
- Fixes to only allow accepted fields for tool json schema - https://github.com/BerriAI/litellm/pull/10062
|
||||
- Add bedrock sonnet prompt caching cost information
|
||||
- Mistral Pixtral support - https://github.com/BerriAI/litellm/pull/10439
|
||||
- Tool caching support - https://github.com/BerriAI/litellm/pull/10897
|
||||
- **Nvidia NIM**
|
||||
- Add tools, tool_choice, parallel_tool_calls support - https://github.com/BerriAI/litellm/pull/10763
|
||||
- **LiteLLM Proxy (`litellm_proxy/`)**
|
||||
- Option to force/always use the litellm proxy when calling via LiteLLM SDK - https://github.com/BerriAI/litellm/pull/10773
|
||||
- **Novita AI**
|
||||
- Support on `/chat/completion`, `/completions`, `/responses` API routes - https://github.com/BerriAI/litellm/pull/9527
|
||||
- **Azure**
|
||||
- Fix azure dall e 3 call with custom model name - https://github.com/BerriAI/litellm/pull/10776
|
||||
2. Add gpt-4o-mini-tts pricing - https://github.com/BerriAI/litellm/pull/10807
|
||||
3. Add cohere embed v4 pricing - https://github.com/BerriAI/litellm/pull/10806
|
||||
- **Cohere**
|
||||
- Migrate embedding to use `/v2/embed` - adds support for output_dimensions param - https://github.com/BerriAI/litellm/pull/10809
|
||||
- **Groq**
|
||||
- Update model max tokens + cost information - https://github.com/BerriAI/litellm/pull/10077
|
||||
- **Anthropic**
|
||||
- Web search tool support - native + openai format - https://github.com/BerriAI/litellm/pull/10846
|
||||
- **VLLM**
|
||||
- Support embedding input as list of integers - https://github.com/BerriAI/litellm/pull/10629
|
||||
- **OpenAI**
|
||||
- Fix - b64 file data input handling - https://github.com/BerriAI/litellm/pull/10897
|
||||
- Add ‘supports_pdf_input’ to all vision models - https://github.com/BerriAI/litellm/pull/10897
|
||||
|
||||
|
||||
- **Gemini ([VertexAI](https://docs.litellm.ai/docs/providers/vertex#usage-with-litellm-proxy-server) + [Google AI Studio](https://docs.litellm.ai/docs/providers/gemini))**
|
||||
- `/chat/completion`
|
||||
- Handle audio input - [PR](https://github.com/BerriAI/litellm/pull/10739)
|
||||
- Fixes maximum recursion depth issue when using deeply nested response schemas with Vertex AI by Increasing DEFAULT_MAX_RECURSE_DEPTH from 10 to 100 in constants. [PR](https://github.com/BerriAI/litellm/pull/10798)
|
||||
- Capture reasoning tokens in streaming mode - [PR](https://github.com/BerriAI/litellm/pull/10789)
|
||||
- **[Google AI Studio](../../docs/providers/google_ai_studio/realtime)**
|
||||
- `/realtime`
|
||||
- Gemini Multimodal Live API support
|
||||
- Audio input/output support, optional param mapping, accurate usage calculation - [PR](https://github.com/BerriAI/litellm/pull/10909)
|
||||
- **[VertexAI](../../docs/providers/vertex#metallama-api)**
|
||||
- `/chat/completion`
|
||||
- Fix llama streaming error - where model response was nested in returned streaming chunk - [PR](https://github.com/BerriAI/litellm/pull/10878)
|
||||
- **[Ollama](../../docs/providers/ollama)**
|
||||
- `/chat/completion`
|
||||
- structure responses fix - [PR](https://github.com/BerriAI/litellm/pull/10617)
|
||||
- **[Bedrock](../../docs/providers/bedrock#litellm-proxy-usage)**
|
||||
- [`/chat/completion`](../../docs/providers/bedrock#litellm-proxy-usage)
|
||||
- Handle thinking_blocks when assistant.content is None - [PR](https://github.com/BerriAI/litellm/pull/10688)
|
||||
- Fixes to only allow accepted fields for tool json schema - [PR](https://github.com/BerriAI/litellm/pull/10062)
|
||||
- Add bedrock sonnet prompt caching cost information
|
||||
- Mistral Pixtral support - [PR](https://github.com/BerriAI/litellm/pull/10439)
|
||||
- Tool caching support - [PR](https://github.com/BerriAI/litellm/pull/10897)
|
||||
- [`/messages`](../../docs/anthropic_unified)
|
||||
- allow using dynamic AWS Params - [PR](https://github.com/BerriAI/litellm/pull/10769)
|
||||
- **[Nvidia NIM](../../docs/providers/nvidia_nim)**
|
||||
- [`/chat/completion`](../../docs/providers/nvidia_nim#usage---litellm-proxy-server) [NEED DOCS ON SUPPORTED PARAMS]
|
||||
- Add tools, tool_choice, parallel_tool_calls support - [PR](https://github.com/BerriAI/litellm/pull/10763)
|
||||
- **[Novita AI](../../docs/providers/novita)**
|
||||
- New Provider added for `/chat/completion` routes - [PR](https://github.com/BerriAI/litellm/pull/9527)
|
||||
- **[Azure](../../docs/providers/azure)**
|
||||
- [`/image/generation`](../../docs/providers/azure#image-generation)
|
||||
- Fix azure dall e 3 call with custom model name - [PR](https://github.com/BerriAI/litellm/pull/10776)
|
||||
- **[Cohere](../../docs/providers/cohere)**
|
||||
- [`/embeddings`](../../docs/providers/cohere#embedding)
|
||||
- Migrate embedding to use `/v2/embed` - adds support for output_dimensions param - [PR](https://github.com/BerriAI/litellm/pull/10809)
|
||||
- **[Anthropic](../../docs/providers/anthropic)**
|
||||
- [`/chat/completion`](../../docs/providers/anthropic#usage-with-litellm-proxy)
|
||||
- Web search tool support - native + openai format - [Get Started](../../docs/providers/anthropic#anthropic-hosted-tools-computer-text-editor-web-search)
|
||||
- **[VLLM](../../docs/providers/vllm)**
|
||||
- [`/embeddings`](../../docs/providers/vllm#embeddings)
|
||||
- Support embedding input as list of integers
|
||||
- **[OpenAI](../../docs/providers/openai)**
|
||||
- [`/chat/completion`](../../docs/providers/openai#usage---litellm-proxy-server)
|
||||
- Fix - b64 file data input handling - [Get Started](../../docs/providers/openai#pdf-file-parsing)
|
||||
- Add ‘supports_pdf_input’ to all vision models - [PR](https://github.com/BerriAI/litellm/pull/10897)
|
||||
|
||||
## LLM API Endpoints
|
||||
- **Responses API**
|
||||
- Fix delete API support - https://github.com/BerriAI/litellm/pull/10845
|
||||
- **Rerank API**
|
||||
- `/v2/rerank` now registered as ‘llm_api_route’ - enabling non-admins to call it - https://github.com/BerriAI/litellm/pull/10861
|
||||
- **Realtime API**
|
||||
- Gemini Multimodal Live API support - https://github.com/BerriAI/litellm/pull/10841
|
||||
|
||||
- [**Responses API**](../../docs/response_api)
|
||||
- Fix delete API support - [PR](https://github.com/BerriAI/litellm/pull/10845)
|
||||
- [**Rerank API**](../../docs/rerank)
|
||||
- `/v2/rerank` now registered as ‘llm_api_route’ - enabling non-admins to call it - [PR](https://github.com/BerriAI/litellm/pull/10861)
|
||||
|
||||
## Spend Tracking Improvements
|
||||
- Anthropic - web search tool cost tracking - https://github.com/BerriAI/litellm/pull/10846
|
||||
- **`/chat/completion`, `/messages`**
|
||||
- Anthropic - web search tool cost tracking - [PR](https://github.com/BerriAI/litellm/pull/10846)
|
||||
- Groq - update model max tokens + cost information - [PR](https://github.com/BerriAI/litellm/pull/10077)
|
||||
- **`/audio/transcription`**
|
||||
- fix tracking spend by tag - https://github.com/BerriAI/litellm/pull/10832
|
||||
- Azure - Add gpt-4o-mini-tts pricing - [PR](https://github.com/BerriAI/litellm/pull/10807)
|
||||
- Proxy - Fix tracking spend by tag - [PR](https://github.com/BerriAI/litellm/pull/10832)
|
||||
- **`/embeddings`**
|
||||
- Azure AI - Add cohere embed v4 pricing - [PR](https://github.com/BerriAI/litellm/pull/10806)
|
||||
|
||||
## Management Endpoints / UI
|
||||
- **Models**
|
||||
@@ -93,50 +125,49 @@ import TabItem from '@theme/TabItem';
|
||||
|
||||
|
||||
## Logging / Alerting Integrations
|
||||
- **StandardLoggingPayload**
|
||||
- Log any `x-` headers in requester metadata - https://github.com/BerriAI/litellm/pull/10818
|
||||
- Guardrail tracing now in standard logging payload - https://github.com/BerriAI/litellm/pull/10893
|
||||
- **Generic API logger**
|
||||
- **[StandardLoggingPayload](../../docs/proxy/logging_spec)**
|
||||
- Log any `x-` headers in requester metadata - [Get Started](../../docs/proxy/logging_spec#standardloggingmetadata)
|
||||
- Guardrail tracing now in standard logging payload - [Get Started](../../docs/proxy/logging_spec#standardloggingguardrailinformation)
|
||||
- **[Generic API Logger](../../docs/proxy/logging#custom-callback-apis-async)**
|
||||
- Support passing application/json header
|
||||
- **Arize Phoenix**
|
||||
- fix: URL encode OTEL_EXPORTER_OTLP_TRACES_HEADERS for Phoenix Integration - https://github.com/BerriAI/litellm/pull/10654
|
||||
- add guardrail tracing to OTEL, Arize phoenix - https://github.com/BerriAI/litellm/pull/10896
|
||||
- **PagerDuty**
|
||||
- Pagerduty is now a free feature - https://github.com/BerriAI/litellm/pull/10857
|
||||
- **Alerting**
|
||||
- Sending slack alerts on virtual key/user/team updates is now free - https://github.com/BerriAI/litellm/pull/10863
|
||||
- **[Arize Phoenix](../../docs/observability/phoenix_integration)**
|
||||
- fix: URL encode OTEL_EXPORTER_OTLP_TRACES_HEADERS for Phoenix Integration - [PR](https://github.com/BerriAI/litellm/pull/10654)
|
||||
- add guardrail tracing to OTEL, Arize phoenix - [PR](https://github.com/BerriAI/litellm/pull/10896)
|
||||
- **[PagerDuty](../../docs/proxy/pagerduty)**
|
||||
- Pagerduty is now a free feature - [PR](https://github.com/BerriAI/litellm/pull/10857)
|
||||
- **[Alerting](../../docs/proxy/alerting)**
|
||||
- Sending slack alerts on virtual key/user/team updates is now free - [PR](https://github.com/BerriAI/litellm/pull/10863)
|
||||
|
||||
|
||||
## Guardrails
|
||||
- **Guardrails**
|
||||
- New `/apply_guardrail` endpoint for directly testing a guardrail - https://github.com/BerriAI/litellm/pull/10867
|
||||
- **Lakera**
|
||||
- `/v2` endpoints support - https://github.com/BerriAI/litellm/pull/10880
|
||||
- **Presidio**
|
||||
- Fixes handling of message content on presidio guardrail integration - https://github.com/BerriAI/litellm/pull/10197
|
||||
- Allow specifying PII Entities Config - https://github.com/BerriAI/litellm/pull/10810
|
||||
- **AIM Guardrails**
|
||||
- Support for anonymization in AIM Guardrails - https://github.com/BerriAI/litellm/pull/10757
|
||||
- New `/apply_guardrail` endpoint for directly testing a guardrail - [PR](https://github.com/BerriAI/litellm/pull/10867)
|
||||
- **[Lakera](../../docs/proxy/guardrails/lakera_ai)**
|
||||
- `/v2` endpoints support - [PR](https://github.com/BerriAI/litellm/pull/10880)
|
||||
- **[Presidio](../../docs/proxy/guardrails/pii_masking_v2)**
|
||||
- Fixes handling of message content on presidio guardrail integration - [PR](https://github.com/BerriAI/litellm/pull/10197)
|
||||
- Allow specifying PII Entities Config - [PR](https://github.com/BerriAI/litellm/pull/10810)
|
||||
- **[Aim Security](../../docs/proxy/guardrails/aim_security)**
|
||||
- Support for anonymization in AIM Guardrails - [PR](https://github.com/BerriAI/litellm/pull/10757)
|
||||
|
||||
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
- **Allow overriding all constants using a .env variable** - https://github.com/BerriAI/litellm/pull/10803
|
||||
- **Maximum retention period for spend logs**
|
||||
- Add retention flag to config - https://github.com/BerriAI/litellm/pull/10815
|
||||
- Support for cleaning up logs based on configured time period - https://github.com/BerriAI/litellm/pull/10872
|
||||
- Support for specifying
|
||||
- **Allow overriding all constants using a .env variable** - [PR](https://github.com/BerriAI/litellm/pull/10803)
|
||||
- **[Maximum retention period for spend logs](../../docs/proxy/spend_logs_deletion)**
|
||||
- Add retention flag to config - [PR](https://github.com/BerriAI/litellm/pull/10815)
|
||||
- Support for cleaning up logs based on configured time period - [PR](https://github.com/BerriAI/litellm/pull/10872)
|
||||
|
||||
## General Proxy Improvements
|
||||
- **Authentication**
|
||||
- Handle Bearer $LITELLM_API_KEY in x-litellm-api-key custom header - https://github.com/BerriAI/litellm/pull/10776
|
||||
- Handle Bearer $LITELLM_API_KEY in x-litellm-api-key custom header [PR](https://github.com/BerriAI/litellm/pull/10776)
|
||||
- **New Enterprise pip package** - `litellm-enterprise` - fixes issue where `enterprise` folder was not found when using pip package
|
||||
- **Proxy CLI**
|
||||
- Add `models import` command - https://github.com/BerriAI/litellm/pull/10581
|
||||
- **Docs**
|
||||
- Document in-memory + disk caching - https://github.com/BerriAI/litellm/pull/10522
|
||||
- **OpenWebUI**
|
||||
- Configure LiteLLM to Parse User Headers from Open Web UI - https://github.com/BerriAI/litellm/pull/9802
|
||||
- **[Proxy CLI](../../docs/proxy/management_cli)**
|
||||
- Add `models import` command - [PR](https://github.com/BerriAI/litellm/pull/10581)
|
||||
- **[OpenWebUI](../../docs/tutorials/openweb_ui#per-user-tracking)**
|
||||
- Configure LiteLLM to Parse User Headers from Open Web UI
|
||||
- **[LiteLLM Proxy w/ LiteLLM SDK](../../docs/providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy)**
|
||||
- Option to force/always use the litellm proxy when calling via LiteLLM SDK
|
||||
|
||||
|
||||
## New Contributors
|
||||
@@ -156,3 +187,17 @@ import TabItem from '@theme/TabItem';
|
||||
* [@damgem](https://github.com/damgem) made their first contribution in PR [#9802](https://github.com/BerriAI/litellm/pull/9802)
|
||||
* [@hxdror](https://github.com/hxdror) made their first contribution in PR [#10757](https://github.com/BerriAI/litellm/pull/10757)
|
||||
* [@wwwillchen](https://github.com/wwwillchen) made their first contribution in PR [#10894](https://github.com/BerriAI/litellm/pull/10894)
|
||||
|
||||
|
||||
## Demo Instance
|
||||
|
||||
Here's a Demo Instance to test changes:
|
||||
|
||||
- Instance: https://demo.litellm.ai/
|
||||
- Login Credentials:
|
||||
- Username: admin
|
||||
- Password: sk-1234
|
||||
|
||||
|
||||
## [Git Diff](https://github.com/BerriAI/litellm/releases)
|
||||
|
||||
|
||||
@@ -62,7 +62,6 @@ const sidebars = {
|
||||
href: "https://litellm-api.up.railway.app/",
|
||||
},
|
||||
"proxy/enterprise",
|
||||
"proxy/management_client",
|
||||
"proxy/management_cli",
|
||||
{
|
||||
type: "category",
|
||||
@@ -293,17 +292,24 @@ const sidebars = {
|
||||
},
|
||||
"providers/text_completion_openai",
|
||||
"providers/openai_compatible",
|
||||
"providers/azure",
|
||||
{
|
||||
type: "category",
|
||||
label: "Azure OpenAI",
|
||||
items: [
|
||||
"providers/azure/azure",
|
||||
"providers/azure/azure_embedding",
|
||||
]
|
||||
},
|
||||
"providers/azure_ai",
|
||||
"providers/aiml",
|
||||
"providers/vertex",
|
||||
|
||||
{
|
||||
type: "category",
|
||||
label: "Google AI Studio",
|
||||
items: [
|
||||
"providers/gemini",
|
||||
"providers/google_ai_studio/files",
|
||||
"providers/google_ai_studio/realtime",
|
||||
]
|
||||
},
|
||||
"providers/anthropic",
|
||||
@@ -491,6 +497,7 @@ const sidebars = {
|
||||
"tutorials/prompt_caching",
|
||||
"tutorials/tag_management",
|
||||
'tutorials/litellm_proxy_aporia',
|
||||
"tutorials/gemini_realtime_with_audio",
|
||||
{
|
||||
type: "category",
|
||||
label: "LiteLLM Python SDK Tutorials",
|
||||
|
||||
@@ -152,6 +152,17 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
||||
]
|
||||
return automatic_activity_dection
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
return [
|
||||
"instructions",
|
||||
"temperature",
|
||||
"max_response_output_tokens",
|
||||
"modalities",
|
||||
"tools",
|
||||
"input_audio_transcription",
|
||||
"turn_detection",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self, optional_params: dict, non_default_params: dict
|
||||
) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user