From b4fc703d3b856ed86cdf93c8598a48ba9dde7b19 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Sat, 17 May 2025 17:54:25 -0700 Subject: [PATCH] LiteLLM Stable release notes (#10919) * docs(index/v1.70.1-stable): style improvements * style: add style improvements to docs * docs: cleanup docs * docs: more style improvements * docs: style improvements * docs(gemini/realtime): add docs on realtime api via Google AI Studio * docs: add openai example to anthropic web search docs * docs: add missing doc links * docs: doc cleanup * docs: add more doc links * fix: cleanup * docs: add docker information * docs: update doc links * docs: add demo instance details to doc s --- .../docs/embedding/supported_embedding.md | 30 -- .../docs/observability/phoenix_integration.md | 2 +- docs/my-website/docs/providers/anthropic.md | 84 +++++- .../docs/providers/{ => azure}/azure.md | 2 +- .../docs/providers/azure/azure_embedding.md | 93 ++++++ .../providers/google_ai_studio/realtime.md | 92 ++++++ .../docs/providers/litellm_proxy.md | 6 +- docs/my-website/docs/providers/nvidia_nim.md | 12 +- docs/my-website/docs/proxy/logging_spec.md | 20 +- .../docs/proxy/management_client.md | 267 ------------------ .../docs/proxy/spend_logs_deletion.md | 29 +- docs/my-website/docs/routing.md | 2 +- .../tutorials/gemini_realtime_with_audio.md | 136 +++++++++ .../release_notes/v1.68.0-stable/index.md | 2 +- .../release_notes/v1.70.1-stable/index.md | 201 ++++++++----- docs/my-website/sidebars.js | 13 +- .../llms/gemini/realtime/transformation.py | 11 + 17 files changed, 588 insertions(+), 414 deletions(-) rename docs/my-website/docs/providers/{ => azure}/azure.md (99%) create mode 100644 docs/my-website/docs/providers/azure/azure_embedding.md create mode 100644 docs/my-website/docs/providers/google_ai_studio/realtime.md delete mode 100644 docs/my-website/docs/proxy/management_client.md create mode 100644 docs/my-website/docs/tutorials/gemini_realtime_with_audio.md diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 06d4107372..6257ca2dba 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -225,36 +225,6 @@ response = embedding( | text-embedding-3-large | `embedding('text-embedding-3-large', input)` | `os.environ['OPENAI_API_KEY']` | | text-embedding-ada-002 | `embedding('text-embedding-ada-002', input)` | `os.environ['OPENAI_API_KEY']` | -## Azure OpenAI Embedding Models - -### API keys -This can be set as env variables or passed as **params to litellm.embedding()** -```python -import os -os.environ['AZURE_API_KEY'] = -os.environ['AZURE_API_BASE'] = -os.environ['AZURE_API_VERSION'] = -``` - -### Usage -```python -from litellm import embedding -response = embedding( - model="azure/", - input=["good morning from litellm"], - api_key=api_key, - api_base=api_base, - api_version=api_version, -) -print(response) -``` - -| Model Name | Function Call | -|----------------------|---------------------------------------------| -| text-embedding-ada-002 | `embedding(model="azure/", input=input)` | - -h/t to [Mikko](https://www.linkedin.com/in/mikkolehtimaki/) for this integration - ## OpenAI Compatible Embedding Models Use this for calling `/embedding` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index 7067a5078b..d15eea9a83 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Phoenix OSS +# Arize Phoenix OSS Open source tracing and evaluation platform diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 8b5c800eaf..990a9b5122 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -847,13 +847,50 @@ curl http://0.0.0.0:4000/v1/chat/completions \ :::info - -Unified web search (same param across OpenAI + Anthropic) coming soon! +Live from v1.70.1+ ::: +LiteLLM maps OpenAI's `search_context_size` param to Anthropic's `max_uses` param. + +| OpenAI | Anthropic | +| --- | --- | +| Low | 1 | +| Medium | 5 | +| High | 10 | + + + + + + +```python +from litellm import completion + +model = "claude-3-5-sonnet-20241022" +messages = [{"role": "user", "content": "What's the weather like today?"}] + +resp = completion( + model=model, + messages=messages, + web_search_options={ + "search_context_size": "medium", + "user_location": { + "type": "approximate", + "approximate": { + "city": "San Francisco", + }, + } + } +) + +print(resp) +``` + + + ```python from litellm import completion @@ -873,8 +910,11 @@ resp = completion( print(resp) ``` - + + + + 1. Setup config.yaml @@ -894,22 +934,56 @@ litellm --config /path/to/config.yaml 3. Test it! + + + + ```bash curl http://0.0.0.0:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $LITELLM_KEY" \ -d '{ "model": "claude-3-5-sonnet-latest", - "messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}], - "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] + "messages": [{"role": "user", "content": "What's the weather like today?"}], + "web_search_options": { + "search_context_size": "medium", + "user_location": { + "type": "approximate", + "approximate": { + "city": "San Francisco", + }, + } + } }' ``` + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "claude-3-5-sonnet-latest", + "messages": [{"role": "user", "content": "What's the weather like today?"}], + "tools": [{ + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 + }] + }' +``` + + + + + + ## Usage - Vision ```python diff --git a/docs/my-website/docs/providers/azure.md b/docs/my-website/docs/providers/azure/azure.md similarity index 99% rename from docs/my-website/docs/providers/azure.md rename to docs/my-website/docs/providers/azure/azure.md index 2ea444b029..d0b0371986 100644 --- a/docs/my-website/docs/providers/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem'; |-------|-------| | Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series | | Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#azure-o-series-models) | -| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](../embedding/supported_embedding#azure-openai-embedding-models), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | +| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | | Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) ## API Keys, Params diff --git a/docs/my-website/docs/providers/azure/azure_embedding.md b/docs/my-website/docs/providers/azure/azure_embedding.md new file mode 100644 index 0000000000..03bb501f36 --- /dev/null +++ b/docs/my-website/docs/providers/azure/azure_embedding.md @@ -0,0 +1,93 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure OpenAI Embeddings + +### API keys +This can be set as env variables or passed as **params to litellm.embedding()** +```python +import os +os.environ['AZURE_API_KEY'] = +os.environ['AZURE_API_BASE'] = +os.environ['AZURE_API_VERSION'] = +``` + +### Usage +```python +from litellm import embedding +response = embedding( + model="azure/", + input=["good morning from litellm"], + api_key=api_key, + api_base=api_base, + api_version=api_version, +) +print(response) +``` + +| Model Name | Function Call | +|----------------------|---------------------------------------------| +| text-embedding-ada-002 | `embedding(model="azure/", input=input)` | + +h/t to [Mikko](https://www.linkedin.com/in/mikkolehtimaki/) for this integration + + +## **Usage - LiteLLM Proxy Server** + +Here's how to call Azure OpenAI models with the LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export AZURE_API_KEY="" +``` + +### 2. Start the proxy + +```yaml +model_list: + - model_name: text-embedding-ada-002 + litellm_params: + model: azure/my-deployment-name + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + api_version: "2023-05-15" + api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. +``` + +### 3. Test it + + + + +```shell +curl --location 'http://0.0.0.0:4000/embeddings' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "text-embedding-ada-002", + "input": ["write a litellm poem"] + }' +``` + + + +```python +import openai +from openai import OpenAI + +# set base_url to your proxy server +# set api_key to send to proxy server +client = OpenAI(api_key="", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + input=["hello from litellm"], + model="text-embedding-ada-002" +) + +print(response) + +``` + + + + diff --git a/docs/my-website/docs/providers/google_ai_studio/realtime.md b/docs/my-website/docs/providers/google_ai_studio/realtime.md new file mode 100644 index 0000000000..50a18e131c --- /dev/null +++ b/docs/my-website/docs/providers/google_ai_studio/realtime.md @@ -0,0 +1,92 @@ +# Gemini Realtime API - Google AI Studio + +| Feature | Description | Comments | +| --- | --- | --- | +| Proxy | ✅ | | +| SDK | ⌛️ | Experimental access via `litellm._arealtime`. | + + +## Proxy Usage + +### Add model to config + +```yaml +model_list: + - model_name: "gemini-2.0-flash" + litellm_params: + model: gemini/gemini-2.0-flash-live-001 + model_info: + mode: realtime +``` + +### Start proxy + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:8000 +``` + +### Test + +Run this script using node - `node test.js` + +```js +// test.js +const WebSocket = require("ws"); + +const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gemini-2.0-flash"; + +const ws = new WebSocket(url, { + headers: { + "api-key": `${LITELLM_API_KEY}`, + "OpenAI-Beta": "realtime=v1", + }, +}); + +ws.on("open", function open() { + console.log("Connected to server."); + ws.send(JSON.stringify({ + type: "response.create", + response: { + modalities: ["text"], + instructions: "Please assist the user.", + } + })); +}); + +ws.on("message", function incoming(message) { + console.log(JSON.parse(message.toString())); +}); + +ws.on("error", function handleError(error) { + console.error("Error: ", error); +}); +``` + +## Limitations + +- Does not support audio transcription. +- Does not support tool calling + +## Supported OpenAI Realtime Events + +- `session.created` +- `response.created` +- `response.output_item.added` +- `conversation.item.created` +- `response.content_part.added` +- `response.text.delta` +- `response.audio.delta` +- `response.text.done` +- `response.audio.done` +- `response.content_part.done` +- `response.output_item.done` +- `response.done` + + + +## [Supported Session Params](https://github.com/BerriAI/litellm/blob/e87b536d038f77c2a2206fd7433e275c487179ee/litellm/llms/gemini/realtime/transformation.py#L155) + +## More Examples +### [Gemini Realtime API with Audio Input/Output](../../../docs/tutorials/gemini_realtime_with_audio) \ No newline at end of file diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md index bebaf1230f..a9de5d5913 100644 --- a/docs/my-website/docs/providers/litellm_proxy.md +++ b/docs/my-website/docs/providers/litellm_proxy.md @@ -163,9 +163,11 @@ LiteLLM Proxy works seamlessly with Langchain, LlamaIndex, OpenAI JS, Anthropic [Learn how to use LiteLLM proxy with these libraries →](../proxy/user_keys) -## Flags to send requests to litellm proxy +## Send all SDK requests to LiteLLM Proxy -Use the following options to route all requests through your LiteLLM proxy, regardless of the model specified. +Use this when calling LiteLLM Proxy from any library / codebase already using the LiteLLM SDK. + +These flags will route all requests through your LiteLLM proxy, regardless of the model specified. When enabled, requests will use `LITELLM_PROXY_API_BASE` with `LITELLM_PROXY_API_KEY` as the authentication. diff --git a/docs/my-website/docs/providers/nvidia_nim.md b/docs/my-website/docs/providers/nvidia_nim.md index 04390e7efe..270b356c91 100644 --- a/docs/my-website/docs/providers/nvidia_nim.md +++ b/docs/my-website/docs/providers/nvidia_nim.md @@ -10,10 +10,19 @@ https://docs.api.nvidia.com/nim/reference/ ::: +| Property | Details | +|-------|-------| +| Description | Nvidia NIM is a platform that provides a simple API for deploying and using AI models. LiteLLM supports all models from [Nvidia NIM](https://developer.nvidia.com/nim/) | +| Provider Route on LiteLLM | `nvidia_nim/` | +| Provider Doc | [Nvidia NIM Docs ↗](https://developer.nvidia.com/nim/) | +| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ | +| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings` | + ## API Key ```python # env variable -os.environ['NVIDIA_NIM_API_KEY'] +os.environ['NVIDIA_NIM_API_KEY'] = "" +os.environ['NVIDIA_NIM_API_BASE'] = "" # [OPTIONAL] - default is https://integrate.api.nvidia.com/v1/ ``` ## Sample Usage @@ -100,6 +109,7 @@ Here's how to call an Nvidia NIM Endpoint with the LiteLLM Proxy Server litellm_params: model: nvidia_nim/ # add nvidia_nim/ prefix to route as Nvidia NIM provider api_key: api-key # api key to send your model + # api_base: "" # [OPTIONAL] - default is https://integrate.api.nvidia.com/v1/ ``` diff --git a/docs/my-website/docs/proxy/logging_spec.md b/docs/my-website/docs/proxy/logging_spec.md index b30d228c44..a39a62318e 100644 --- a/docs/my-website/docs/proxy/logging_spec.md +++ b/docs/my-website/docs/proxy/logging_spec.md @@ -60,6 +60,8 @@ Inherits from `StandardLoggingUserAPIKeyMetadata` and adds: | `requester_ip_address` | `Optional[str]` | Requester's IP address | | `requester_metadata` | `Optional[dict]` | Additional requester metadata | | `vector_store_request_metadata` | `Optional[List[StandardLoggingVectorStoreRequest]]` | Vector store request metadata | +| `requester_custom_headers` | Dict[str, str] | Any custom (`x-`) headers sent by the client to the proxy. | +| `guardrail_information` | `Optional[StandardLoggingGuardrailInformation]` | Guardrail information | ## StandardLoggingVectorStoreRequest @@ -127,4 +129,20 @@ Inherits from `StandardLoggingUserAPIKeyMetadata` and adds: A literal type with two possible values: - `"success"` -- `"failure"` \ No newline at end of file +- `"failure"` + +## StandardLoggingGuardrailInformation + +| Field | Type | Description | +|-------|------|-------------| +| `guardrail_name` | `Optional[str]` | Guardrail name | +| `guardrail_mode` | `Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks]]]` | Guardrail mode | +| `guardrail_request` | `Optional[dict]` | Guardrail request | +| `guardrail_response` | `Optional[Union[dict, str, List[dict]]]` | Guardrail response | +| `guardrail_status` | `Literal["success", "failure"]` | Guardrail status | +| `start_time` | `Optional[float]` | Start time of the guardrail | +| `end_time` | `Optional[float]` | End time of the guardrail | +| `duration` | `Optional[float]` | Duration of the guardrail in seconds | +| `masked_entity_count` | `Optional[Dict[str, int]]` | Count of masked entities | + + diff --git a/docs/my-website/docs/proxy/management_client.md b/docs/my-website/docs/proxy/management_client.md deleted file mode 100644 index bb19aaf36b..0000000000 --- a/docs/my-website/docs/proxy/management_client.md +++ /dev/null @@ -1,267 +0,0 @@ -# LiteLLM Proxy Client - -> **See also:** [LiteLLM Proxy CLI Management Tool](./management_cli.md) - -A Python client library for interacting with the LiteLLM proxy server. This client provides a clean, typed interface for managing models, keys, credentials, and making chat completions. - -## Installation - -```bash -pip install litellm -``` - -## Quick Start - -```python -from litellm.proxy.client import Client - -# Initialize the client -client = Client( - base_url="http://localhost:4000", # Your LiteLLM proxy server URL - api_key="sk-api-key" # Optional: API key for authentication -) - -# Make a chat completion request -response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - {"role": "user", "content": "Hello, how are you?"} - ] -) -print(response.choices[0].message.content) -``` - -## Features - -The client is organized into several resource clients for different functionality: - -- `chat`: Chat completions -- `models`: Model management -- `model_groups`: Model group management -- `keys`: API key management -- `credentials`: Credential management -- `http`: Low-level HTTP client - -## Chat Completions - -Make chat completion requests to your LiteLLM proxy: - -```python -# Basic chat completion -response = client.chat.completions.create( - model="gpt-4", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the capital of France?"} - ] -) - -# Stream responses -for chunk in client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Tell me a story"}], - stream=True -): - print(chunk.choices[0].delta.content or "", end="") -``` - -## Model Management - -Manage available models on your proxy: - -```python -# List available models -models = client.models.list() - -# Add a new model -client.models.add( - model_name="gpt-4", - litellm_params={ - "api_key": "your-openai-key", - "api_base": "https://api.openai.com/v1" - } -) - -# Delete a model -client.models.delete(model_name="gpt-4") -``` - -## API Key Management - -Manage virtual API keys: - -```python -# Generate a new API key -key = client.keys.generate( - models=["gpt-4", "gpt-3.5-turbo"], - aliases={"gpt4": "gpt-4"}, - duration="24h", - key_alias="my-key", - team_id="team123" -) - -# List all keys -keys = client.keys.list( - page=1, - size=10, - return_full_object=True -) - -# Delete keys -client.keys.delete( - keys=["sk-key1", "sk-key2"], - key_aliases=["alias1", "alias2"] -) -``` - -## Credential Management - -Manage model credentials: - -```python -# Create new credentials -client.credentials.create( - credential_name="azure1", - credential_info={"api_type": "azure"}, - credential_values={ - "api_key": "your-azure-key", - "api_base": "https://example.azure.openai.com" - } -) - -# List all credentials -credentials = client.credentials.list() - -# Get a specific credential -credential = client.credentials.get(credential_name="azure1") - -# Delete credentials -client.credentials.delete(credential_name="azure1") -``` - -## Model Groups - -Manage model groups for load balancing and fallbacks: - -```python -# Create a model group -client.model_groups.create( - name="gpt4-group", - models=[ - {"model_name": "gpt-4", "litellm_params": {"api_key": "key1"}}, - {"model_name": "gpt-4-backup", "litellm_params": {"api_key": "key2"}} - ] -) - -# List model groups -groups = client.model_groups.list() - -# Delete a model group -client.model_groups.delete(name="gpt4-group") -``` - -## Low-Level HTTP Client - -The client provides access to a low-level HTTP client for making direct requests -to the LiteLLM proxy server. This is useful when you need more control or when -working with endpoints that don't yet have a high-level interface. - -```python -# Access the HTTP client -client = Client( - base_url="http://localhost:4000", - api_key="sk-api-key" -) - -# Make a custom request -response = client.http.request( - method="POST", - uri="/health/test_connection", - json={ - "litellm_params": { - "model": "gpt-4", - "api_key": "your-api-key", - "api_base": "https://api.openai.com/v1" - }, - "mode": "chat" - } -) - -# The response is automatically parsed from JSON -print(response) -``` - -### HTTP Client Features - -- Automatic URL handling (handles trailing/leading slashes) -- Built-in authentication (adds Bearer token if `api_key` is provided) -- JSON request/response handling -- Configurable timeout (default: 30 seconds) -- Comprehensive error handling -- Support for custom headers and request parameters - -### HTTP Client `request` method parameters - -- `method`: HTTP method (GET, POST, PUT, DELETE, etc.) -- `uri`: URI path (will be appended to base_url) -- `data`: (optional) Data to send in the request body -- `json`: (optional) JSON data to send in the request body -- `headers`: (optional) Custom HTTP headers -- Additional keyword arguments are passed to the underlying requests library - -## Error Handling - -The client provides clear error handling with custom exceptions: - -```python -from litellm.proxy.client.exceptions import UnauthorizedError - -try: - response = client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Hello"}] - ) -except UnauthorizedError as e: - print("Authentication failed:", e) -except Exception as e: - print("Request failed:", e) -``` - -## Advanced Usage - -### Request Customization - -All methods support returning the raw request object for inspection or modification: - -```python -# Get the prepared request without sending it -request = client.models.list(return_request=True) -print(request.method) # GET -print(request.url) # http://localhost:8000/models -print(request.headers) # {'Content-Type': 'application/json', ...} -``` - -### Pagination - -Methods that return lists support pagination: - -```python -# Get the first page of keys -page1 = client.keys.list(page=1, size=10) - -# Get the second page -page2 = client.keys.list(page=2, size=10) -``` - -### Filtering - -Many list methods support filtering: - -```python -# Filter keys by user and team -keys = client.keys.list( - user_id="user123", - team_id="team456", - include_team_keys=True -) -``` \ No newline at end of file diff --git a/docs/my-website/docs/proxy/spend_logs_deletion.md b/docs/my-website/docs/proxy/spend_logs_deletion.md index 94a6044192..45bb69f0b3 100644 --- a/docs/my-website/docs/proxy/spend_logs_deletion.md +++ b/docs/my-website/docs/proxy/spend_logs_deletion.md @@ -1,21 +1,14 @@ ---- -id: spend_logs_deletion -title: Spend Logs Deletion ---- +# Maximum Retention Period for Spend Logs -# Spend Log Cleanup - -LiteLLM stores a log for every request. Over time, these logs can grow large and slow down your database. The Spend Log Cleanup feature helps manage database size by deleting old logs automatically. - ---- - -## Usage +This walks through how to set the maximum retention period for spend logs. This helps manage database size by deleting old logs automatically. ### Requirements - **Postgres** (for log storage) - **Redis** *(optional)* — required only if you're running multiple proxy instances and want to enable distributed locking +## Usage + ### Setup Add this to your `proxy_config.yaml` under `general_settings`: @@ -24,8 +17,8 @@ Add this to your `proxy_config.yaml` under `general_settings`: general_settings: maximum_spend_logs_retention_period: "7d" # Keep logs for 7 days - # Optional: set how frequently cleanup should run - maximum_spend_logs_retention_interval: "1d" # Run cleanup every day + # Optional: set how frequently cleanup should run - default is daily + maximum_spend_logs_retention_interval: "1d" # Run cleanup daily litellm_settings: cache: true @@ -48,8 +41,6 @@ How long logs should be kept before deletion. Supported formats: How often the cleanup job should run. Uses the same format as above. If not set, cleanup will run every 24 hours if and only if `maximum_spend_logs_retention_period` is set. ---- - ## How it works ### Step 1. Lock Acquisition (Optional with Redis) @@ -65,8 +56,6 @@ If Redis is enabled, LiteLLM uses it to make sure only one instance runs the cle ![Working of spend log deletions](../../img/spend_log_deletion_working.png) *Working of spend log deletions* ---- - ### Step 2. Batch Deletion Once cleanup starts: @@ -90,9 +79,3 @@ This would allow up to 200,000 logs to be deleted in one run. ![Batch deletion of old logs](../../img/spend_log_deletion_multi_pod.jpg) *Batch deletion of old logs* - ---- - -## Summary - -Spend Log Cleanup helps keep your database fast by regularly deleting old logs. It’s safe, customizable, and works well for both single-node and multi-node deployments. \ No newline at end of file diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 967d5ad483..fa784a719c 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -25,7 +25,7 @@ If you want a server to load balance across different LLM APIs, use our [LiteLLM ### Quick Start -Loadbalance across multiple [azure](./providers/azure.md)/[bedrock](./providers/bedrock.md)/[provider](./providers/) deployments. LiteLLM will handle retrying in different regions if a call fails. +Loadbalance across multiple [azure](./providers/azure)/[bedrock](./providers/bedrock.md)/[provider](./providers/) deployments. LiteLLM will handle retrying in different regions if a call fails. diff --git a/docs/my-website/docs/tutorials/gemini_realtime_with_audio.md b/docs/my-website/docs/tutorials/gemini_realtime_with_audio.md new file mode 100644 index 0000000000..e6814c5690 --- /dev/null +++ b/docs/my-website/docs/tutorials/gemini_realtime_with_audio.md @@ -0,0 +1,136 @@ +# Call Gemini Realtime API with Audio Input/Output + +:::info +Requires LiteLLM Proxy v1.70.1+ +::: + +1. Setup config.yaml for LiteLLM Proxy + +```yaml +model_list: + - model_name: "gemini-2.0-flash" + litellm_params: + model: gemini/gemini-2.0-flash-live-001 + model_info: + mode: realtime +``` + +2. Start LiteLLM Proxy + +```bash +litellm-proxy start +``` + +3. Run test script + +```python +import asyncio +import websockets +import json +import base64 +from dotenv import load_dotenv +import wave +import base64 +import soundfile as sf +import sounddevice as sd +import io +import numpy as np + +# Load environment variables + +OPENAI_API_KEY = "sk-1234" # Replace with your LiteLLM API key +OPENAI_API_URL = 'ws://{PROXY_URL}/v1/realtime?model=gemini-2.0-flash' # REPLACE WITH `wss://{PROXY_URL}/v1/realtime?model=gemini-2.0-flash` for secure connection +WAV_FILE_PATH = "/path/to/audio.wav" # Replace with your .wav file path + +async def send_session_update(ws): + session_update = { + "type": "session.update", + "session": { + "conversation_id": "123456", + "language": "en-US", + "transcription_mode": "fast", + "modalities": ["text"] + } + } + await ws.send(json.dumps(session_update)) + +async def send_audio_file(ws, file_path): + with wave.open(file_path, 'rb') as wav_file: + chunk_size = 1024 # Adjust as needed + while True: + chunk = wav_file.readframes(chunk_size) + if not chunk: + break + base64_audio = base64.b64encode(chunk).decode('utf-8') + audio_message = { + "type": "input_audio_buffer.append", + "audio": base64_audio + } + await ws.send(json.dumps(audio_message)) + await asyncio.sleep(0.1) # Add a small delay to simulate real-time streaming + + # Send end of audio stream message + await ws.send(json.dumps({"type": "input_audio_buffer.end"})) + +def play_base64_audio(base64_string, sample_rate=24000, channels=1): + # Decode the base64 string + audio_data = base64.b64decode(base64_string) + + # Convert to numpy array + audio_np = np.frombuffer(audio_data, dtype=np.int16) + + # Reshape if stereo + if channels == 2: + audio_np = audio_np.reshape(-1, 2) + + # Normalize + audio_float = audio_np.astype(np.float32) / 32768.0 + + # Play the audio + sd.play(audio_float, sample_rate) + sd.wait() + + +def combine_base64_audio(base64_strings): + # Step 1: Decode base64 strings to binary + binary_data = [base64.b64decode(s) for s in base64_strings] + + # Step 2: Concatenate binary data + combined_binary = b''.join(binary_data) + + # Step 3: Encode combined binary back to base64 + combined_base64 = base64.b64encode(combined_binary).decode('utf-8') + + return combined_base64 + +async def listen_in_background(ws): + combined_b64_audio_str = [] + try: + while True: + response = await ws.recv() + message_json = json.loads(response) + print(f"message_json: {message_json}") + + if message_json['type'] == 'response.audio.delta' and message_json.get('delta'): + play_base64_audio(message_json["delta"]) + except Exception: + print("END OF STREAM") + +async def main(): + async with websockets.connect( + OPENAI_API_URL, + additional_headers={ + "Authorization": f"Bearer {OPENAI_API_KEY}", + "OpenAI-Beta": "realtime=v1" + } + ) as ws: + asyncio.create_task(listen_in_background(ws=ws)) + await send_session_update(ws) + await send_audio_file(ws, WAV_FILE_PATH) + + + +if __name__ == "__main__": + asyncio.run(main()) +``` + diff --git a/docs/my-website/release_notes/v1.68.0-stable/index.md b/docs/my-website/release_notes/v1.68.0-stable/index.md index 782e076bb2..4d456d9c85 100644 --- a/docs/my-website/release_notes/v1.68.0-stable/index.md +++ b/docs/my-website/release_notes/v1.68.0-stable/index.md @@ -175,7 +175,7 @@ export LITELLM_RATE_LIMIT_ACCURACY=true - **Auth** - Support [`x-litellm-api-key` header param by default](../../docs/pass_through/vertex_ai#use-with-virtual-keys), this fixes an issue from the prior release where `x-litellm-api-key` was not being used on vertex ai passthrough requests - [PR](https://github.com/BerriAI/litellm/pull/10392) - Allow key at max budget to call non-llm api endpoints - [PR](https://github.com/BerriAI/litellm/pull/10392) -- 🆕 **[Python Client Library](../../docs/proxy/management_client) for LiteLLM Proxy management endpoints** +- 🆕 **[Python Client Library](../../docs/proxy/management_cli) for LiteLLM Proxy management endpoints** - Initial PR - [PR](https://github.com/BerriAI/litellm/pull/10445) - Support for doing HTTP requests - [PR](https://github.com/BerriAI/litellm/pull/10452) - **Dependencies** diff --git a/docs/my-website/release_notes/v1.70.1-stable/index.md b/docs/my-website/release_notes/v1.70.1-stable/index.md index 669c136152..32c63013ce 100644 --- a/docs/my-website/release_notes/v1.70.1-stable/index.md +++ b/docs/my-website/release_notes/v1.70.1-stable/index.md @@ -20,60 +20,92 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; + +## Deploy this version + + + + +``` showLineNumbers title="docker run litellm" +docker run +-e STORE_MODEL_IN_DB=True +-p 4000:4000 +ghcr.io/berriai/litellm:main-v1.70.1-stable +``` + + + + +``` showLineNumbers title="pip install litellm" +pip install litellm==1.70.1 +``` + + + + ## New Models / Updated Models -- **Gemini** (VertexAI + Google AI Studio) - - /chat/completion - Handle audio input - https://github.com/BerriAI/litellm/pull/10739 - - Fixes maximum recursion depth issue when using deeply nested response schemas with Vertex AI by Increasing DEFAULT_MAX_RECURSE_DEPTH from 10 to 100 in constants. https://github.com/BerriAI/litellm/pull/10798 - - Capture reasoning tokens in streaming mode - https://github.com/BerriAI/litellm/pull/10789 -- **VertexAI** - - Fix llama streaming error - where model response was nested in returned streaming chunk - https://github.com/BerriAI/litellm/pull/10878 -- **Ollama** - - structure responses fix - https://github.com/BerriAI/litellm/pull/10617 -- **Bedrock** - - `/chat/completion` - Handle thinking_blocks when assistant.content is None - https://github.com/BerriAI/litellm/pull/10688 - - `/messages` - allow using dynamic AWS Params - - Fixes to only allow accepted fields for tool json schema - https://github.com/BerriAI/litellm/pull/10062 - - Add bedrock sonnet prompt caching cost information - - Mistral Pixtral support - https://github.com/BerriAI/litellm/pull/10439 - - Tool caching support - https://github.com/BerriAI/litellm/pull/10897 -- **Nvidia NIM** - - Add tools, tool_choice, parallel_tool_calls support - https://github.com/BerriAI/litellm/pull/10763 -- **LiteLLM Proxy (`litellm_proxy/`)** - - Option to force/always use the litellm proxy when calling via LiteLLM SDK - https://github.com/BerriAI/litellm/pull/10773 -- **Novita AI** - - Support on `/chat/completion`, `/completions`, `/responses` API routes - https://github.com/BerriAI/litellm/pull/9527 -- **Azure** - - Fix azure dall e 3 call with custom model name - https://github.com/BerriAI/litellm/pull/10776 - 2. Add gpt-4o-mini-tts pricing - https://github.com/BerriAI/litellm/pull/10807 - 3. Add cohere embed v4 pricing - https://github.com/BerriAI/litellm/pull/10806 -- **Cohere** - - Migrate embedding to use `/v2/embed` - adds support for output_dimensions param - https://github.com/BerriAI/litellm/pull/10809 -- **Groq** - - Update model max tokens + cost information - https://github.com/BerriAI/litellm/pull/10077 -- **Anthropic** - - Web search tool support - native + openai format - https://github.com/BerriAI/litellm/pull/10846 -- **VLLM** - - Support embedding input as list of integers - https://github.com/BerriAI/litellm/pull/10629 -- **OpenAI** - - Fix - b64 file data input handling - https://github.com/BerriAI/litellm/pull/10897 - - Add ‘supports_pdf_input’ to all vision models - https://github.com/BerriAI/litellm/pull/10897 - - +- **Gemini ([VertexAI](https://docs.litellm.ai/docs/providers/vertex#usage-with-litellm-proxy-server) + [Google AI Studio](https://docs.litellm.ai/docs/providers/gemini))** + - `/chat/completion` + - Handle audio input - [PR](https://github.com/BerriAI/litellm/pull/10739) + - Fixes maximum recursion depth issue when using deeply nested response schemas with Vertex AI by Increasing DEFAULT_MAX_RECURSE_DEPTH from 10 to 100 in constants. [PR](https://github.com/BerriAI/litellm/pull/10798) + - Capture reasoning tokens in streaming mode - [PR](https://github.com/BerriAI/litellm/pull/10789) +- **[Google AI Studio](../../docs/providers/google_ai_studio/realtime)** + - `/realtime` + - Gemini Multimodal Live API support + - Audio input/output support, optional param mapping, accurate usage calculation - [PR](https://github.com/BerriAI/litellm/pull/10909) +- **[VertexAI](../../docs/providers/vertex#metallama-api)** + - `/chat/completion` + - Fix llama streaming error - where model response was nested in returned streaming chunk - [PR](https://github.com/BerriAI/litellm/pull/10878) +- **[Ollama](../../docs/providers/ollama)** + - `/chat/completion` + - structure responses fix - [PR](https://github.com/BerriAI/litellm/pull/10617) +- **[Bedrock](../../docs/providers/bedrock#litellm-proxy-usage)** + - [`/chat/completion`](../../docs/providers/bedrock#litellm-proxy-usage) + - Handle thinking_blocks when assistant.content is None - [PR](https://github.com/BerriAI/litellm/pull/10688) + - Fixes to only allow accepted fields for tool json schema - [PR](https://github.com/BerriAI/litellm/pull/10062) + - Add bedrock sonnet prompt caching cost information + - Mistral Pixtral support - [PR](https://github.com/BerriAI/litellm/pull/10439) + - Tool caching support - [PR](https://github.com/BerriAI/litellm/pull/10897) + - [`/messages`](../../docs/anthropic_unified) + - allow using dynamic AWS Params - [PR](https://github.com/BerriAI/litellm/pull/10769) +- **[Nvidia NIM](../../docs/providers/nvidia_nim)** + - [`/chat/completion`](../../docs/providers/nvidia_nim#usage---litellm-proxy-server) [NEED DOCS ON SUPPORTED PARAMS] + - Add tools, tool_choice, parallel_tool_calls support - [PR](https://github.com/BerriAI/litellm/pull/10763) +- **[Novita AI](../../docs/providers/novita)** + - New Provider added for `/chat/completion` routes - [PR](https://github.com/BerriAI/litellm/pull/9527) +- **[Azure](../../docs/providers/azure)** + - [`/image/generation`](../../docs/providers/azure#image-generation) + - Fix azure dall e 3 call with custom model name - [PR](https://github.com/BerriAI/litellm/pull/10776) +- **[Cohere](../../docs/providers/cohere)** + - [`/embeddings`](../../docs/providers/cohere#embedding) + - Migrate embedding to use `/v2/embed` - adds support for output_dimensions param - [PR](https://github.com/BerriAI/litellm/pull/10809) +- **[Anthropic](../../docs/providers/anthropic)** + - [`/chat/completion`](../../docs/providers/anthropic#usage-with-litellm-proxy) + - Web search tool support - native + openai format - [Get Started](../../docs/providers/anthropic#anthropic-hosted-tools-computer-text-editor-web-search) +- **[VLLM](../../docs/providers/vllm)** + - [`/embeddings`](../../docs/providers/vllm#embeddings) + - Support embedding input as list of integers +- **[OpenAI](../../docs/providers/openai)** + - [`/chat/completion`](../../docs/providers/openai#usage---litellm-proxy-server) + - Fix - b64 file data input handling - [Get Started](../../docs/providers/openai#pdf-file-parsing) + - Add ‘supports_pdf_input’ to all vision models - [PR](https://github.com/BerriAI/litellm/pull/10897) ## LLM API Endpoints -- **Responses API** - - Fix delete API support - https://github.com/BerriAI/litellm/pull/10845 -- **Rerank API** - - `/v2/rerank` now registered as ‘llm_api_route’ - enabling non-admins to call it - https://github.com/BerriAI/litellm/pull/10861 -- **Realtime API** - - Gemini Multimodal Live API support - https://github.com/BerriAI/litellm/pull/10841 - +- [**Responses API**](../../docs/response_api) + - Fix delete API support - [PR](https://github.com/BerriAI/litellm/pull/10845) +- [**Rerank API**](../../docs/rerank) + - `/v2/rerank` now registered as ‘llm_api_route’ - enabling non-admins to call it - [PR](https://github.com/BerriAI/litellm/pull/10861) ## Spend Tracking Improvements -- Anthropic - web search tool cost tracking - https://github.com/BerriAI/litellm/pull/10846 +- **`/chat/completion`, `/messages`** + - Anthropic - web search tool cost tracking - [PR](https://github.com/BerriAI/litellm/pull/10846) + - Groq - update model max tokens + cost information - [PR](https://github.com/BerriAI/litellm/pull/10077) - **`/audio/transcription`** - - fix tracking spend by tag - https://github.com/BerriAI/litellm/pull/10832 + - Azure - Add gpt-4o-mini-tts pricing - [PR](https://github.com/BerriAI/litellm/pull/10807) + - Proxy - Fix tracking spend by tag - [PR](https://github.com/BerriAI/litellm/pull/10832) +- **`/embeddings`** + - Azure AI - Add cohere embed v4 pricing - [PR](https://github.com/BerriAI/litellm/pull/10806) ## Management Endpoints / UI - **Models** @@ -93,50 +125,49 @@ import TabItem from '@theme/TabItem'; ## Logging / Alerting Integrations -- **StandardLoggingPayload** - - Log any `x-` headers in requester metadata - https://github.com/BerriAI/litellm/pull/10818 - - Guardrail tracing now in standard logging payload - https://github.com/BerriAI/litellm/pull/10893 -- **Generic API logger** +- **[StandardLoggingPayload](../../docs/proxy/logging_spec)** + - Log any `x-` headers in requester metadata - [Get Started](../../docs/proxy/logging_spec#standardloggingmetadata) + - Guardrail tracing now in standard logging payload - [Get Started](../../docs/proxy/logging_spec#standardloggingguardrailinformation) +- **[Generic API Logger](../../docs/proxy/logging#custom-callback-apis-async)** - Support passing application/json header -- **Arize Phoenix** - - fix: URL encode OTEL_EXPORTER_OTLP_TRACES_HEADERS for Phoenix Integration - https://github.com/BerriAI/litellm/pull/10654 - - add guardrail tracing to OTEL, Arize phoenix - https://github.com/BerriAI/litellm/pull/10896 -- **PagerDuty** - - Pagerduty is now a free feature - https://github.com/BerriAI/litellm/pull/10857 -- **Alerting** - - Sending slack alerts on virtual key/user/team updates is now free - https://github.com/BerriAI/litellm/pull/10863 +- **[Arize Phoenix](../../docs/observability/phoenix_integration)** + - fix: URL encode OTEL_EXPORTER_OTLP_TRACES_HEADERS for Phoenix Integration - [PR](https://github.com/BerriAI/litellm/pull/10654) + - add guardrail tracing to OTEL, Arize phoenix - [PR](https://github.com/BerriAI/litellm/pull/10896) +- **[PagerDuty](../../docs/proxy/pagerduty)** + - Pagerduty is now a free feature - [PR](https://github.com/BerriAI/litellm/pull/10857) +- **[Alerting](../../docs/proxy/alerting)** + - Sending slack alerts on virtual key/user/team updates is now free - [PR](https://github.com/BerriAI/litellm/pull/10863) ## Guardrails - **Guardrails** - - New `/apply_guardrail` endpoint for directly testing a guardrail - https://github.com/BerriAI/litellm/pull/10867 -- **Lakera** - - `/v2` endpoints support - https://github.com/BerriAI/litellm/pull/10880 -- **Presidio** - - Fixes handling of message content on presidio guardrail integration - https://github.com/BerriAI/litellm/pull/10197 - - Allow specifying PII Entities Config - https://github.com/BerriAI/litellm/pull/10810 -- **AIM Guardrails** - - Support for anonymization in AIM Guardrails - https://github.com/BerriAI/litellm/pull/10757 + - New `/apply_guardrail` endpoint for directly testing a guardrail - [PR](https://github.com/BerriAI/litellm/pull/10867) +- **[Lakera](../../docs/proxy/guardrails/lakera_ai)** + - `/v2` endpoints support - [PR](https://github.com/BerriAI/litellm/pull/10880) +- **[Presidio](../../docs/proxy/guardrails/pii_masking_v2)** + - Fixes handling of message content on presidio guardrail integration - [PR](https://github.com/BerriAI/litellm/pull/10197) + - Allow specifying PII Entities Config - [PR](https://github.com/BerriAI/litellm/pull/10810) +- **[Aim Security](../../docs/proxy/guardrails/aim_security)** + - Support for anonymization in AIM Guardrails - [PR](https://github.com/BerriAI/litellm/pull/10757) ## Performance / Loadbalancing / Reliability improvements -- **Allow overriding all constants using a .env variable** - https://github.com/BerriAI/litellm/pull/10803 -- **Maximum retention period for spend logs** - - Add retention flag to config - https://github.com/BerriAI/litellm/pull/10815 - - Support for cleaning up logs based on configured time period - https://github.com/BerriAI/litellm/pull/10872 - - Support for specifying +- **Allow overriding all constants using a .env variable** - [PR](https://github.com/BerriAI/litellm/pull/10803) +- **[Maximum retention period for spend logs](../../docs/proxy/spend_logs_deletion)** + - Add retention flag to config - [PR](https://github.com/BerriAI/litellm/pull/10815) + - Support for cleaning up logs based on configured time period - [PR](https://github.com/BerriAI/litellm/pull/10872) ## General Proxy Improvements - **Authentication** - - Handle Bearer $LITELLM_API_KEY in x-litellm-api-key custom header - https://github.com/BerriAI/litellm/pull/10776 + - Handle Bearer $LITELLM_API_KEY in x-litellm-api-key custom header [PR](https://github.com/BerriAI/litellm/pull/10776) - **New Enterprise pip package** - `litellm-enterprise` - fixes issue where `enterprise` folder was not found when using pip package -- **Proxy CLI** - - Add `models import` command - https://github.com/BerriAI/litellm/pull/10581 -- **Docs** - - Document in-memory + disk caching - https://github.com/BerriAI/litellm/pull/10522 -- **OpenWebUI** - - Configure LiteLLM to Parse User Headers from Open Web UI - https://github.com/BerriAI/litellm/pull/9802 +- **[Proxy CLI](../../docs/proxy/management_cli)** + - Add `models import` command - [PR](https://github.com/BerriAI/litellm/pull/10581) +- **[OpenWebUI](../../docs/tutorials/openweb_ui#per-user-tracking)** + - Configure LiteLLM to Parse User Headers from Open Web UI +- **[LiteLLM Proxy w/ LiteLLM SDK](../../docs/providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy)** + - Option to force/always use the litellm proxy when calling via LiteLLM SDK ## New Contributors @@ -156,3 +187,17 @@ import TabItem from '@theme/TabItem'; * [@damgem](https://github.com/damgem) made their first contribution in PR [#9802](https://github.com/BerriAI/litellm/pull/9802) * [@hxdror](https://github.com/hxdror) made their first contribution in PR [#10757](https://github.com/BerriAI/litellm/pull/10757) * [@wwwillchen](https://github.com/wwwillchen) made their first contribution in PR [#10894](https://github.com/BerriAI/litellm/pull/10894) + + +## Demo Instance + +Here's a Demo Instance to test changes: + +- Instance: https://demo.litellm.ai/ +- Login Credentials: + - Username: admin + - Password: sk-1234 + + +## [Git Diff](https://github.com/BerriAI/litellm/releases) + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3c51e78b69..bfe642d513 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -62,7 +62,6 @@ const sidebars = { href: "https://litellm-api.up.railway.app/", }, "proxy/enterprise", - "proxy/management_client", "proxy/management_cli", { type: "category", @@ -293,17 +292,24 @@ const sidebars = { }, "providers/text_completion_openai", "providers/openai_compatible", - "providers/azure", + { + type: "category", + label: "Azure OpenAI", + items: [ + "providers/azure/azure", + "providers/azure/azure_embedding", + ] + }, "providers/azure_ai", "providers/aiml", "providers/vertex", - { type: "category", label: "Google AI Studio", items: [ "providers/gemini", "providers/google_ai_studio/files", + "providers/google_ai_studio/realtime", ] }, "providers/anthropic", @@ -491,6 +497,7 @@ const sidebars = { "tutorials/prompt_caching", "tutorials/tag_management", 'tutorials/litellm_proxy_aporia', + "tutorials/gemini_realtime_with_audio", { type: "category", label: "LiteLLM Python SDK Tutorials", diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 59360281ac..01fc6b86e3 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -152,6 +152,17 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ] return automatic_activity_dection + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "instructions", + "temperature", + "max_response_output_tokens", + "modalities", + "tools", + "input_audio_transcription", + "turn_detection", + ] + def map_openai_params( self, optional_params: dict, non_default_params: dict ) -> dict: