From 2a852352e7ea804202be4ed1dad5efa320dc4c85 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 22 Mar 2025 18:57:48 -0700 Subject: [PATCH] docs(release_notes/): add more to docs --- docs/my-website/docs/providers/azure.md | 194 +++++++++--------- docs/my-website/docs/providers/bedrock.md | 4 + docs/my-website/docs/providers/openrouter.md | 6 +- .../release_notes/v1.63.11-stable/index.md | 8 - .../release_notes/v1.63.14/index.md | 142 +++++++++++++ 5 files changed, 247 insertions(+), 107 deletions(-) create mode 100644 docs/my-website/release_notes/v1.63.14/index.md diff --git a/docs/my-website/docs/providers/azure.md b/docs/my-website/docs/providers/azure.md index 111738a449..2cfb92546a 100644 --- a/docs/my-website/docs/providers/azure.md +++ b/docs/my-website/docs/providers/azure.md @@ -291,14 +291,15 @@ response = completion( ) ``` -## Azure O1 Models +## O-Series Models -| Model Name | Function Call | -|---------------------|----------------------------------------------------| -| o1-mini | `response = completion(model="azure/", messages=messages)` | -| o1-preview | `response = completion(model="azure/", messages=messages)` | +Azure OpenAI O-Series models are supported on LiteLLM. -Set `litellm.enable_preview_features = True` to use Azure O1 Models with streaming support. +LiteLLM routes any deployment name with `o1` or `o3` in the model name, to the O-Series [transformation](https://github.com/BerriAI/litellm/blob/91ed05df2962b8eee8492374b048d27cc144d08c/litellm/llms/azure/chat/o1_transformation.py#L4) logic. + +To set this explicitly, set `model` to `azure/o_series/`. + +**Automatic Routing** @@ -306,60 +307,112 @@ Set `litellm.enable_preview_features = True` to use Azure O1 Models with streami ```python import litellm -litellm.enable_preview_features = True # 👈 KEY CHANGE - -response = litellm.completion( - model="azure/", - messages=[{"role": "user", "content": "What is the weather like in Boston?"}], - stream=True -) - -for chunk in response: - print(chunk) +litellm.completion(model="azure/my-o3-deployment", messages=[{"role": "user", "content": "Hello, world!"}]) # 👈 Note: 'o3' in the deployment name ``` - + -1. Setup config.yaml ```yaml model_list: - - model_name: o1-mini + - model_name: o3-mini litellm_params: - model: azure/o1-mini - api_base: "os.environ/AZURE_API_BASE" - api_key: "os.environ/AZURE_API_KEY" - api_version: "os.environ/AZURE_API_VERSION" - -litellm_settings: - enable_preview_features: true # 👈 KEY CHANGE + model: azure/o3-model + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY ``` -2. Start proxy + + + +**Explicit Routing** + + + + +```python +import litellm + +litellm.completion(model="azure/o_series/my-random-deployment-name", messages=[{"role": "user", "content": "Hello, world!"}]) # 👈 Note: 'o_series/' in the deployment name +``` + + + +```yaml +model_list: + - model_name: o3-mini + litellm_params: + model: azure/o_series/my-random-deployment-name + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY +``` + + + + +## Azure Audio Model + + + + +```python +from litellm import completion +import os + +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + +response = completion( + model="azure/azure-openai-4o-audio", + messages=[ + { + "role": "user", + "content": "I want to try out speech to speech" + } + ], + modalities=["text","audio"], + audio={"voice": "alloy", "format": "wav"} +) + +print(response) +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: azure-openai-4o-audio + litellm_params: + model: azure/azure-openai-4o-audio + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: os.environ/AZURE_API_VERSION +``` + +2. Start proxy ```bash litellm --config /path/to/config.yaml ``` -3. Test it +3. Test it! -```python -import openai -client = openai.OpenAI( - api_key="anything", - base_url="http://0.0.0.0:4000" -) -response = client.chat.completions.create(model="o1-mini", messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } -], -stream=True) - -for chunk in response: - print(chunk) +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-openai-4o-audio", + "messages": [{"role": "user", "content": "I want to try out speech to speech"}], + "modalities": ["text","audio"], + "audio": {"voice": "alloy", "format": "wav"} + }' ``` + + @@ -948,62 +1001,9 @@ Expected Response: {"data":[{"id":"batch_R3V...} ``` -## O-Series Models -Azure OpenAI O-Series models are supported on LiteLLM. -LiteLLM routes any deployment name with `o1` or `o3` in the model name, to the O-Series [transformation](https://github.com/BerriAI/litellm/blob/91ed05df2962b8eee8492374b048d27cc144d08c/litellm/llms/azure/chat/o1_transformation.py#L4) logic. -To set this explicitly, set `model` to `azure/o_series/`. - -**Automatic Routing** - - - - -```python -import litellm - -litellm.completion(model="azure/my-o3-deployment", messages=[{"role": "user", "content": "Hello, world!"}]) # 👈 Note: 'o3' in the deployment name -``` - - - -```yaml -model_list: - - model_name: o3-mini - litellm_params: - model: azure/o3-model - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY -``` - - - - -**Explicit Routing** - - - - -```python -import litellm - -litellm.completion(model="azure/o_series/my-random-deployment-name", messages=[{"role": "user", "content": "Hello, world!"}]) # 👈 Note: 'o_series/' in the deployment name -``` - - - -```yaml -model_list: - - model_name: o3-mini - litellm_params: - model: azure/o_series/my-random-deployment-name - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY -``` - - diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 45ad3f0c61..58b44b04f2 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -1428,10 +1428,14 @@ response = litellm.embedding( ## Supported AWS Bedrock Models + +LiteLLM supports ALL Bedrock models. + Here's an example of using a bedrock model with LiteLLM. For a complete list, refer to the [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) | Model Name | Command | |----------------------------|------------------------------------------------------------------| +| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | diff --git a/docs/my-website/docs/providers/openrouter.md b/docs/my-website/docs/providers/openrouter.md index 09669c9f9e..58a87f6849 100644 --- a/docs/my-website/docs/providers/openrouter.md +++ b/docs/my-website/docs/providers/openrouter.md @@ -10,9 +10,11 @@ LiteLLM supports all the text / chat / vision models from [OpenRouter](https://o import os from litellm import completion os.environ["OPENROUTER_API_KEY"] = "" +os.environ["OPENROUTER_API_BASE"] = "" # [OPTIONAL] defaults to https://openrouter.ai/api/v1 -os.environ["OR_SITE_URL"] = "" # optional -os.environ["OR_APP_NAME"] = "" # optional + +os.environ["OR_SITE_URL"] = "" # [OPTIONAL] +os.environ["OR_APP_NAME"] = "" # [OPTIONAL] response = completion( model="openrouter/google/palm-2-chat-bison", diff --git a/docs/my-website/release_notes/v1.63.11-stable/index.md b/docs/my-website/release_notes/v1.63.11-stable/index.md index f502420507..91336fc681 100644 --- a/docs/my-website/release_notes/v1.63.11-stable/index.md +++ b/docs/my-website/release_notes/v1.63.11-stable/index.md @@ -26,14 +26,6 @@ This release is primarily focused on: - UI - Credential Management, re-use credentials when adding new models - UI - Test Connection to LLM Provider before adding a model -:::info - -This release will be live on 03/16/2025 - -::: - - - ## Known Issues - 🚨 Known issue on Azure OpenAI - We don't recommend upgrading if you use Azure OpenAI. This version failed our Azure OpenAI load test diff --git a/docs/my-website/release_notes/v1.63.14/index.md b/docs/my-website/release_notes/v1.63.14/index.md new file mode 100644 index 0000000000..4fee2e8470 --- /dev/null +++ b/docs/my-website/release_notes/v1.63.14/index.md @@ -0,0 +1,142 @@ +--- +title: v1.63.14-stable +slug: v1.63.14-stable +date: 2025-03-22T10:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://media.licdn.com/dms/image/v2/D4D03AQGrlsJ3aqpHmQ/profile-displayphoto-shrink_400_400/B4DZSAzgP7HYAg-/0/1737327772964?e=1743638400&v=beta&t=39KOXMUFedvukiWWVPHf3qI45fuQD7lNglICwN31DrI + - name: Ishaan Jaffer + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +tags: [credential management, thinking content, responses api, snowflake] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +These are the changes since `v1.63.11-stable`. + +This release brings: +- LLM Translation Improvements (MCP Support and Bedrock Application Profiles) +- Perf improvements for Usage-based Routing +- Streaming guardrail support via websockets + +## Docker Run LiteLLM Proxy + +``` +docker run +-e STORE_MODEL_IN_DB=True +-p 4000:4000 +ghcr.io/berriai/litellm:main-v1.63.14-stable +``` + +## Demo Instance + +Here's a Demo Instance to test changes: +- Instance: https://demo.litellm.ai/ +- Login Credentials: + - Username: admin + - Password: sk-1234 + + + +## New Models / Updated Models + +- Azure gpt-4o - fixed pricing to latest global pricing - [PR](https://github.com/BerriAI/litellm/pull/9361) +- O1-Pro - add pricing + model information - [PR](https://github.com/BerriAI/litellm/pull/9397) +- Azure AI - mistral 3.1 small pricing added - [PR](https://github.com/BerriAI/litellm/pull/9453) +- Azure - gpt-4.5-preview pricing added - [PR](https://github.com/BerriAI/litellm/pull/9453) + + + +## LLM Translation + +1. **New LLM Features** + +- Bedrock: Support bedrock application inference profiles [Docs](https://docs.litellm.ai/docs/providers/bedrock#bedrock-application-inference-profile) + - Infer aws region from bedrock application profile id +- Ollama - support calling via `/v1/completions` - [NEEDS DOCS]https://github.com/BerriAI/litellm/pull/9333 +- Bedrock - support `us.deepseek.r1-v1:0` model name [Docs](../../docs/providers/bedrock#supported-aws-bedrock-models) + +https://github.com/BerriAI/litellm/pull/9363 +- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter.md) +- Azure - add audio model parameter support - https://github.com/BerriAI/litellm/commit/fe24b9d90b95012ac030f6919a766cbeab1b1ae3 +- OpenAI - ‘file’ message type support - https://github.com/BerriAI/litellm/commit/12e730885bd3948543dca902293f461c1bc4fb60 +- OpenAI - o1-pro Responses API streaming support - https://github.com/BerriAI/litellm/pull/9419 +- Passthrough Endpoints - support returning api-base on pass-through endpoints - https://github.com/BerriAI/litellm/pull/9439 +- [BETA] MCP - Use MCP Tools with LiteLLM SDK - https://github.com/BerriAI/litellm/pull/9436 [NEEDS NOTE RE: advanced section not live yet RELEASE] + +2. **Bug Fixes** + +- Voyage: prompt token on embedding tracking fix - https://github.com/BerriAI/litellm/commit/56d3e75b330c3c3862dc6e1c51c1210e48f1068e +- Streaming - Prevents final chunk w/ usage from being ignored (impacted bedrock streaming + cost tracking) - https://github.com/BerriAI/litellm/commit/dd2c980d5bb9e1a3b125e364c5d841751e67c96d +- Sagemaker - Fix ‘Too little data for declared Content-Length’ error - https://github.com/BerriAI/litellm/pull/9326 +- OpenAI-compatible models - fix issue when calling openai-compatible models w/ custom_llm_provider set - https://github.com/BerriAI/litellm/pull/9355 +- VertexAI - Embedding ‘outputDimensionality’ support - https://github.com/BerriAI/litellm/commit/437dbe724620675295f298164a076cbd8019d304 +- Anthropic - return consistent json response format on streaming/non-streaming - https://github.com/BerriAI/litellm/pull/9437 + +## Spend Tracking Improvements + +- `litellm_proxy/` - support reading litellm response cost header from proxy, when using client sdk +- Reset Budget Job - fix budget reset error on keys/teams/users - https://github.com/BerriAI/litellm/pull/9329 + +## UI + +1. Users Page + - Feature: Control default internal user settings +2. Icons: + - Feature: Replace external "artificialanalysis.ai" icons by local svg [PR](https://github.com/BerriAI/litellm/pull/9374) +3. Sign In/Sign Out + - Fix: Default login when `default_user_id` user does not exist in DB [PR](https://github.com/BerriAI/litellm/pull/9395) + + +## Security + +1. Support for Rotating Master Keys [Getting Started](https://docs.litellm.ai/docs/proxy/master_key_rotations) +2. Fix: Internal User Viewer Permissions, don't allow `internal_user_viewer` role to see `Test Key Page` or `Create Key Button` [More information on role based access controls](https://docs.litellm.ai/docs/proxy/access_control) +3. Emit audit logs on All user + model Create/Update/Delete endpoints [Getting Started](https://docs.litellm.ai/docs/proxy/multiple_admins) +4. JWT + - Support multiple JWT OIDC providers [Getting Started](https://docs.litellm.ai/docs/proxy/token_auth) + - Fix JWT access with Groups not working when team is assigned All Proxy Models access +5. Using K/V pairs in 1 AWS Secret [Getting Started](https://docs.litellm.ai/docs/secret#using-kv-pairs-in-1-aws-secret) + + +## Logging Integrations + +- Support post-call guardrails for streaming responses - https://github.com/BerriAI/litellm/commit/4a31b32a88b7729a032e58ab046079d17000087f [NEEDS DOCS] +- Arize - fix invalid package import - https://github.com/BerriAI/litellm/pull/9338 +- Arize - migrate to using standardloggingpayload for metadata, ensures spans land successfully - https://github.com/BerriAI/litellm/pull/9338 +- Arize - fix logging to just log the LLM I/O - https://github.com/BerriAI/litellm/pull/9353 +- Arize - key/team based logging support - https://github.com/BerriAI/litellm/pull/9353 +- StandardLoggingPayload - Log litellm_model_name in payload. Allows knowing what the model sent to API provider was - https://github.com/BerriAI/litellm/commit/a34cc2031dbebf9d0d26f9f96724cca37b690c57 +- Prompt Management - Allow building custom prompt management integration - https://github.com/BerriAI/litellm/pull/9384 + +## Performance / Reliability improvements + +- Redis Caching - add 5s default timeout, prevents hanging redis connection from impacting llm calls - https://github.com/BerriAI/litellm/commit/db92956ae33ed4c4e3233d7e1b0c7229817159bf +- Allow disabling all spend updates / writes to DB - patch to allow disabling all spend updates to DB with a flag - https://github.com/BerriAI/litellm/pull/9331 +- Azure OpenAI - correctly re-use azure openai client - https://github.com/BerriAI/litellm/commit/f2026ef907c06d94440930917add71314b901413 +- Azure OpenAI - uses litellm.ssl_verify on Azure/OpenAI clients +- Usage-based routing - Wildcard model support https://github.com/BerriAI/litellm/pull/9355 +- Usage-based routing - Support batch writing increments to redis - reduces latency to same as ‘simple-shuffle’ - https://github.com/BerriAI/litellm/pull/9357 +- Router - show reason for model cooldown on ‘no healthy deployments available error’ - https://github.com/BerriAI/litellm/pull/9438 +- Caching - add max value limit to an item in in-memory cache (1MB) - prevents OOM errors on large image url’s being sent through proxy - https://github.com/BerriAI/litellm/pull/9448 + + +## General Improvements + +- Multiple OIDC Provider support - https://github.com/BerriAI/litellm/commit/324864b7750747ae40345def796c1578263f5896 +- SSL - support reading ssl security level from env var - Allows user to specify lower security settings - https://github.com/BerriAI/litellm/pull/9330 +- Credentials - only poll Credentials table when `STORE_MODEL_IN_DB` is True - https://github.com/BerriAI/litellm/pull/9376 +- Image URL Handling - new architecture doc on image url handling - https://docs.litellm.ai/docs/proxy/image_handling +- OpenAI - bump to pip install "openai==1.68.2" - https://github.com/BerriAI/litellm/commit/e85e3bc52a9de86ad85c3dbb12d87664ee567a5a +- Gunicorn - security fix - bump gunicorn==23.0.0 # server dep + + +## Complete Git Diff + +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) \ No newline at end of file