diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md
index 43ab82b8e6..817d70b87c 100644
--- a/docs/my-website/docs/benchmarks.md
+++ b/docs/my-website/docs/benchmarks.md
@@ -18,17 +18,13 @@ model_list:
### 1 Instance LiteLLM Proxy
-In these tests the baseline latency characteristics are measured against a fake-openai-endpoint.
+In these tests the median latency of directly calling the fake-openai-endpoint is 60ms.
-#### Performance Metrics
-
-| Metric | Value |
-|--------|-------|
-| **Requests per Second (RPS)** | 475 |
-| **End-to-End Latency P50 (ms)** | 100 |
-| **LiteLLM Overhead P50 (ms)** | 3 |
-| **LiteLLM Overhead P90 (ms)** | 17 |
-| **LiteLLM Overhead P99 (ms)** | 31 |
+| Metric | Litellm Proxy (1 Instance) |
+|--------|------------------------|
+| RPS | 475 |
+| Median Latency (ms) | 100 |
+| Latency overhead added by LiteLLM Proxy | 40ms |
@@ -37,8 +33,7 @@ In these tests the baseline latency characteristics are measured against a fake-
-->
#### Key Findings
-- Single instance: 475 RPS @ 100ms median latency
-- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead
+- Single instance: 475 RPS @ 100ms latency
- 2 LiteLLM instances: 950 RPS @ 100ms latency
- 4 LiteLLM instances: 1900 RPS @ 100ms latency
@@ -59,62 +54,6 @@ Each machine deploying LiteLLM had the following specs:
- 2 CPU
- 4GB RAM
-## How to measure LiteLLM Overhead
-
-All responses from litellm will include the `x-litellm-overhead-duration-ms` header, this is the latency overhead in milliseconds added by LiteLLM Proxy.
-
-
-If you want to measure this on locust you can use the following code:
-
-```python showLineNumbers title="Locust Code for measuring LiteLLM Overhead"
-import os
-import uuid
-from locust import HttpUser, task, between, events
-
-# Custom metric to track LiteLLM overhead duration
-overhead_durations = []
-
-@events.request.add_listener
-def on_request(request_type, name, response_time, response_length, response, context, exception, start_time, url, **kwargs):
- if response and hasattr(response, 'headers'):
- overhead_duration = response.headers.get('x-litellm-overhead-duration-ms')
- if overhead_duration:
- try:
- duration_ms = float(overhead_duration)
- overhead_durations.append(duration_ms)
- # Report as custom metric
- events.request.fire(
- request_type="Custom",
- name="LiteLLM Overhead Duration (ms)",
- response_time=duration_ms,
- response_length=0,
- )
- except (ValueError, TypeError):
- pass
-
-class MyUser(HttpUser):
- wait_time = between(0.5, 1) # Random wait time between requests
-
- def on_start(self):
- self.api_key = os.getenv('API_KEY', 'sk-1234567890')
- self.client.headers.update({'Authorization': f'Bearer {self.api_key}'})
-
- @task
- def litellm_completion(self):
- # no cache hits with this
- payload = {
- "model": "db-openai-endpoint",
- "messages": [{"role": "user", "content": f"{uuid.uuid4()} This is a test there will be no cache hits and we'll fill up the context" * 150}],
- "user": "my-new-end-user-1"
- }
- response = self.client.post("chat/completions", json=payload)
-
- if response.status_code != 200:
- # log the errors in error.txt
- with open("error.txt", "a") as error_log:
- error_log.write(response.text + "\n")
-```
-
## Logging Callbacks
diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md
index 792a21fc1a..33e4c7b4cc 100644
--- a/docs/my-website/docs/image_generation.md
+++ b/docs/my-website/docs/image_generation.md
@@ -207,26 +207,7 @@ Use this for Stable Diffusion models hosted on Xinference
See Xinference usage with LiteLLM [here](./providers/xinference.md#image-generation)
-## Recraft Image Generation Models
-Use this for AI-powered design and image generation with Recraft
-
-#### Usage
-
-```python showLineNumbers
-from litellm import image_generation
-import os
-
-os.environ['RECRAFT_API_KEY'] = "your-api-key"
-
-response = image_generation(
- model="recraft/recraftv3",
- prompt="A beautiful sunset over a calm ocean",
-)
-print(response)
-```
-
-See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
## OpenAI Compatible Image Generation Models
Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
diff --git a/docs/my-website/docs/projects/HolmesGPT.md b/docs/my-website/docs/projects/HolmesGPT.md
deleted file mode 100644
index 608d526368..0000000000
--- a/docs/my-website/docs/projects/HolmesGPT.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# HolmesGPT
-
-[HolmesGPT](https://github.com/robusta-dev/holmesgpt) is an AI-powered observability tool designed to enhance incident response and troubleshooting processes. It's like your 24/7 on-call assistant, helps you solve alerts faster with Automatic Correlations, Investigations, and More.
-
-LiteLLM helps HolmesGPT integrate with multiple LLM providers or bring their own model and self-host it.
-
-🔗 Try HolmesGPT → [https://github.com/robusta-dev/holmesgpt](https://github.com/robusta-dev/holmesgpt)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md
index ab4391798f..5317b744ab 100644
--- a/docs/my-website/docs/providers/azure/azure.md
+++ b/docs/my-website/docs/providers/azure/azure.md
@@ -618,43 +618,23 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
### Azure AD Token Refresh - `DefaultAzureCredential`
-Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests. `DefaultAzureCredential` automatically discovers and uses available Azure credentials from multiple sources.
+Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests
-**Option 1: Explicit DefaultAzureCredential (Recommended)**
```python
from litellm import completion
from azure.identity import DefaultAzureCredential, get_bearer_token_provider
-# DefaultAzureCredential automatically discovers credentials from:
-# - Environment variables (AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID)
-# - Managed Identity (AKS, Azure VMs, etc.)
-# - Azure CLI credentials
-# - And other Azure identity sources
token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")
+
response = completion(
model = "azure/", # model = azure/
api_base = "", # azure api base
api_version = "", # azure api version
- azure_ad_token_provider=token_provider,
- messages = [{"role": "user", "content": "good morning"}],
-)
-```
-
-**Option 2: LiteLLM Auto-Fallback to DefaultAzureCredential**
-```python
-import litellm
-
-# Enable automatic fallback to DefaultAzureCredential
-litellm.enable_azure_ad_token_refresh = True
-
-response = litellm.completion(
- model = "azure/",
- api_base = "",
- api_version = "",
+ azure_ad_token_provider=token_provider
messages = [{"role": "user", "content": "good morning"}],
)
```
@@ -662,8 +642,6 @@ response = litellm.completion(
-**Scenario 1: With Environment Variables (Traditional)**
-
1. Add relevant env vars
```bash
@@ -685,48 +663,12 @@ litellm_settings:
enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
```
-**Scenario 2: Managed Identity (AKS, Azure VMs) - No Hard-coded Credentials Required**
-
-Perfect for AKS clusters, Azure VMs, or other managed environments where Azure automatically injects credentials.
-
-```yaml
-model_list:
- - model_name: gpt-3.5-turbo
- litellm_params:
- model: azure/your-deployment-name
- api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
-
-litellm_settings:
- enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
-```
-
-**Scenario 3: Azure CLI Authentication**
-
-If you're authenticated via `az login`, no additional configuration needed:
-
-```yaml
-model_list:
- - model_name: gpt-3.5-turbo
- litellm_params:
- model: azure/your-deployment-name
- api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
-
-litellm_settings:
- enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
-```
-
3. Start proxy
```bash
litellm --config /path/to/config.yaml
```
-**How it works**:
-- LiteLLM first tries Service Principal authentication (if environment variables are available)
-- If that fails, it automatically falls back to `DefaultAzureCredential`
-- `DefaultAzureCredential` will use Managed Identity, Azure CLI credentials, or other available Azure identity sources
-- This eliminates the need for hard-coded credentials in managed environments like AKS
-
diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md
index 59668b5eb5..5b74151d73 100644
--- a/docs/my-website/docs/providers/groq.md
+++ b/docs/my-website/docs/providers/groq.md
@@ -158,7 +158,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` |
-| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
+| qwen-qwq-32b | `completion(model="groq/qwen-qwq-32b", messages)` |
## Groq - Tool / Function Calling Example
diff --git a/docs/my-website/docs/providers/hyperbolic.md b/docs/my-website/docs/providers/hyperbolic.md
deleted file mode 100644
index 7bad527fcf..0000000000
--- a/docs/my-website/docs/providers/hyperbolic.md
+++ /dev/null
@@ -1,331 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Hyperbolic
-
-## Overview
-
-| Property | Details |
-|-------|-------|
-| Description | Hyperbolic provides access to the latest models at a fraction of legacy cloud costs, with OpenAI-compatible APIs for LLMs, image generation, and more. |
-| Provider Route on LiteLLM | `hyperbolic/` |
-| Link to Provider Doc | [Hyperbolic Documentation ↗](https://docs.hyperbolic.xyz) |
-| Base URL | `https://api.hyperbolic.xyz/v1` |
-| Supported Operations | [`/chat/completions`](#sample-usage) |
-
-
-
-
-https://docs.hyperbolic.xyz
-
-**We support ALL Hyperbolic models, just set `hyperbolic/` as a prefix when sending completion requests**
-
-## Available Models
-
-### Language Models
-
-| Model | Description | Context Window | Pricing per 1M tokens |
-|-------|-------------|----------------|----------------------|
-| `hyperbolic/deepseek-ai/DeepSeek-V3` | DeepSeek V3 - Fast and efficient | 131,072 tokens | $0.25 |
-| `hyperbolic/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 March 2024 version | 131,072 tokens | $0.25 |
-| `hyperbolic/deepseek-ai/DeepSeek-R1` | DeepSeek R1 - Reasoning model | 131,072 tokens | $2.00 |
-| `hyperbolic/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 May 2028 version | 131,072 tokens | $0.25 |
-| `hyperbolic/Qwen/Qwen2.5-72B-Instruct` | Qwen 2.5 72B Instruct | 131,072 tokens | $0.40 |
-| `hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct` | Qwen 2.5 Coder 32B for code generation | 131,072 tokens | $0.20 |
-| `hyperbolic/Qwen/Qwen3-235B-A22B` | Qwen 3 235B A22B variant | 131,072 tokens | $2.00 |
-| `hyperbolic/Qwen/QwQ-32B` | Qwen QwQ 32B | 131,072 tokens | $0.20 |
-| `hyperbolic/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B Instruct | 131,072 tokens | $0.80 |
-| `hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct` | Llama 3.1 405B Instruct | 131,072 tokens | $5.00 |
-| `hyperbolic/moonshotai/Kimi-K2-Instruct` | Kimi K2 Instruct | 131,072 tokens | $2.00 |
-
-## Required Variables
-
-```python showLineNumbers title="Environment Variables"
-os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
-```
-
-Get your API key from [Hyperbolic dashboard](https://app.hyperbolic.ai).
-
-## Usage - LiteLLM Python SDK
-
-### Non-streaming
-
-```python showLineNumbers title="Hyperbolic Non-streaming Completion"
-import os
-import litellm
-from litellm import completion
-
-os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
-
-messages = [{"content": "What is the capital of France?", "role": "user"}]
-
-# Hyperbolic call
-response = completion(
- model="hyperbolic/Qwen/Qwen2.5-72B-Instruct",
- messages=messages
-)
-
-print(response)
-```
-
-### Streaming
-
-```python showLineNumbers title="Hyperbolic Streaming Completion"
-import os
-import litellm
-from litellm import completion
-
-os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
-
-messages = [{"content": "Write a short poem about AI", "role": "user"}]
-
-# Hyperbolic call with streaming
-response = completion(
- model="hyperbolic/deepseek-ai/DeepSeek-V3",
- messages=messages,
- stream=True
-)
-
-for chunk in response:
- print(chunk)
-```
-
-### Function Calling
-
-```python showLineNumbers title="Hyperbolic Function Calling"
-import os
-import litellm
-from litellm import completion
-
-os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
-
-tools = [
- {
- "type": "function",
- "function": {
- "name": "get_weather",
- "description": "Get the current weather in a location",
- "parameters": {
- "type": "object",
- "properties": {
- "location": {
- "type": "string",
- "description": "The city and state, e.g. San Francisco, CA"
- },
- "unit": {
- "type": "string",
- "enum": ["celsius", "fahrenheit"]
- }
- },
- "required": ["location"]
- }
- }
- }
-]
-
-response = completion(
- model="hyperbolic/deepseek-ai/DeepSeek-V3",
- messages=[{"role": "user", "content": "What's the weather like in New York?"}],
- tools=tools,
- tool_choice="auto"
-)
-
-print(response)
-```
-
-## Usage - LiteLLM Proxy
-
-Add the following to your LiteLLM Proxy configuration file:
-
-```yaml showLineNumbers title="config.yaml"
-model_list:
- - model_name: deepseek-fast
- litellm_params:
- model: hyperbolic/deepseek-ai/DeepSeek-V3
- api_key: os.environ/HYPERBOLIC_API_KEY
-
- - model_name: qwen-coder
- litellm_params:
- model: hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct
- api_key: os.environ/HYPERBOLIC_API_KEY
-
- - model_name: deepseek-reasoning
- litellm_params:
- model: hyperbolic/deepseek-ai/DeepSeek-R1
- api_key: os.environ/HYPERBOLIC_API_KEY
-```
-
-Start your LiteLLM Proxy server:
-
-```bash showLineNumbers title="Start LiteLLM Proxy"
-litellm --config config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-
-
-
-```python showLineNumbers title="Hyperbolic via Proxy - Non-streaming"
-from openai import OpenAI
-
-# Initialize client with your proxy URL
-client = OpenAI(
- base_url="http://localhost:4000", # Your proxy URL
- api_key="your-proxy-api-key" # Your proxy API key
-)
-
-# Non-streaming response
-response = client.chat.completions.create(
- model="deepseek-fast",
- messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}]
-)
-
-print(response.choices[0].message.content)
-```
-
-```python showLineNumbers title="Hyperbolic via Proxy - Streaming"
-from openai import OpenAI
-
-# Initialize client with your proxy URL
-client = OpenAI(
- base_url="http://localhost:4000", # Your proxy URL
- api_key="your-proxy-api-key" # Your proxy API key
-)
-
-# Streaming response
-response = client.chat.completions.create(
- model="qwen-coder",
- messages=[{"role": "user", "content": "Write a Python function to sort a list"}],
- stream=True
-)
-
-for chunk in response:
- if chunk.choices[0].delta.content is not None:
- print(chunk.choices[0].delta.content, end="")
-```
-
-
-
-
-
-```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK"
-import litellm
-
-# Configure LiteLLM to use your proxy
-response = litellm.completion(
- model="litellm_proxy/deepseek-fast",
- messages=[{"role": "user", "content": "What are the benefits of renewable energy?"}],
- api_base="http://localhost:4000",
- api_key="your-proxy-api-key"
-)
-
-print(response.choices[0].message.content)
-```
-
-```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK Streaming"
-import litellm
-
-# Configure LiteLLM to use your proxy with streaming
-response = litellm.completion(
- model="litellm_proxy/qwen-coder",
- messages=[{"role": "user", "content": "Implement a binary search algorithm"}],
- api_base="http://localhost:4000",
- api_key="your-proxy-api-key",
- stream=True
-)
-
-for chunk in response:
- if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
- print(chunk.choices[0].delta.content, end="")
-```
-
-
-
-
-
-```bash showLineNumbers title="Hyperbolic via Proxy - cURL"
-curl http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer your-proxy-api-key" \
- -d '{
- "model": "deepseek-fast",
- "messages": [{"role": "user", "content": "What is machine learning?"}]
- }'
-```
-
-```bash showLineNumbers title="Hyperbolic via Proxy - cURL Streaming"
-curl http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer your-proxy-api-key" \
- -d '{
- "model": "qwen-coder",
- "messages": [{"role": "user", "content": "Write a REST API in Python"}],
- "stream": true
- }'
-```
-
-
-
-
-For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
-
-## Supported OpenAI Parameters
-
-Hyperbolic supports the following OpenAI-compatible parameters:
-
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
-| `model` | string | **Required**. Model ID (e.g., deepseek-ai/DeepSeek-V3, Qwen/Qwen2.5-72B-Instruct) |
-| `stream` | boolean | Optional. Enable streaming responses |
-| `temperature` | float | Optional. Sampling temperature (0.0 to 2.0) |
-| `top_p` | float | Optional. Nucleus sampling parameter |
-| `max_tokens` | integer | Optional. Maximum tokens to generate |
-| `frequency_penalty` | float | Optional. Penalize frequent tokens |
-| `presence_penalty` | float | Optional. Penalize tokens based on presence |
-| `stop` | string/array | Optional. Stop sequences |
-| `n` | integer | Optional. Number of completions to generate |
-| `tools` | array | Optional. List of available tools/functions |
-| `tool_choice` | string/object | Optional. Control tool/function calling |
-| `response_format` | object | Optional. Response format specification |
-| `seed` | integer | Optional. Random seed for reproducibility |
-| `user` | string | Optional. User identifier |
-
-## Advanced Usage
-
-### Custom API Base
-
-If you're using a custom Hyperbolic deployment:
-
-```python showLineNumbers title="Custom API Base"
-import litellm
-
-response = litellm.completion(
- model="hyperbolic/deepseek-ai/DeepSeek-V3",
- messages=[{"role": "user", "content": "Hello"}],
- api_base="https://your-custom-hyperbolic-endpoint.com/v1",
- api_key="your-api-key"
-)
-```
-
-### Rate Limits
-
-Hyperbolic offers different tiers:
-- **Basic**: 60 requests per minute (RPM)
-- **Pro**: 600 RPM
-- **Enterprise**: Custom limits
-
-## Pricing
-
-Hyperbolic offers competitive pay-as-you-go pricing with no hidden fees or long-term commitments. See the model table above for specific pricing per million tokens.
-
-### Precision Options
-- **BF16**: Best precision and performance, suitable for tasks where accuracy is critical
-- **FP8**: Optimized for efficiency and speed, ideal for high-throughput applications at lower cost
-
-## Additional Resources
-
-- [Hyperbolic Official Documentation](https://docs.hyperbolic.xyz)
-- [Hyperbolic Dashboard](https://app.hyperbolic.ai)
-- [API Reference](https://docs.hyperbolic.xyz/docs/rest-api)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/lambda_ai.md b/docs/my-website/docs/providers/lambda_ai.md
deleted file mode 100644
index 91800faab7..0000000000
--- a/docs/my-website/docs/providers/lambda_ai.md
+++ /dev/null
@@ -1,280 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Lambda AI
-
-## Overview
-
-| Property | Details |
-|-------|-------|
-| Description | Lambda AI provides access to a wide range of open-source language models through their cloud GPU infrastructure, optimized for inference at scale. |
-| Provider Route on LiteLLM | `lambda_ai/` |
-| Link to Provider Doc | [Lambda AI API Documentation ↗](https://docs.lambda.ai/api) |
-| Base URL | `https://api.lambda.ai/v1` |
-| Supported Operations | [`/chat/completions`](#sample-usage) |
-
-
-
-
-https://docs.lambda.ai/api
-
-**We support ALL Lambda AI models, just set `lambda_ai/` as a prefix when sending completion requests**
-
-## Available Models
-
-Lambda AI offers a diverse selection of state-of-the-art open-source models:
-
-### Large Language Models
-
-| Model | Description | Context Window |
-|-------|-------------|----------------|
-| `lambda_ai/llama3.3-70b-instruct-fp8` | Llama 3.3 70B with FP8 quantization | 8,192 tokens |
-| `lambda_ai/llama3.1-405b-instruct-fp8` | Llama 3.1 405B with FP8 quantization | 8,192 tokens |
-| `lambda_ai/llama3.1-70b-instruct-fp8` | Llama 3.1 70B with FP8 quantization | 8,192 tokens |
-| `lambda_ai/llama3.1-8b-instruct` | Llama 3.1 8B instruction-tuned | 8,192 tokens |
-| `lambda_ai/llama3.1-nemotron-70b-instruct-fp8` | Llama 3.1 Nemotron 70B | 8,192 tokens |
-
-### DeepSeek Models
-
-| Model | Description | Context Window |
-|-------|-------------|----------------|
-| `lambda_ai/deepseek-llama3.3-70b` | DeepSeek Llama 3.3 70B | 8,192 tokens |
-| `lambda_ai/deepseek-r1-0528` | DeepSeek R1 0528 | 8,192 tokens |
-| `lambda_ai/deepseek-r1-671b` | DeepSeek R1 671B | 8,192 tokens |
-| `lambda_ai/deepseek-v3-0324` | DeepSeek V3 0324 | 8,192 tokens |
-
-### Hermes Models
-
-| Model | Description | Context Window |
-|-------|-------------|----------------|
-| `lambda_ai/hermes3-405b` | Hermes 3 405B | 8,192 tokens |
-| `lambda_ai/hermes3-70b` | Hermes 3 70B | 8,192 tokens |
-| `lambda_ai/hermes3-8b` | Hermes 3 8B | 8,192 tokens |
-
-### Coding Models
-
-| Model | Description | Context Window |
-|-------|-------------|----------------|
-| `lambda_ai/qwen25-coder-32b-instruct` | Qwen 2.5 Coder 32B | 8,192 tokens |
-| `lambda_ai/qwen3-32b-fp8` | Qwen 3 32B with FP8 | 8,192 tokens |
-
-### Vision Models
-
-| Model | Description | Context Window |
-|-------|-------------|----------------|
-| `lambda_ai/llama3.2-11b-vision-instruct` | Llama 3.2 11B with vision capabilities | 8,192 tokens |
-
-### Specialized Models
-
-| Model | Description | Context Window |
-|-------|-------------|----------------|
-| `lambda_ai/llama-4-maverick-17b-128e-instruct-fp8` | Llama 4 Maverick with 128k context | 131,072 tokens |
-| `lambda_ai/llama-4-scout-17b-16e-instruct` | Llama 4 Scout with 16k context | 16,384 tokens |
-| `lambda_ai/lfm-40b` | LFM 40B model | 8,192 tokens |
-| `lambda_ai/lfm-7b` | LFM 7B model | 8,192 tokens |
-
-## Required Variables
-
-```python showLineNumbers title="Environment Variables"
-os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
-```
-
-## Usage - LiteLLM Python SDK
-
-### Non-streaming
-
-```python showLineNumbers title="Lambda AI Non-streaming Completion"
-import os
-import litellm
-from litellm import completion
-
-os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
-
-messages = [{"content": "Hello, how are you?", "role": "user"}]
-
-# Lambda AI call
-response = completion(
- model="lambda_ai/llama3.1-8b-instruct",
- messages=messages
-)
-
-print(response)
-```
-
-### Streaming
-
-```python showLineNumbers title="Lambda AI Streaming Completion"
-import os
-import litellm
-from litellm import completion
-
-os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
-
-messages = [{"content": "Write a short story about AI", "role": "user"}]
-
-# Lambda AI call with streaming
-response = completion(
- model="lambda_ai/llama3.1-70b-instruct-fp8",
- messages=messages,
- stream=True
-)
-
-for chunk in response:
- print(chunk)
-```
-
-### Vision/Multimodal Support
-
-The Llama 3.2 Vision model supports image inputs:
-
-```python showLineNumbers title="Lambda AI Vision/Multimodal"
-import os
-import litellm
-from litellm import completion
-
-os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
-
-messages = [{
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": "What's in this image?"
- },
- {
- "type": "image_url",
- "image_url": {
- "url": "https://example.com/image.jpg"
- }
- }
- ]
-}]
-
-# Lambda AI vision model call
-response = completion(
- model="lambda_ai/llama3.2-11b-vision-instruct",
- messages=messages
-)
-
-print(response)
-```
-
-### Function Calling
-
-Lambda AI models support function calling:
-
-```python showLineNumbers title="Lambda AI Function Calling"
-import os
-import litellm
-from litellm import completion
-
-os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
-
-# Define tools
-tools = [{
- "type": "function",
- "function": {
- "name": "get_weather",
- "description": "Get the current weather in a location",
- "parameters": {
- "type": "object",
- "properties": {
- "location": {
- "type": "string",
- "description": "The city and state, e.g. San Francisco, CA"
- }
- },
- "required": ["location"]
- }
- }
-}]
-
-messages = [{"role": "user", "content": "What's the weather in Boston?"}]
-
-# Lambda AI call with function calling
-response = completion(
- model="lambda_ai/hermes3-70b",
- messages=messages,
- tools=tools,
- tool_choice="auto"
-)
-
-print(response)
-```
-
-## Usage - LiteLLM Proxy Server
-
-```yaml showLineNumbers title="config.yaml"
-model_list:
- - model_name: llama-8b
- litellm_params:
- model: lambda_ai/llama3.1-8b-instruct
- api_key: os.environ/LAMBDA_API_KEY
- - model_name: deepseek-70b
- litellm_params:
- model: lambda_ai/deepseek-llama3.3-70b
- api_key: os.environ/LAMBDA_API_KEY
- - model_name: hermes-405b
- litellm_params:
- model: lambda_ai/hermes3-405b
- api_key: os.environ/LAMBDA_API_KEY
- - model_name: qwen-coder
- litellm_params:
- model: lambda_ai/qwen25-coder-32b-instruct
- api_key: os.environ/LAMBDA_API_KEY
-```
-
-## Custom API Base
-
-If you need to use a custom API base URL:
-
-```python showLineNumbers title="Custom API Base"
-import os
-import litellm
-from litellm import completion
-
-# Using environment variable
-os.environ["LAMBDA_API_BASE"] = "https://custom.lambda-api.com/v1"
-os.environ["LAMBDA_API_KEY"] = "" # your API key
-
-# Or pass directly
-response = completion(
- model="lambda_ai/llama3.1-8b-instruct",
- messages=[{"content": "Hello!", "role": "user"}],
- api_base="https://custom.lambda-api.com/v1",
- api_key="your-api-key"
-)
-```
-
-## Supported OpenAI Parameters
-
-Lambda AI supports all standard OpenAI parameters since it's fully OpenAI-compatible:
-
-- `temperature`
-- `max_tokens`
-- `top_p`
-- `frequency_penalty`
-- `presence_penalty`
-- `stop`
-- `n`
-- `stream`
-- `tools`
-- `tool_choice`
-- `response_format`
-- `seed`
-- `user`
-- `logit_bias`
-
-Example with parameters:
-
-```python showLineNumbers title="Lambda AI with Parameters"
-response = completion(
- model="lambda_ai/hermes3-405b",
- messages=[{"content": "Explain quantum computing", "role": "user"}],
- temperature=0.7,
- max_tokens=500,
- top_p=0.9,
- frequency_penalty=0.2,
- presence_penalty=0.1
-)
-```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/moonshot.md b/docs/my-website/docs/providers/moonshot.md
index 2e00bae355..ee4f04bd7a 100644
--- a/docs/my-website/docs/providers/moonshot.md
+++ b/docs/my-website/docs/providers/moonshot.md
@@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem';
| Description | Moonshot AI provides large language models including the moonshot-v1 series and kimi models. |
| Provider Route on LiteLLM | `moonshot/` |
| Link to Provider Doc | [Moonshot AI ↗](https://platform.moonshot.ai/) |
-| Base URL | `https://api.moonshot.ai/` |
+| Base URL | `https://api.moonshot.cn/` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
@@ -26,18 +26,6 @@ https://platform.moonshot.ai/
os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key
```
-**ATTENTION:**
-
-Moonshot AI offers two distinct API endpoints: a global one and a China-specific one.
-- Global API Base URL: `https://api.moonshot.ai/v1` (This is the one currently implemented)
-- China API Base URL: `https://api.moonshot.cn/v1`
-
-You can overwrite the base url with:
-
-```
-os.environ["MOONSHOT_API_BASE"] = "https://api.moonshot.cn/v1"
-```
-
## Usage - LiteLLM Python SDK
### Non-streaming
diff --git a/docs/my-website/docs/providers/morph.md b/docs/my-website/docs/providers/morph.md
deleted file mode 100644
index e49c60b566..0000000000
--- a/docs/my-website/docs/providers/morph.md
+++ /dev/null
@@ -1,123 +0,0 @@
-# Morph
-
-LiteLLM supports all models on [Morph](https://morphllm.com)
-
-## Overview
-
-Morph provides specialized AI models designed for agentic workflows, particularly excelling at precise code editing and manipulation. Their "Apply" models enable targeted code changes without full file rewrites, making them ideal for AI agents that need to make intelligent, context-aware code modifications.
-
-## API Key
-```python
-import os
-os.environ["MORPH_API_KEY"] = "your-api-key"
-```
-
-## Sample Usage
-
-```python
-from litellm import completion
-
-# set env variable
-os.environ["MORPH_API_KEY"] = "your-api-key"
-
-messages = [
- {"role": "user", "content": "Write a Python function to calculate factorial"}
-]
-
-## Morph v3 Fast - Optimized for speed
-response = completion(
- model="morph/morph-v3-fast",
- messages=messages,
-)
-print(response)
-
-## Morph v3 Large - Most capable model
-response = completion(
- model="morph/morph-v3-large",
- messages=messages,
-)
-print(response)
-```
-
-## Sample Usage - Streaming
-```python
-from litellm import completion
-
-# set env variable
-os.environ["MORPH_API_KEY"] = "your-api-key"
-
-messages = [
- {"role": "user", "content": "Write a Python function to calculate factorial"}
-]
-
-## Morph v3 Fast with streaming
-response = completion(
- model="morph/morph-v3-fast",
- messages=messages,
- stream=True,
-)
-
-for chunk in response:
- print(chunk)
-```
-
-## Supported Models
-
-| Model Name | Function Call | Description | Context Window |
-|--------------------------|--------------------------------------------|-----------------------|----------------|
-| morph-v3-fast | `completion('morph/morph-v3-fast', messages)` | Fastest model, optimized for quick responses | 16k tokens |
-| morph-v3-large | `completion('morph/morph-v3-large', messages)` | Most capable model for complex tasks | 16k tokens |
-
-## Usage - LiteLLM Proxy Server
-
-Here's how to use Morph with the LiteLLM Proxy Server:
-
-1. Save API key in your environment
-```bash
-export MORPH_API_KEY="your-api-key"
-```
-
-2. Add model to config.yaml
-```yaml
-model_list:
- - model_name: morph-v3-fast
- litellm_params:
- model: morph/morph-v3-fast
-
- - model_name: morph-v3-large
- litellm_params:
- model: morph/morph-v3-large
-```
-
-3. Start the proxy server
-```bash
-litellm --config config.yaml
-```
-
-## Advanced Usage
-
-### Setting API Base
-```python
-import litellm
-
-# set custom api base
-response = completion(
- model="morph/morph-v3-large",
- messages=[{"role": "user", "content": "Hello, world!"}],
- api_base="https://api.morphllm.com/v1"
-)
-print(response)
-```
-
-### Setting API Key
-```python
-import litellm
-
-# set api key via completion
-response = completion(
- model="morph/morph-v3-large",
- messages=[{"role": "user", "content": "Hello, world!"}],
- api_key="your-api-key"
-)
-print(response)
-```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/recraft.md b/docs/my-website/docs/providers/recraft.md
deleted file mode 100644
index dd4a91ddcf..0000000000
--- a/docs/my-website/docs/providers/recraft.md
+++ /dev/null
@@ -1,161 +0,0 @@
-# Recraft
-https://www.recraft.ai/
-
-## Overview
-
-| Property | Details |
-|-------|-------|
-| Description | Recraft is an AI-powered design tool that generates high-quality images with precise control over style and content. |
-| Provider Route on LiteLLM | `recraft/` |
-| Link to Provider Doc | [Recraft ↗](https://www.recraft.ai/docs) |
-| Supported Operations | [`/images/generations`](#image-generation) |
-
-LiteLLM supports Recraft Image Generation calls.
-
-## API Base, Key
-```python
-# env variable
-os.environ['RECRAFT_API_KEY'] = "your-api-key"
-os.environ['RECRAFT_API_BASE'] = "https://external.api.recraft.ai" # [optional]
-```
-
-## Image Generation
-
-### Usage - LiteLLM Python SDK
-
-```python showLineNumbers
-from litellm import image_generation
-import os
-
-os.environ['RECRAFT_API_KEY'] = "your-api-key"
-
-# recraft image generation call
-response = image_generation(
- model="recraft/recraftv3",
- prompt="A beautiful sunset over a calm ocean",
-)
-print(response)
-```
-
-### Usage - LiteLLM Proxy Server
-
-#### 1. Setup config.yaml
-
-```yaml showLineNumbers
-model_list:
- - model_name: recraft-v3
- litellm_params:
- model: recraft/recraftv3
- api_key: os.environ/RECRAFT_API_KEY
- model_info:
- mode: image_generation
-
-general_settings:
- master_key: sk-1234
-```
-
-#### 2. Start the proxy
-
-```bash showLineNumbers
-litellm --config config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-#### 3. Test it
-
-```bash showLineNumbers
-curl --location 'http://0.0.0.0:4000/v1/images/generations' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer sk-1234' \
---data '{
- "model": "recraft-v3",
- "prompt": "A beautiful sunset over a calm ocean",
-}'
-```
-
-### Advanced Usage - With Additional Parameters
-
-```python showLineNumbers
-from litellm import image_generation
-import os
-
-os.environ['RECRAFT_API_KEY'] = "your-api-key"
-
-response = image_generation(
- model="recraft/recraftv3",
- prompt="A beautiful sunset over a calm ocean",
-)
-print(response)
-```
-
-### Supported Parameters
-
-Recraft supports the following OpenAI-compatible parameters:
-
-| Parameter | Type | Description | Example |
-|-----------|------|-------------|---------|
-| `n` | integer | Number of images to generate (1-4) | `1` |
-| `response_format` | string | Format of response (`url` or `b64_json`) | `"url"` |
-| `size` | string | Image dimensions | `"1024x1024"` |
-| `style` | string | Image style/artistic direction | `"realistic"` |
-
-### Using Non-OpenAI Parameters
-
-If you want to pass parameters that are not supported by OpenAI, you can pass them in your request body, LiteLLM will automatically route it to recraft.
-
-In this example we will pass `style_id` parameter to the recraft image generation call.
-
-**Usage with LiteLLM Python SDK**
-
-```python showLineNumbers
-from litellm import image_generation
-import os
-
-os.environ['RECRAFT_API_KEY'] = "your-api-key"
-
-response = image_generation(
- model="recraft/recraftv3",
- prompt="A beautiful sunset over a calm ocean",
- style_id="your-style-id",
-)
-```
-
-**Usage with LiteLLM Proxy Server + OpenAI Python SDK**
-
-```python showLineNumbers
-from openai import OpenAI
-import os
-
-os.environ['RECRAFT_API_KEY'] = "your-api-key"
-
-client = OpenAI(api_key=os.environ['RECRAFT_API_KEY'])
-
-response = client.images.generate(
- model="recraft/recraftv3",
- prompt="A beautiful sunset over a calm ocean",
- extra_body={
- "style_id": "your-style-id",
- },
-)
-print(response)
-```
-
-### Supported Image Generation Models
-
-**Note: All recraft models are supported by LiteLLM** Just pass the model name with `recraft/` and litellm will route it to recraft.
-
-| Model Name | Function Call |
-|------------|---------------|
-| recraftv3 | `image_generation(model="recraft/recraftv3", prompt="...")` |
-| recraftv2 | `image_generation(model="recraft/recraftv2", prompt="...")` |
-
-For more details on available models and features, see: https://www.recraft.ai/docs
-
-## API Key Setup
-
-Get your API key from [Recraft's website](https://www.recraft.ai/) and set it as an environment variable:
-
-```bash
-export RECRAFT_API_KEY="your-api-key"
-```
diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md
index c6e324f295..2600b179f6 100644
--- a/docs/my-website/docs/providers/vertex_partner.md
+++ b/docs/my-website/docs/providers/vertex_partner.md
@@ -313,22 +313,11 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
## VertexAI Mistral API
[**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137)
-
-**LiteLLM Supports all Vertex AI Mistral Models.** Ensure you use the `vertex_ai/mistral-` prefix for all Vertex AI Mistral models.
-
-Overview
-
-| Property | Details |
-|----------|---------|
-| Provider Route | `vertex_ai/mistral-{MODEL}` |
-| Vertex Documentation | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
| Model Name | Function Call |
|------------------|--------------------------------------|
| mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` |
| mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` |
-| mistral-small-2503 | `completion('vertex_ai/mistral-small-2503', messages)` |
-| mistral-large-2411 | `completion('vertex_ai/mistral-large-2411', messages)` |
| mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` |
| codestral@latest | `completion('vertex_ai/codestral@latest', messages)` |
| codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` |
diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md
index 8e802f5c68..b1dfac9ac2 100644
--- a/docs/my-website/docs/proxy/config_settings.md
+++ b/docs/my-website/docs/proxy/config_settings.md
@@ -189,7 +189,6 @@ general_settings:
| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
-| proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
| allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. |
@@ -623,7 +622,6 @@ router_settings:
| PROXY_ADMIN_ID | Admin identifier for proxy server
| PROXY_BASE_URL | Base URL for proxy service
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
-| PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597
| PROXY_LOGOUT_URL | URL for logging out of the proxy service
diff --git a/docs/my-website/docs/proxy/guardrails/model_armor.md b/docs/my-website/docs/proxy/guardrails/model_armor.md
deleted file mode 100644
index a7463a8eee..0000000000
--- a/docs/my-website/docs/proxy/guardrails/model_armor.md
+++ /dev/null
@@ -1,93 +0,0 @@
-import Image from '@theme/IdealImage';
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Google Cloud Model Armor
-
-LiteLLM supports Google Cloud Model Armor guardrails via the [Model Armor API](https://cloud.google.com/security-command-center/docs/model-armor-overview).
-
-
-## Supported Guardrails
-
-- [Model Armor Templates](https://cloud.google.com/security-command-center/docs/manage-model-armor-templates) - Content sanitization and blocking based on configured templates
-
-## Quick Start
-### 1. Define Guardrails on your LiteLLM config.yaml
-
-Define your guardrails under the `guardrails` section
-
-```yaml
-model_list:
- - model_name: gpt-3.5-turbo
- litellm_params:
- model: openai/gpt-3.5-turbo
- api_key: os.environ/OPENAI_API_KEY
-
-guardrails:
- - guardrail_name: model-armor-shield
- litellm_params:
- guardrail: model_armor
- mode: [pre_call, post_call] # Run on both input and output
- template_id: "your-template-id" # Required: Your Model Armor template ID
- project_id: "your-project-id" # Your GCP project ID
- location: "us-central1" # GCP location (default: us-central1)
- credentials: "path/to/credentials.json" # Path to service account key
- mask_request_content: true # Enable request content masking
- mask_response_content: true # Enable response content masking
- fail_on_error: true # Fail request if Model Armor errors (default: true)
- default_on: true # Run by default for all requests
-```
-
-#### Supported values for `mode`
-
-- `pre_call` Run **before** LLM call, on **input**
-- `post_call` Run **after** LLM call, on **input & output**
-
-### 2. Start LiteLLM Gateway
-
-
-```shell
-litellm --config config.yaml --detailed_debug
-```
-
-### 3. Test request
-
-**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
-
-```shell
-curl -i http://localhost:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
- -d '{
- "model": "gpt-3.5-turbo",
- "messages": [
- {"role": "user", "content": "Hi, my email is test@example.com"}
- ],
- "guardrails": ["model-armor-shield"]
- }'
-```
-
-## Supported Params
-
-### Common Params
-
-- `api_key` - str - Google Cloud service account credentials (optional if using ADC)
-- `api_base` - str - Custom Model Armor API endpoint (optional)
-- `default_on` - bool - Whether to run the guardrail by default. Default is `false`.
-- `mode` - Union[str, list[str]] - Mode to run the guardrail. Either `pre_call` or `post_call`. Default is `pre_call`.
-
-### Model Armor Specific
-
-- `template_id` - str - The ID of your Model Armor template (required)
-- `project_id` - str - Google Cloud project ID (defaults to credentials project)
-- `location` - str - Google Cloud location/region. Default is `us-central1`
-- `credentials` - Union[str, dict] - Path to service account JSON file or credentials dictionary
-- `api_endpoint` - str - Custom API endpoint for Model Armor (optional)
-- `fail_on_error` - bool - Whether to fail requests if Model Armor encounters errors. Default is `true`
-- `mask_request_content` - bool - Enable masking of sensitive content in requests. Default is `false`
-- `mask_response_content` - bool - Enable masking of sensitive content in responses. Default is `false`
-
-
-## Further Reading
-
-- [Control Guardrails per API Key](./quick_start#-control-guardrails-per-api-key)
\ No newline at end of file
diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json
index 232bbf25c2..293124bdfd 100644
--- a/docs/my-website/package-lock.json
+++ b/docs/my-website/package-lock.json
@@ -9918,9 +9918,9 @@
}
},
"node_modules/form-data": {
- "version": "4.0.4",
- "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
- "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.3.tgz",
+ "integrity": "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
diff --git a/docs/my-website/package.json b/docs/my-website/package.json
index 24d212ea2c..d6d35d3134 100644
--- a/docs/my-website/package.json
+++ b/docs/my-website/package.json
@@ -47,7 +47,6 @@
"node": ">=16.14"
},
"overrides": {
- "webpack-dev-server": ">=5.2.1",
- "form-data": ">=4.0.4"
+ "webpack-dev-server": ">=5.2.1"
}
}
diff --git a/docs/my-website/release_notes/v1.74.7/index.md b/docs/my-website/release_notes/v1.74.7/index.md
index 41e88824e0..26a09542ae 100644
--- a/docs/my-website/release_notes/v1.74.7/index.md
+++ b/docs/my-website/release_notes/v1.74.7/index.md
@@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
-ghcr.io/berriai/litellm:v1.74.7.rc.1
+ghcr.io/berriai/litellm:v1.74.7
```
``` showLineNumbers title="pip install litellm"
-pip install litellm==1.74.7rc1
+pip install litellm==1.74.7
```
diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js
index 1fe19fe979..b78410448c 100644
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@@ -39,7 +39,6 @@ const sidebars = {
"proxy/guardrails/lasso_security",
"proxy/guardrails/guardrails_ai",
"proxy/guardrails/lakera_ai",
- "proxy/guardrails/model_armor",
"proxy/guardrails/openai_moderation",
"proxy/guardrails/pangea",
"proxy/guardrails/pii_masking_v2",
@@ -412,7 +411,6 @@ const sidebars = {
"providers/huggingface_rerank",
]
},
- "providers/hyperbolic",
"providers/databricks",
"providers/deepgram",
"providers/watsonx",
@@ -446,12 +444,9 @@ const sidebars = {
"providers/github_copilot",
"providers/ai21",
"providers/nlp_cloud",
- "providers/recraft",
"providers/replicate",
"providers/togetherai",
"providers/v0",
- "providers/morph",
- "providers/lambda_ai",
"providers/novita",
"providers/voyage",
"providers/jina_ai",
@@ -628,7 +623,6 @@ const sidebars = {
"projects/llm_cord",
"projects/pgai",
"projects/GPTLocalhost",
- "projects/HolmesGPT"
],
},
"extras/code_quality",
diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index 6edd198cd8..d8b8efeef4 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -83,25 +83,15 @@ class CheckBatchCost:
)
continue
- verbose_proxy_logger.info(
- f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}"
+ response = await self.llm_router.aretrieve_batch(
+ model=model_id,
+ batch_id=batch_id,
+ litellm_metadata={
+ "user_api_key_user_id": job.created_by or "default-user-id",
+ "batch_ignore_default_logging": True,
+ },
)
- try:
- response = await self.llm_router.aretrieve_batch(
- model=model_id,
- batch_id=batch_id,
- litellm_metadata={
- "user_api_key_user_id": job.created_by or "default-user-id",
- "batch_ignore_default_logging": True,
- },
- )
- except Exception as e:
- verbose_proxy_logger.info(
- f"Skipping job {unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
- )
- continue
-
## RETRIEVE THE BATCH JOB OUTPUT FILE
managed_files_obj = cast(
Optional[_PROXY_LiteLLMManagedFiles],
@@ -112,9 +102,6 @@ class CheckBatchCost:
and response.output_file_id is not None
and managed_files_obj is not None
):
- verbose_proxy_logger.info(
- f"Batch ID: {batch_id} is complete, tracking cost and usage"
- )
# track cost
model_file_id_mapping = {
response.output_file_id: {model_id: response.output_file_id}
diff --git a/litellm/__init__.py b/litellm/__init__.py
index c056e66726..99a5454ace 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -144,22 +144,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
-gcs_pub_sub_use_v1: Optional[
- bool
-] = False # if you want to use v1 gcs pubsub logged payload
-generic_api_use_v1: Optional[
- bool
-] = False # if you want to use v1 generic api logged payload
+gcs_pub_sub_use_v1: Optional[bool] = (
+ False # if you want to use v1 gcs pubsub logged payload
+)
+generic_api_use_v1: Optional[bool] = (
+ False # if you want to use v1 generic api logged payload
+)
argilla_transformation_object: Optional[Dict[str, Any]] = None
-_async_input_callback: List[
- Union[str, Callable, CustomLogger]
-] = [] # internal variable - async custom callbacks are routed here.
-_async_success_callback: List[
- Union[str, Callable, CustomLogger]
-] = [] # internal variable - async custom callbacks are routed here.
-_async_failure_callback: List[
- Union[str, Callable, CustomLogger]
-] = [] # internal variable - async custom callbacks are routed here.
+_async_input_callback: List[Union[str, Callable, CustomLogger]] = (
+ []
+) # internal variable - async custom callbacks are routed here.
+_async_success_callback: List[Union[str, Callable, CustomLogger]] = (
+ []
+) # internal variable - async custom callbacks are routed here.
+_async_failure_callback: List[Union[str, Callable, CustomLogger]] = (
+ []
+) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
@@ -167,18 +167,18 @@ log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
filter_invalid_headers: Optional[bool] = False
-add_user_information_to_llm_headers: Optional[
- bool
-] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
+add_user_information_to_llm_headers: Optional[bool] = (
+ None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
+)
store_audit_logs = False # Enterprise feature, allow users to see audit logs
### end of callbacks #############
-email: Optional[
- str
-] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
-token: Optional[
- str
-] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+email: Optional[str] = (
+ None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+)
+token: Optional[str] = (
+ None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+)
telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
@@ -266,11 +266,15 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
enable_caching_on_provider_specific_optional_params: bool = (
False # feature-flag for caching on optional params - e.g. 'top_k'
)
-caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
-caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
-cache: Optional[
- Cache
-] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
+caching: bool = (
+ False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+)
+caching_with_models: bool = (
+ False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+)
+cache: Optional[Cache] = (
+ None # cache object <- use this - https://docs.litellm.ai/docs/caching
+)
default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
@@ -278,9 +282,9 @@ model_alias_map: Dict[str, str] = {}
model_group_alias_map: Dict[str, str] = {}
model_group_settings: Optional["ModelGroupSettings"] = None
max_budget: float = 0.0 # set the max budget across all providers
-budget_duration: Optional[
- str
-] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
+budget_duration: Optional[str] = (
+ None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
+)
default_soft_budget: float = (
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
)
@@ -289,11 +293,15 @@ forward_traceparent_to_llm_provider: bool = False
_current_cost = 0.0 # private variable, used if max budget is set
error_logs: Dict = {}
-add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
+add_function_to_prompt: bool = (
+ False # if function calling not supported by api, append function call details to system prompt
+)
client_session: Optional[httpx.Client] = None
aclient_session: Optional[httpx.AsyncClient] = None
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
-model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+model_cost_map_url: str = (
+ "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+)
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
@@ -321,7 +329,9 @@ prometheus_metrics_config: Optional[List] = None
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
-disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
+disable_copilot_system_to_assistant: bool = (
+ False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
+)
public_model_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION #####
@@ -329,13 +339,17 @@ priority_reservation: Optional[Dict[str, float]] = None
######## Networking Settings ########
-use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
+use_aiohttp_transport: bool = (
+ True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
+)
aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
disable_aiohttp_trust_env: bool = (
False # When False, aiohttp will respect HTTP(S)_PROXY env vars
)
-force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
+force_ipv4: bool = (
+ False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
+)
module_level_aclient = AsyncHTTPHandler(
timeout=request_timeout, client_alias="module level aclient"
)
@@ -349,13 +363,13 @@ fallbacks: Optional[List] = None
context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
-num_retries_per_request: Optional[
- int
-] = None # for the request overall (incl. fallbacks + model retries)
+num_retries_per_request: Optional[int] = (
+ None # for the request overall (incl. fallbacks + model retries)
+)
####### SECRET MANAGERS #####################
-secret_manager_client: Optional[
- Any
-] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
+secret_manager_client: Optional[Any] = (
+ None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
+)
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional[KeyManagementSystem] = None
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
@@ -489,10 +503,6 @@ elevenlabs_models: List = []
dashscope_models: List = []
moonshot_models: List = []
v0_models: List = []
-morph_models: List = []
-lambda_ai_models: List = []
-hyperbolic_models: List = []
-recraft_models: List = []
def is_bedrock_pricing_only_model(key: str) -> bool:
"""
@@ -673,14 +683,6 @@ def add_known_models():
moonshot_models.append(key)
elif value.get("litellm_provider") == "v0":
v0_models.append(key)
- elif value.get("litellm_provider") == "morph":
- morph_models.append(key)
- elif value.get("litellm_provider") == "lambda_ai":
- lambda_ai_models.append(key)
- elif value.get("litellm_provider") == "hyperbolic":
- hyperbolic_models.append(key)
- elif value.get("litellm_provider") == "recraft":
- recraft_models.append(key)
add_known_models()
@@ -766,9 +768,6 @@ model_list = (
+ dashscope_models
+ moonshot_models
+ v0_models
- + morph_models
- + lambda_ai_models
- + recraft_models
)
model_list_set = set(model_list)
@@ -837,10 +836,6 @@ models_by_provider: dict = {
"dashscope": dashscope_models,
"moonshot": moonshot_models,
"v0": v0_models,
- "morph": morph_models,
- "lambda_ai": lambda_ai_models,
- "hyperbolic": hyperbolic_models,
- "recraft": recraft_models,
}
# mapping for those models which have larger equivalents
@@ -1161,9 +1156,6 @@ from .llms.nebius.chat.transformation import NebiusConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig
from .llms.v0.chat.transformation import V0ChatConfig
-from .llms.morph.chat.transformation import MorphChatConfig
-from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
-from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
from .main import * # type: ignore
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
@@ -1222,12 +1214,12 @@ from .types.llms.custom_llm import CustomLLMItem
from .types.utils import GenericStreamingChunk
custom_provider_map: List[CustomLLMItem] = []
-_custom_providers: List[
- str
-] = [] # internal helper util, used to track names of custom providers
-disable_hf_tokenizer_download: Optional[
- bool
-] = None # disable huggingface tokenizer download. Defaults to openai clk100
+_custom_providers: List[str] = (
+ []
+) # internal helper util, used to track names of custom providers
+disable_hf_tokenizer_download: Optional[bool] = (
+ None # disable huggingface tokenizer download. Defaults to openai clk100
+)
global_disable_no_log_param: bool = False
### PASSTHROUGH ###
diff --git a/litellm/constants.py b/litellm/constants.py
index 970b16617f..afdd95385c 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -279,8 +279,6 @@ LITELLM_CHAT_PROVIDERS = [
"dashscope",
"moonshot",
"v0",
- "morph",
- "lambda_ai",
]
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@@ -410,9 +408,6 @@ openai_compatible_endpoints: List = [
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"https://api.moonshot.ai/v1",
"https://api.v0.dev/v1",
- "https://api.morphllm.com/v1",
- "https://api.lambda.ai/v1",
- "https://api.hyperbolic.xyz/v1",
]
@@ -451,9 +446,6 @@ openai_compatible_providers: List = [
"dashscope",
"moonshot",
"v0",
- "morph",
- "lambda_ai",
- "hyperbolic",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@@ -466,9 +458,6 @@ openai_text_completion_compatible_providers: List = (
"nebius",
"dashscope",
"moonshot",
- "v0",
- "lambda_ai",
- "hyperbolic",
]
)
_openai_like_providers: List = [
@@ -819,7 +808,6 @@ DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int(
PROXY_BUDGET_RESCHEDULER_MIN_TIME = int(
os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
)
-PROXY_BATCH_POLLING_INTERVAL = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600))
PROXY_BUDGET_RESCHEDULER_MAX_TIME = int(
os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
)
diff --git a/litellm/images/main.py b/litellm/images/main.py
index 3a675a8168..cf9ea7a662 100644
--- a/litellm/images/main.py
+++ b/litellm/images/main.py
@@ -1,7 +1,7 @@
import asyncio
import contextvars
from functools import partial
-from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast, overload
+from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
@@ -14,11 +14,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
-from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.custom_llm import CustomLLM
#################### Initialize provider clients ####################
-llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@@ -28,8 +26,6 @@ from litellm.main import (
openai_image_variations,
vertex_image_generation,
)
-
-###########################################
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import ImageGenerationRequestQuality
@@ -82,20 +78,17 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
-
- response: Optional[ImageResponse] = None
- if isinstance(init_response, dict):
- response = ImageResponse(**init_response)
- elif isinstance(init_response, ImageResponse): ## CACHING SCENARIO
+ if isinstance(init_response, dict) or isinstance(
+ init_response, ImageResponse
+ ): ## CACHING SCENARIO
+ if isinstance(init_response, dict):
+ init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
-
- if response is None:
- raise ValueError(
- "Unable to get Image Response. Please pass a valid llm_provider."
- )
-
+ else:
+ # Call the synchronous function using run_in_executor
+ response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
@@ -108,54 +101,6 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
)
-# Overload for when aimg_generation=True (returns Coroutine)
-@overload
-def image_generation(
- prompt: str,
- model: Optional[str] = None,
- n: Optional[int] = None,
- quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
- response_format: Optional[str] = None,
- size: Optional[str] = None,
- style: Optional[str] = None,
- user: Optional[str] = None,
- input_fidelity: Optional[str] = None,
- timeout=600, # default to 10 minutes
- api_key: Optional[str] = None,
- api_base: Optional[str] = None,
- api_version: Optional[str] = None,
- custom_llm_provider=None,
- *,
- aimg_generation: Literal[True],
- **kwargs,
-) -> Coroutine[Any, Any, ImageResponse]:
- ...
-
-
-# Overload for when aimg_generation=False or not specified (returns ImageResponse)
-@overload
-def image_generation(
- prompt: str,
- model: Optional[str] = None,
- n: Optional[int] = None,
- quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
- response_format: Optional[str] = None,
- size: Optional[str] = None,
- style: Optional[str] = None,
- user: Optional[str] = None,
- input_fidelity: Optional[str] = None,
- timeout=600, # default to 10 minutes
- api_key: Optional[str] = None,
- api_base: Optional[str] = None,
- api_version: Optional[str] = None,
- custom_llm_provider=None,
- *,
- aimg_generation: Literal[False] = False,
- **kwargs,
-) -> ImageResponse:
- ...
-
-
@client
def image_generation( # noqa: PLR0915
prompt: str,
@@ -173,10 +118,7 @@ def image_generation( # noqa: PLR0915
api_version: Optional[str] = None,
custom_llm_provider=None,
**kwargs,
-) -> Union[
- ImageResponse,
- Coroutine[Any, Any, ImageResponse],
- ]:
+) -> ImageResponse:
"""
Maps the https://api.openai.com/v1/images/generations endpoint.
@@ -406,26 +348,6 @@ def image_generation( # noqa: PLR0915
api_base=api_base,
client=client,
)
- #########################################################
- # Providers using llm_http_handler
- #########################################################
- elif custom_llm_provider in (
- litellm.LlmProviders.RECRAFT,
- ):
- if image_generation_config is None:
- raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
-
- return llm_http_handler.image_generation_handler(
- model=model,
- prompt=prompt,
- image_generation_provider_config=image_generation_config,
- image_generation_optional_request_params=optional_params,
- custom_llm_provider=custom_llm_provider,
- litellm_params=litellm_params_dict,
- logging_obj=litellm_logging_obj,
- timeout=timeout,
- client=client,
- )
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider
diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py
index 4e0a2efb0c..84c25d4932 100644
--- a/litellm/litellm_core_utils/get_llm_provider_logic.py
+++ b/litellm/litellm_core_utils/get_llm_provider_logic.py
@@ -240,12 +240,6 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.v0.dev/v1":
custom_llm_provider = "v0"
dynamic_api_key = get_secret_str("V0_API_KEY")
- elif endpoint == "https://api.lambda.ai/v1":
- custom_llm_provider = "lambda_ai"
- dynamic_api_key = get_secret_str("LAMBDA_API_KEY")
- elif endpoint == "https://api.hyperbolic.xyz/v1":
- custom_llm_provider = "hyperbolic"
- dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception(
@@ -536,7 +530,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
# DataRobot is OpenAI compatible.
(
api_base,
- dynamic_api_key,
+ dynamic_api_key
) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
@@ -697,27 +691,6 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.V0ChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
- elif custom_llm_provider == "morph":
- (
- api_base,
- dynamic_api_key,
- ) = litellm.MorphChatConfig()._get_openai_compatible_provider_info(
- api_base, api_key
- )
- elif custom_llm_provider == "lambda_ai":
- (
- api_base,
- dynamic_api_key,
- ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info(
- api_base, api_key
- )
- elif custom_llm_provider == "hyperbolic":
- (
- api_base,
- dynamic_api_key,
- ) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
- api_base, api_key
- )
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))
diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py
index 620d2f0511..04cf66cf02 100644
--- a/litellm/llms/anthropic/chat/handler.py
+++ b/litellm/llms/anthropic/chat/handler.py
@@ -27,7 +27,6 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
- _get_httpx_client,
get_async_httpx_client,
)
from litellm.types.llms.anthropic import (
@@ -434,9 +433,7 @@ class AnthropicChatCompletion(BaseLLM):
else:
if client is None or not isinstance(client, HTTPHandler):
- client = _get_httpx_client(
- params={"timeout": timeout}
- )
+ client = HTTPHandler(timeout=timeout) # type: ignore
else:
client = client
diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py
index 0ed4627908..f2a8defe13 100644
--- a/litellm/llms/azure/common_utils.py
+++ b/litellm/llms/azure/common_utils.py
@@ -278,7 +278,6 @@ def get_azure_ad_token(
3. From username and password
4. From OIDC token
5. From a service principal with secret workflow
- 6. From DefaultAzureCredential
Args:
litellm_params: Dictionary containing authentication parameters
@@ -353,27 +352,18 @@ def get_azure_ad_token(
azure_tenant_id=tenant_id,
scope=scope,
)
- # Try to get token provider from service principal or DefaultAzureCredential
+ # Try to get token provider from service principal
elif (
azure_ad_token_provider is None
and litellm.enable_azure_ad_token_refresh is True
):
verbose_logger.debug(
- "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
+ "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
)
try:
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
except ValueError:
verbose_logger.debug("Azure AD Token Provider could not be used.")
-
- #########################################################
- # If litellm.enable_azure_ad_token_refresh is True and no other token provider is available,
- # try to get DefaultAzureCredential provider
- #########################################################
- if azure_ad_token_provider is None and azure_ad_token is None:
- azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider(
- scope=scope,
- )
# Execute the token provider to get the token if available
if azure_ad_token_provider and callable(azure_ad_token_provider):
@@ -397,38 +387,6 @@ def get_azure_ad_token(
class BaseAzureLLM(BaseOpenAILLM):
- @staticmethod
- def _try_get_default_azure_credential_provider(
- scope: str,
- ) -> Optional[Callable[[], str]]:
- """
- Try to get DefaultAzureCredential provider
-
- Args:
- scope: Azure scope for the token
-
- Returns:
- Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
- """
- from litellm.types.secret_managers.get_azure_ad_token_provider import (
- AzureCredentialType,
- )
-
- verbose_logger.debug(
- "Attempting to use DefaultAzureCredential for Azure Auth"
- )
-
- try:
- azure_ad_token_provider = get_azure_ad_token_provider(
- azure_scope=scope,
- azure_credential=AzureCredentialType.DefaultAzureCredential,
- )
- verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
- return azure_ad_token_provider
- except Exception as e:
- verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}")
- return None
-
def get_azure_openai_client(
self,
api_key: Optional[str],
diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py
index fc8db8c65c..134c95b1c8 100644
--- a/litellm/llms/base_llm/image_generation/transformation.py
+++ b/litellm/llms/base_llm/image_generation/transformation.py
@@ -3,12 +3,12 @@ from typing import TYPE_CHECKING, Any, List, Optional, Union
import httpx
-from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
-from litellm.types.utils import ImageResponse
+from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -18,23 +18,12 @@ else:
LiteLLMLoggingObj = Any
-class BaseImageGenerationConfig(ABC):
+class BaseImageGenerationConfig(BaseConfig, ABC):
@abstractmethod
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
pass
-
- @abstractmethod
- def map_openai_params(
- self,
- non_default_params: dict,
- optional_params: dict,
- model: str,
- drop_params: bool,
- ) -> dict:
- pass
-
def get_complete_url(
self,
@@ -75,10 +64,10 @@ class BaseImageGenerationConfig(ABC):
headers=headers,
)
- def transform_image_generation_request(
+ def transform_request(
self,
model: str,
- prompt: str,
+ messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
@@ -87,19 +76,20 @@ class BaseImageGenerationConfig(ABC):
"ImageVariationConfig implementa 'transform_request_image_variation' for image variation models"
)
- def transform_image_generation_response(
+ def transform_response(
self,
model: str,
raw_response: httpx.Response,
- model_response: ImageResponse,
+ model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
+ messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
- ) -> ImageResponse:
+ ) -> ModelResponse:
raise NotImplementedError(
"ImageVariationConfig implements 'transform_response_image_variation' for image variation models"
)
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 46fd866be2..f77f87507e 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -35,9 +35,6 @@ from litellm.llms.base_llm.google_genai.transformation import (
BaseGoogleGenAIGenerateContentConfig,
)
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
-from litellm.llms.base_llm.image_generation.transformation import (
- BaseImageGenerationConfig,
-)
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
@@ -2373,7 +2370,6 @@ class BaseLLMHTTPHandler:
BaseRerankConfig,
BaseResponsesAPIConfig,
BaseImageEditConfig,
- BaseImageGenerationConfig,
BaseVectorStoreConfig,
BaseGoogleGenAIGenerateContentConfig,
BaseAnthropicMessagesConfig,
@@ -2661,216 +2657,6 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
- def image_generation_handler(
- self,
- model: str,
- prompt: str,
- image_generation_provider_config: BaseImageGenerationConfig,
- image_generation_optional_request_params: Dict,
- custom_llm_provider: str,
- litellm_params: Dict,
- logging_obj: LiteLLMLoggingObj,
- timeout: Union[float, httpx.Timeout],
- extra_headers: Optional[Dict[str, Any]] = None,
- extra_body: Optional[Dict[str, Any]] = None,
- client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
- _is_async: bool = False,
- fake_stream: bool = False,
- litellm_metadata: Optional[Dict[str, Any]] = None,
- ) -> Union[
- ImageResponse,
- Coroutine[Any, Any, ImageResponse],
- ]:
- """
- Handles image generation requests.
- When _is_async=True, returns a coroutine instead of making the call directly.
- """
- if _is_async:
- # Return the async coroutine if called with _is_async=True
- return self.async_image_generation_handler(
- model=model,
- prompt=prompt,
- image_generation_provider_config=image_generation_provider_config,
- image_generation_optional_request_params=image_generation_optional_request_params,
- custom_llm_provider=custom_llm_provider,
- litellm_params=litellm_params,
- logging_obj=logging_obj,
- extra_headers=extra_headers,
- extra_body=extra_body,
- timeout=timeout,
- client=client if isinstance(client, AsyncHTTPHandler) else None,
- fake_stream=fake_stream,
- litellm_metadata=litellm_metadata,
- )
-
- if client is None or not isinstance(client, HTTPHandler):
- sync_httpx_client = _get_httpx_client(
- params={"ssl_verify": litellm_params.get("ssl_verify", None)}
- )
- else:
- sync_httpx_client = client
-
- headers = image_generation_provider_config.validate_environment(
- api_key=litellm_params.get("api_key", None),
- headers=image_generation_optional_request_params.get("extra_headers", {}) or {},
- model=model,
- messages=[],
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- )
-
- if extra_headers:
- headers.update(extra_headers)
-
- api_base = image_generation_provider_config.get_complete_url(
- model=model,
- api_base=litellm_params.get("api_base", None),
- api_key=litellm_params.get("api_key", None),
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- )
-
- data = image_generation_provider_config.transform_image_generation_request(
- model=model,
- prompt=prompt,
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- headers=headers,
- )
-
- ## LOGGING
- logging_obj.pre_call(
- input=prompt,
- api_key="",
- additional_args={
- "complete_input_dict": data,
- "api_base": api_base,
- "headers": headers,
- },
- )
-
- try:
- response = sync_httpx_client.post(
- url=api_base,
- headers=headers,
- json=data,
- timeout=timeout,
- )
-
- except Exception as e:
- raise self._handle_error(
- e=e,
- provider_config=image_generation_provider_config,
- )
-
- model_response: ImageResponse = image_generation_provider_config.transform_image_generation_response(
- model=model,
- raw_response=response,
- model_response=litellm.ImageResponse(),
- logging_obj=logging_obj,
- request_data=data,
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- encoding=None,
- )
-
- return model_response
-
- async def async_image_generation_handler(
- self,
- model: str,
- prompt: str,
- image_generation_provider_config: BaseImageGenerationConfig,
- image_generation_optional_request_params: Dict,
- custom_llm_provider: str,
- litellm_params: Dict,
- logging_obj: LiteLLMLoggingObj,
- timeout: Union[float, httpx.Timeout],
- extra_headers: Optional[Dict[str, Any]] = None,
- extra_body: Optional[Dict[str, Any]] = None,
- client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
- fake_stream: bool = False,
- litellm_metadata: Optional[Dict[str, Any]] = None,
- ) -> ImageResponse:
- """
- Async version of the image generation handler.
- Uses async HTTP client to make requests.
- """
- if client is None or not isinstance(client, AsyncHTTPHandler):
- async_httpx_client = get_async_httpx_client(
- llm_provider=litellm.LlmProviders(custom_llm_provider),
- params={"ssl_verify": litellm_params.get("ssl_verify", None)},
- )
- else:
- async_httpx_client = client
-
-
- headers = image_generation_provider_config.validate_environment(
- api_key=litellm_params.get("api_key", None),
- headers=image_generation_optional_request_params.get("extra_headers", {}) or {},
- model=model,
- messages=[],
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- )
-
- if extra_headers:
- headers.update(extra_headers)
-
- api_base = image_generation_provider_config.get_complete_url(
- model=model,
- api_base=litellm_params.get("api_base", None),
- api_key=litellm_params.get("api_key", None),
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- )
-
- data = image_generation_provider_config.transform_image_generation_request(
- model=model,
- prompt=prompt,
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- headers=headers,
- )
-
- ## LOGGING
- logging_obj.pre_call(
- input=prompt,
- api_key="",
- additional_args={
- "complete_input_dict": data,
- "api_base": api_base,
- "headers": headers,
- },
- )
-
- try:
- response = await async_httpx_client.post(
- url=api_base,
- headers=headers,
- json=data,
- timeout=timeout,
- )
-
- except Exception as e:
- raise self._handle_error(
- e=e,
- provider_config=image_generation_provider_config,
- )
-
- model_response: ImageResponse = image_generation_provider_config.transform_image_generation_response(
- model=model,
- raw_response=response,
- model_response=litellm.ImageResponse(),
- logging_obj=logging_obj,
- request_data=data,
- optional_params=image_generation_optional_request_params,
- litellm_params=dict(litellm_params),
- encoding=None,
- )
-
- return model_response
-
###### VECTOR STORE HANDLER ######
async def async_vector_store_search_handler(
self,
diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py
index 7d7ef522a4..8ec9cd544f 100644
--- a/litellm/llms/github_copilot/authenticator.py
+++ b/litellm/llms/github_copilot/authenticator.py
@@ -132,23 +132,6 @@ class Authenticator:
status_code=401,
)
- def get_api_base(self) -> Optional[str]:
- """
- Get the API endpoint from the api-key.json file.
-
- Returns:
- Optional[str]: The GitHub Copilot API endpoint, or None if not found.
- """
- try:
- with open(self.api_key_file, "r") as f:
- api_key_info = json.load(f)
- endpoints = api_key_info.get("endpoints", {})
- api_endpoint = endpoints.get("api")
- return api_endpoint
- except (IOError, json.JSONDecodeError, KeyError) as e:
- verbose_logger.warning(f"Error reading API endpoint from file: {str(e)}")
- return None
-
def _refresh_api_key(self) -> Dict[str, Any]:
"""
Refresh the API key using the access token.
diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py
index 5f821bd9f5..b0aaaf9a51 100644
--- a/litellm/llms/github_copilot/chat/transformation.py
+++ b/litellm/llms/github_copilot/chat/transformation.py
@@ -25,7 +25,7 @@ class GithubCopilotConfig(OpenAIConfig):
api_key: Optional[str],
custom_llm_provider: str,
) -> Tuple[Optional[str], Optional[str], str]:
- dynamic_api_base = self.authenticator.get_api_base() or self.GITHUB_COPILOT_API_BASE
+ api_base = self.GITHUB_COPILOT_API_BASE
try:
dynamic_api_key = self.authenticator.get_api_key()
except GetAPIKeyError as e:
@@ -34,7 +34,7 @@ class GithubCopilotConfig(OpenAIConfig):
llm_provider=custom_llm_provider,
message=str(e),
)
- return dynamic_api_base, dynamic_api_key, custom_llm_provider
+ return api_base, dynamic_api_key, custom_llm_provider
def _transform_messages(
self,
diff --git a/litellm/llms/hyperbolic/__init__.py b/litellm/llms/hyperbolic/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/litellm/llms/hyperbolic/chat/__init__.py b/litellm/llms/hyperbolic/chat/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/litellm/llms/hyperbolic/chat/transformation.py b/litellm/llms/hyperbolic/chat/transformation.py
deleted file mode 100644
index 48af9fa68a..0000000000
--- a/litellm/llms/hyperbolic/chat/transformation.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""
-Translate from OpenAI's `/v1/chat/completions` to Hyperbolic's `/v1/chat/completions`
-"""
-
-from typing import Optional, Tuple
-
-from litellm.secret_managers.main import get_secret_str
-
-from ...openai_like.chat.transformation import OpenAILikeChatConfig
-
-
-class HyperbolicChatConfig(OpenAILikeChatConfig):
- """
- Hyperbolic is OpenAI-compatible with standard endpoints
- """
-
- @property
- def custom_llm_provider(self) -> Optional[str]:
- return "hyperbolic"
-
- def _get_openai_compatible_provider_info(
- self, api_base: Optional[str], api_key: Optional[str]
- ) -> Tuple[Optional[str], Optional[str]]:
- # Hyperbolic is openai compatible, we just need to set the api_base
- api_base = (
- api_base
- or get_secret_str("HYPERBOLIC_API_BASE")
- or "https://api.hyperbolic.xyz/v1" # Default Hyperbolic API base URL
- ) # type: ignore
- dynamic_api_key = api_key or get_secret_str("HYPERBOLIC_API_KEY")
- return api_base, dynamic_api_key
-
- def get_supported_openai_params(self, model: str) -> list:
- """
- Hyperbolic supports standard OpenAI parameters
- Reference: https://docs.hyperbolic.xyz/docs/rest-api
- """
- return [
- "messages", # Required
- "model", # Required
- "stream", # Optional
- "temperature", # Optional
- "top_p", # Optional
- "max_tokens", # Optional
- "frequency_penalty", # Optional
- "presence_penalty", # Optional
- "stop", # Optional
- "n", # Optional
- "tools", # Optional
- "tool_choice", # Optional
- "response_format", # Optional
- "seed", # Optional
- "user", # Optional
- ]
diff --git a/litellm/llms/lambda_ai/__init__.py b/litellm/llms/lambda_ai/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/litellm/llms/lambda_ai/chat/__init__.py b/litellm/llms/lambda_ai/chat/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/litellm/llms/lambda_ai/chat/transformation.py b/litellm/llms/lambda_ai/chat/transformation.py
deleted file mode 100644
index 2d481d6682..0000000000
--- a/litellm/llms/lambda_ai/chat/transformation.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-Translate from OpenAI's `/v1/chat/completions` to Lambda's `/v1/chat/completions`
-"""
-
-from typing import Optional, Tuple
-
-from litellm.secret_managers.main import get_secret_str
-
-from ...openai_like.chat.transformation import OpenAILikeChatConfig
-
-
-class LambdaAIChatConfig(OpenAILikeChatConfig):
- """
- Lambda AI is OpenAI-compatible with standard endpoints
- """
-
- @property
- def custom_llm_provider(self) -> Optional[str]:
- return "lambda_ai"
-
- def _get_openai_compatible_provider_info(
- self, api_base: Optional[str], api_key: Optional[str]
- ) -> Tuple[Optional[str], Optional[str]]:
- # Lambda AI is openai compatible, we just need to set the api_base
- api_base = (
- api_base
- or get_secret_str("LAMBDA_API_BASE")
- or "https://api.lambda.ai/v1" # Default Lambda API base URL
- ) # type: ignore
- dynamic_api_key = api_key or get_secret_str("LAMBDA_API_KEY")
- return api_base, dynamic_api_key
\ No newline at end of file
diff --git a/litellm/llms/morph/__init__.py b/litellm/llms/morph/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/litellm/llms/morph/chat/__init__.py b/litellm/llms/morph/chat/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/litellm/llms/morph/chat/transformation.py b/litellm/llms/morph/chat/transformation.py
deleted file mode 100644
index f37ed8e45e..0000000000
--- a/litellm/llms/morph/chat/transformation.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""
-Transform request from OpenAI format to Morph format.
-
-[TODO] Docs: Morph supports the OpenAI API format.
-https://docs.morphllm.com/quickstart
-"""
-
-from typing import Optional, Tuple
-
-from litellm.secret_managers.main import get_secret_str
-
-from ...openai_like.chat.transformation import OpenAILikeChatConfig
-
-
-class MorphChatConfig(OpenAILikeChatConfig):
- """
- Transform request from OpenAI format to Morph format.
- """
-
- @property
- def custom_llm_provider(self) -> Optional[str]:
- return "morph"
-
- def _get_openai_compatible_provider_info(
- self, api_base: Optional[str], api_key: Optional[str]
- ) -> Tuple[Optional[str], Optional[str]]:
- api_base = (
- api_base
- or get_secret_str("MORPH_API_BASE")
- or "https://api.morphllm.com/v1" # default api base
- )
- dynamic_api_key = api_key or get_secret_str("MORPH_API_KEY")
- return api_base, dynamic_api_key
-
- def get_supported_openai_params(self, model: str) -> list:
- return [
- "messages",
- "model",
- "stream",
- ]
-
- def pre_call(self, messages: list, model: str, api_key: str, api_base: str):
- """
- Hook for any pre-processing before the API call.
- """
- return
\ No newline at end of file
diff --git a/litellm/llms/recraft/image_generation/__init__.py b/litellm/llms/recraft/image_generation/__init__.py
deleted file mode 100644
index cb8c5624db..0000000000
--- a/litellm/llms/recraft/image_generation/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from litellm.llms.base_llm.image_generation.transformation import (
- BaseImageGenerationConfig,
-)
-
-from .transformation import RecraftImageGenerationConfig
-
-__all__ = [
- "RecraftImageGenerationConfig",
-]
-
-
-def get_recraft_image_generation_config(model: str) -> BaseImageGenerationConfig:
- return RecraftImageGenerationConfig()
diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py
deleted file mode 100644
index f632b49f3a..0000000000
--- a/litellm/llms/recraft/image_generation/transformation.py
+++ /dev/null
@@ -1,163 +0,0 @@
-from typing import TYPE_CHECKING, Any, List, Optional
-
-import httpx
-
-from litellm.llms.base_llm.image_generation.transformation import (
- BaseImageGenerationConfig,
-)
-from litellm.secret_managers.main import get_secret_str
-from litellm.types.llms.openai import (
- AllMessageValues,
- OpenAIImageGenerationOptionalParams,
-)
-from litellm.types.llms.recraft import RecraftImageGenerationRequestParams
-from litellm.types.utils import ImageObject, ImageResponse
-
-if TYPE_CHECKING:
- from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
-
- LiteLLMLoggingObj = _LiteLLMLoggingObj
-else:
- LiteLLMLoggingObj = Any
-
-
-class RecraftImageGenerationConfig(BaseImageGenerationConfig):
- DEFAULT_BASE_URL: str = "https://external.api.recraft.ai"
- IMAGE_GENERATION_ENDPOINT: str = "v1/images/generations"
-
- def get_supported_openai_params(
- self, model: str
- ) -> List[OpenAIImageGenerationOptionalParams]:
- """
- https://www.recraft.ai/docs#generate-image
- """
- return [
- "n",
- "response_format",
- "size",
- "style"
- ]
-
- def map_openai_params(
- self,
- non_default_params: dict,
- optional_params: dict,
- model: str,
- drop_params: bool,
- ) -> dict:
- supported_params = self.get_supported_openai_params(model)
- for k in non_default_params.keys():
- if k not in optional_params.keys():
- if k in supported_params:
- optional_params[k] = non_default_params[k]
- elif drop_params:
- pass
- else:
- raise ValueError(
- f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
- )
-
- return optional_params
-
- def get_complete_url(
- self,
- api_base: Optional[str],
- api_key: Optional[str],
- model: str,
- optional_params: dict,
- litellm_params: dict,
- stream: Optional[bool] = None,
- ) -> str:
- """
- Get the complete url for the request
-
- Some providers need `model` in `api_base`
- """
- complete_url: str = (
- api_base
- or get_secret_str("RECRAFT_API_BASE")
- or self.DEFAULT_BASE_URL
- )
-
- complete_url = complete_url.rstrip("/")
- complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}"
- return complete_url
-
- def validate_environment(
- self,
- headers: dict,
- model: str,
- messages: List[AllMessageValues],
- optional_params: dict,
- litellm_params: dict,
- api_key: Optional[str] = None,
- api_base: Optional[str] = None,
- ) -> dict:
- final_api_key: Optional[str] = (
- api_key or
- get_secret_str("RECRAFT_API_KEY")
- )
- if not final_api_key:
- raise ValueError("RECRAFT_API_KEY is not set")
-
- headers["Authorization"] = f"Bearer {final_api_key}"
- return headers
-
-
-
- def transform_image_generation_request(
- self,
- model: str,
- prompt: str,
- optional_params: dict,
- litellm_params: dict,
- headers: dict,
- ) -> dict:
- """
- Transform the image generation request to the recraft image generation request body
-
- https://www.recraft.ai/docs#generate-image
- """
- recratft_image_generation_request_body: RecraftImageGenerationRequestParams = RecraftImageGenerationRequestParams(
- prompt=prompt,
- model=model,
- **optional_params,
- )
- return dict(recratft_image_generation_request_body)
-
- def transform_image_generation_response(
- self,
- model: str,
- raw_response: httpx.Response,
- model_response: ImageResponse,
- logging_obj: LiteLLMLoggingObj,
- request_data: dict,
- optional_params: dict,
- litellm_params: dict,
- encoding: Any,
- api_key: Optional[str] = None,
- json_mode: Optional[bool] = None,
- ) -> ImageResponse:
- """
- Transform the image generation response to the litellm image response
-
- https://www.recraft.ai/docs#generate-image
- """
- try:
- response_data = raw_response.json()
- except Exception as e:
- raise self.get_error_class(
- error_message=f"Error transforming image generation response: {e}",
- status_code=raw_response.status_code,
- headers=raw_response.headers,
- )
- if not model_response.data:
- model_response.data = []
-
- for image_data in response_data["data"]:
- model_response.data.append(ImageObject(
- url=image_data.get("url", None),
- b64_json=image_data.get("b64_json", None),
- ))
-
- return model_response
\ No newline at end of file
diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
index e4a68dd82e..d09599e878 100644
--- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
@@ -35,7 +35,6 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
- _get_httpx_client,
get_async_httpx_client,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
@@ -1882,7 +1881,7 @@ class VertexLLM(VertexBase):
if isinstance(timeout, float) or isinstance(timeout, int):
timeout = httpx.Timeout(timeout)
_params["timeout"] = timeout
- client = _get_httpx_client(params=_params)
+ client = HTTPHandler(**_params) # type: ignore
else:
client = client
diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py
index 6b0dd5a39a..71d8bba4ef 100644
--- a/litellm/llms/watsonx/chat/transformation.py
+++ b/litellm/llms/watsonx/chat/transformation.py
@@ -25,7 +25,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
"seed", # equivalent to random_seed
"stream", # equivalent to stream
"tools",
- "tool_choice", # equivalent to tool_choice + tool_choice_option
+ "tool_choice", # equivalent to tool_choice + tool_choice_options
"logprobs",
"top_logprobs",
"n",
@@ -61,7 +61,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
_tool_choice = non_default_params.pop("tool_choice", None)
if self.is_tool_choice_option(_tool_choice):
- optional_params["tool_choice_option"] = _tool_choice
+ optional_params["tool_choice_options"] = _tool_choice
elif _tool_choice is not None:
optional_params["tool_choice"] = _tool_choice
return super().map_openai_params(
diff --git a/litellm/main.py b/litellm/main.py
index e0b76d57a6..e0d41260a3 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -3486,13 +3486,13 @@ async def acompletion_with_retries(*args, **kwargs):
retry_strategy = kwargs.pop("retry_strategy", "constant_retry")
original_function = kwargs.pop("original_function", completion)
if retry_strategy == "exponential_backoff_retry":
- retryer = tenacity.Retrying(
+ retryer = tenacity.AsyncRetrying(
wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(num_retries),
reraise=True,
)
else:
- retryer = tenacity.Retrying(
+ retryer = tenacity.AsyncRetrying(
stop=tenacity.stop_after_attempt(num_retries), reraise=True
)
return await retryer(original_function, *args, **kwargs)
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 6e9a757c4a..58ec3df932 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -5417,12 +5417,12 @@
"supports_tool_choice": true,
"deprecation_date": "2025-01-06"
},
- "groq/qwen/qwen3-32b": {
- "max_tokens": 131000,
- "max_input_tokens": 131000,
- "max_output_tokens": 131000,
+ "groq/qwen-qwq-32b": {
+ "max_tokens": 128000,
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
"input_cost_per_token": 2.9e-07,
- "output_cost_per_token": 5.9e-07,
+ "output_cost_per_token": 3.9e-07,
"litellm_provider": "groq",
"mode": "chat",
"supports_function_calling": true,
@@ -10898,16 +10898,6 @@
"mode": "chat",
"supports_tool_choice": true
},
- "openrouter/qwen/qwen-vl-plus": {
- "max_tokens": 8192,
- "max_input_tokens": 8192,
- "max_output_tokens": 2048,
- "input_cost_per_token": 0.00021,
- "output_cost_per_token": 0.00063,
- "litellm_provider": "openrouter",
- "mode": "chat",
- "supports_tool_choice": false
- },
"openrouter/switchpoint/router": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@@ -14472,19 +14462,6 @@
"supports_tool_choice": false,
"supports_response_schema": true
},
- "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 16384,
- "input_cost_per_token": 0.6e-06,
- "output_cost_per_token": 2.5e-06,
- "litellm_provider": "fireworks_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_response_schema": true,
- "supports_tool_choice": true,
- "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct"
- },
"fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": {
"max_tokens": 16384,
"max_input_tokens": 128000,
@@ -14803,478 +14780,13 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
- "lambda_ai/deepseek-llama3.3-70b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 6e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "lambda_ai/deepseek-r1-0528": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 6e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "lambda_ai/deepseek-r1-671b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 8e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "lambda_ai/deepseek-v3-0324": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 6e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/hermes3-405b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 8e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/hermes3-70b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/hermes3-8b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-08,
- "output_cost_per_token": 4e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/lfm-40b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
+ "voyage/voyage-01": {
+ "max_tokens": 4096,
+ "max_input_tokens": 4096,
"input_cost_per_token": 1e-07,
- "output_cost_per_token": 2e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/lfm-7b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-08,
- "output_cost_per_token": 4e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama-4-maverick-17b-128e-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 8192,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama-4-scout-17b-16e-instruct": {
- "max_tokens": 16384,
- "max_input_tokens": 16384,
- "max_output_tokens": 8192,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-405b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 8e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-70b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-8b-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-08,
- "output_cost_per_token": 4e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-nemotron-70b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.2-11b-vision-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.5e-08,
- "output_cost_per_token": 2.5e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_vision": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.2-3b-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.5e-08,
- "output_cost_per_token": 2.5e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.3-70b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/qwen25-coder-32b-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/qwen3-32b-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "hyperbolic/moonshotai/Kimi-K2-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-06,
- "output_cost_per_token": 2e-06,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-R1-0528": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-07,
- "output_cost_per_token": 2.5e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/Qwen3-235B-A22B": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-06,
- "output_cost_per_token": 2e-06,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-V3-0324": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 4e-07,
- "output_cost_per_token": 4e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/QwQ-32B": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 2e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-R1": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 4e-07,
- "output_cost_per_token": 4e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-V3": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 2e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Llama-3.3-70B-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Llama-3.2-3B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/Qwen2.5-72B-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
+ "output_cost_per_token": 0.0,
+ "litellm_provider": "voyage",
+ "mode": "embedding"
},
"voyage/voyage-lite-01": {
"max_tokens": 4096,
@@ -17112,51 +16624,5 @@
"supports_vision": true,
"mode": "chat",
"source": "https://platform.moonshot.ai/docs/pricing"
- },
- "recraft/recraftv3": {
- "mode": "image_generation",
- "input_cost_per_image": 0.04,
- "litellm_provider": "recraft",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "source": "https://www.recraft.ai/docs#pricing"
- },
- "recraft/recraftv2": {
- "mode": "image_generation",
- "input_cost_per_image": 0.022,
- "litellm_provider": "recraft",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "source": "https://www.recraft.ai/docs#pricing"
- },
- "morph/morph-v3-fast": {
- "max_tokens": 16000,
- "max_input_tokens": 16000,
- "max_output_tokens": 16000,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 1.2e-06,
- "litellm_provider": "morph",
- "mode": "chat",
- "supports_function_calling": false,
- "supports_parallel_function_calling": false,
- "supports_vision": false,
- "supports_system_messages": true,
- "supports_tool_choice": false
- },
- "morph/morph-v3-large": {
- "max_tokens": 16000,
- "max_input_tokens": 16000,
- "max_output_tokens": 16000,
- "input_cost_per_token": 9e-07,
- "output_cost_per_token": 1.9e-06,
- "litellm_provider": "morph",
- "mode": "chat",
- "supports_function_calling": false,
- "supports_parallel_function_calling": false,
- "supports_vision": false,
- "supports_system_messages": true,
- "supports_tool_choice": false
}
}
diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml
index 00d3ef1adf..b2642d8a60 100644
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@@ -1,11 +1,25 @@
model_list:
- - model_name: gpt-4o-mini-batch
+ - model_name: gpt-3.5-turbo-allow
litellm_params:
- model: azure/gpt-4o-mini
- api_key: os.environ/AZURE_API_KEY_HIDDEN
- api_base: os.environ/AZURE_API_BASE_HIDDEN
+ model: gpt-3.5-turbo
model_info:
version: 2
+ - model_name: gpt-3.5-turbo-disallow
+ litellm_params:
+ model: gpt-3.5-turbo
+ model_info:
+ version: 2
+ - model_name: zapier-byok-provider/openai/*
+ litellm_params:
+ model: openai/*
+ api_base: http://0.0.0.0:8090
+ - model_name: openai/gpt-4o-mini
+ litellm_params:
+ model: openai/gpt-4o-mini
-general_settings:
- proxy_batch_polling_interval: 10
\ No newline at end of file
+litellm_settings:
+ model_group_alias: {"gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow"}
+ model_group_settings:
+ forward_client_headers_to_llm_api:
+ - "gpt-3.5-turbo-allow"
+ - "zapier-byok-provider/openai/*"
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index c2bf2f4608..50a044d983 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -132,7 +132,6 @@ from litellm.constants import (
DEFAULT_MODEL_CREATED_AT_TIME,
LITELLM_PROXY_ADMIN_NAME,
PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS,
- PROXY_BATCH_POLLING_INTERVAL,
PROXY_BATCH_WRITE_AT,
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
@@ -539,7 +538,7 @@ async def proxy_shutdown_event():
@asynccontextmanager
async def proxy_startup_event(app: FastAPI):
- global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check, proxy_batch_polling_interval
+ global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check
import json
init_verbose_loggers()
@@ -941,7 +940,6 @@ litellm_proxy_admin_name = LITELLM_PROXY_ADMIN_NAME
ui_access_mode: Union[Literal["admin", "all"], Dict] = "all"
proxy_budget_rescheduler_min_time = PROXY_BUDGET_RESCHEDULER_MIN_TIME
proxy_budget_rescheduler_max_time = PROXY_BUDGET_RESCHEDULER_MAX_TIME
-proxy_batch_polling_interval = PROXY_BATCH_POLLING_INTERVAL
proxy_batch_write_at = PROXY_BATCH_WRITE_AT
litellm_master_key_hash = None
disable_spend_logs = False
@@ -1699,7 +1697,7 @@ class ProxyConfig:
"""
Load config values into proxy global state
"""
- global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings, proxy_batch_polling_interval
+ global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings
config: dict = await self.get_config(config_file_path=config_file_path)
@@ -2042,10 +2040,6 @@ class ProxyConfig:
proxy_budget_rescheduler_max_time = general_settings.get(
"proxy_budget_rescheduler_max_time", proxy_budget_rescheduler_max_time
)
- ## BATCH POLLING INTERVAL ##
- proxy_batch_polling_interval = general_settings.get(
- "proxy_batch_polling_interval", proxy_batch_polling_interval
- )
## BATCH WRITER ##
proxy_batch_write_at = general_settings.get(
"proxy_batch_write_at", proxy_batch_write_at
@@ -3570,7 +3564,7 @@ class ProxyStartupEvent:
scheduler.add_job(
check_batch_cost_job.check_batch_cost,
"interval",
- seconds=proxy_batch_polling_interval, # these can run infrequently, as batch jobs take time to complete
+ seconds=3600, # these can run infrequently, as batch jobs take time to complete
)
except Exception:
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index a3aafda223..14f34b1585 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -480,7 +480,7 @@ async def update_sso_settings(sso_config: SSOConfig):
elif field_name == "ui_access_mode" and value is not None:
config["general_settings"]["ui_access_mode"] = value
- elif field_name in env_var_mapping and value is not None and len(value) > 0:
+ elif field_name in env_var_mapping and value is not None:
env_var_name = env_var_mapping[field_name]
# Update in config
config["environment_variables"][env_var_name] = value
diff --git a/litellm/router_utils/forward_clientside_headers_by_model_group.py b/litellm/router_utils/forward_clientside_headers_by_model_group.py
index 87c2e48f68..2e1a066a6c 100644
--- a/litellm/router_utils/forward_clientside_headers_by_model_group.py
+++ b/litellm/router_utils/forward_clientside_headers_by_model_group.py
@@ -31,6 +31,18 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger):
"model_group_alias": model_group_alias,
}
+ def filter_headers(self, headers: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Filter the headers to only include the headers that are forwarded to the LLM API.
+
+ E.g. passing 'connection': 'keep-alive' will cause the request to hang, and not be acknowledged on the other side.
+ """
+ return {
+ k: v
+ for k, v in headers.items()
+ if k.lower() not in ["connection", "content-length"]
+ }
+
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
) -> Optional[dict]:
@@ -66,7 +78,7 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger):
in litellm.model_group_settings.forward_client_headers_to_llm_api
):
kwargs.setdefault("headers", {}).update(
- kwargs["secret_fields"]["raw_headers"]
+ self.filter_headers(kwargs["secret_fields"]["raw_headers"])
)
return kwargs
diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py
index 1ce7264b0b..e4b749a659 100644
--- a/litellm/secret_managers/get_azure_ad_token_provider.py
+++ b/litellm/secret_managers/get_azure_ad_token_provider.py
@@ -6,10 +6,7 @@ from litellm.types.secret_managers.get_azure_ad_token_provider import (
)
-def get_azure_ad_token_provider(
- azure_scope: Optional[str] = None,
- azure_credential: Optional[AzureCredentialType] = None,
-) -> Callable[[], str]:
+def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[], str]:
"""
Get Azure AD token provider based on Service Principal with Secret workflow.
@@ -30,7 +27,6 @@ def get_azure_ad_token_provider(
from azure.identity import (
CertificateCredential,
ClientSecretCredential,
- DefaultAzureCredential,
ManagedIdentityCredential,
get_bearer_token_provider,
)
@@ -41,17 +37,14 @@ def get_azure_ad_token_provider(
or "https://cognitiveservices.azure.com/.default"
)
- cred: str = (
- azure_credential.value if azure_credential else None
- or os.environ.get("AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential)
- or AzureCredentialType.ClientSecretCredential
+ cred: str = os.environ.get(
+ "AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential
)
credential: Optional[
Union[
ClientSecretCredential,
ManagedIdentityCredential,
CertificateCredential,
- DefaultAzureCredential,
Any,
]
] = None
@@ -69,15 +62,10 @@ def get_azure_ad_token_provider(
tenant_id=os.environ["AZURE_TENANT_ID"],
certificate_path=os.environ["AZURE_CERTIFICATE_PATH"],
)
- elif cred == AzureCredentialType.DefaultAzureCredential:
- # DefaultAzureCredential doesn't require explicit environment variables
- # It automatically discovers credentials from the environment (managed identity, CLI, etc.)
- credential = DefaultAzureCredential()
else:
cred_cls = getattr(identity, cred)
credential = cred_cls()
if credential is None:
raise ValueError("No credential provided")
-
return get_bearer_token_provider(credential, azure_scope)
diff --git a/litellm/types/llms/recraft.py b/litellm/types/llms/recraft.py
deleted file mode 100644
index 176810970b..0000000000
--- a/litellm/types/llms/recraft.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from typing import Dict, List, Optional
-
-from typing_extensions import TypedDict
-
-
-class RecraftImageGenerationRequestParams(TypedDict, total=False):
- prompt: str
- text_layout: Optional[List[Dict]]
- n: Optional[int]
- style_id: Optional[str]
- style: Optional[str]
- substyle: Optional[str]
- model: Optional[str]
- response_format: Optional[str]
- size: Optional[str]
- negative_prompt: Optional[str]
- controls: Optional[Dict]
\ No newline at end of file
diff --git a/litellm/types/secret_managers/get_azure_ad_token_provider.py b/litellm/types/secret_managers/get_azure_ad_token_provider.py
index 5d2f7409f9..f318b4333b 100644
--- a/litellm/types/secret_managers/get_azure_ad_token_provider.py
+++ b/litellm/types/secret_managers/get_azure_ad_token_provider.py
@@ -5,4 +5,3 @@ class AzureCredentialType(str, Enum):
ClientSecretCredential = "ClientSecretCredential"
ManagedIdentityCredential = "ManagedIdentityCredential"
CertificateCredential = "CertificateCredential"
- DefaultAzureCredential = "DefaultAzureCredential"
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index acff566a3f..5ff17bd337 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2276,8 +2276,6 @@ class LlmProviders(str, Enum):
DASHSCOPE = "dashscope"
MOONSHOT = "moonshot"
V0 = "v0"
- MORPH = "morph"
- LAMBDA_AI = "lambda_ai"
DEEPSEEK = "deepseek"
SAMBANOVA = "sambanova"
MARITALK = "maritalk"
@@ -2315,8 +2313,6 @@ class LlmProviders(str, Enum):
LLAMA = "meta_llama"
NSCALE = "nscale"
PG_VECTOR = "pg_vector"
- HYPERBOLIC = "hyperbolic"
- RECRAFT = "recraft"
# Create a set of all provider values for quick lookup
diff --git a/litellm/utils.py b/litellm/utils.py
index b2a717814f..e91c626dcb 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -6684,8 +6684,6 @@ class ProviderConfigManager:
return litellm.DatabricksConfig()
elif litellm.LlmProviders.XAI == provider:
return litellm.XAIChatConfig()
- elif litellm.LlmProviders.LAMBDA_AI == provider:
- return litellm.LambdaAIChatConfig()
elif litellm.LlmProviders.LLAMA == provider:
return litellm.LlamaAPIConfig()
elif litellm.LlmProviders.TEXT_COMPLETION_OPENAI == provider:
@@ -6834,8 +6832,6 @@ class ProviderConfigManager:
return litellm.MoonshotChatConfig()
elif litellm.LlmProviders.V0 == provider:
return litellm.V0ChatConfig()
- elif litellm.LlmProviders.MORPH == provider:
- return litellm.MorphChatConfig()
elif litellm.LlmProviders.BEDROCK == provider:
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider(
@@ -6884,8 +6880,6 @@ class ProviderConfigManager:
return litellm.OpenAIGPTConfig()
elif litellm.LlmProviders.NSCALE == provider:
return litellm.NscaleConfig()
- elif litellm.LlmProviders.HYPERBOLIC == provider:
- return litellm.HyperbolicChatConfig()
return None
@staticmethod
@@ -7153,12 +7147,6 @@ class ProviderConfigManager:
)
return get_xinference_image_generation_config(model)
- elif LlmProviders.RECRAFT == provider:
- from litellm.llms.recraft.image_generation import (
- get_recraft_image_generation_config,
- )
-
- return get_recraft_image_generation_config(model)
return None
@staticmethod
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 6e9a757c4a..58ec3df932 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -5417,12 +5417,12 @@
"supports_tool_choice": true,
"deprecation_date": "2025-01-06"
},
- "groq/qwen/qwen3-32b": {
- "max_tokens": 131000,
- "max_input_tokens": 131000,
- "max_output_tokens": 131000,
+ "groq/qwen-qwq-32b": {
+ "max_tokens": 128000,
+ "max_input_tokens": 128000,
+ "max_output_tokens": 128000,
"input_cost_per_token": 2.9e-07,
- "output_cost_per_token": 5.9e-07,
+ "output_cost_per_token": 3.9e-07,
"litellm_provider": "groq",
"mode": "chat",
"supports_function_calling": true,
@@ -10898,16 +10898,6 @@
"mode": "chat",
"supports_tool_choice": true
},
- "openrouter/qwen/qwen-vl-plus": {
- "max_tokens": 8192,
- "max_input_tokens": 8192,
- "max_output_tokens": 2048,
- "input_cost_per_token": 0.00021,
- "output_cost_per_token": 0.00063,
- "litellm_provider": "openrouter",
- "mode": "chat",
- "supports_tool_choice": false
- },
"openrouter/switchpoint/router": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@@ -14472,19 +14462,6 @@
"supports_tool_choice": false,
"supports_response_schema": true
},
- "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 16384,
- "input_cost_per_token": 0.6e-06,
- "output_cost_per_token": 2.5e-06,
- "litellm_provider": "fireworks_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_response_schema": true,
- "supports_tool_choice": true,
- "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct"
- },
"fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": {
"max_tokens": 16384,
"max_input_tokens": 128000,
@@ -14803,478 +14780,13 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
- "lambda_ai/deepseek-llama3.3-70b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 6e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "lambda_ai/deepseek-r1-0528": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 6e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "lambda_ai/deepseek-r1-671b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 8e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "lambda_ai/deepseek-v3-0324": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 6e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/hermes3-405b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 8e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/hermes3-70b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/hermes3-8b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-08,
- "output_cost_per_token": 4e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/lfm-40b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
+ "voyage/voyage-01": {
+ "max_tokens": 4096,
+ "max_input_tokens": 4096,
"input_cost_per_token": 1e-07,
- "output_cost_per_token": 2e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/lfm-7b": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-08,
- "output_cost_per_token": 4e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama-4-maverick-17b-128e-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 8192,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama-4-scout-17b-16e-instruct": {
- "max_tokens": 16384,
- "max_input_tokens": 16384,
- "max_output_tokens": 8192,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-405b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 8e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-70b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-8b-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-08,
- "output_cost_per_token": 4e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.1-nemotron-70b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.2-11b-vision-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.5e-08,
- "output_cost_per_token": 2.5e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_vision": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.2-3b-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.5e-08,
- "output_cost_per_token": 2.5e-08,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/llama3.3-70b-instruct-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/qwen25-coder-32b-instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "lambda_ai/qwen3-32b-fp8": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 5e-08,
- "output_cost_per_token": 1e-07,
- "litellm_provider": "lambda_ai",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true,
- "supports_reasoning": true
- },
- "hyperbolic/moonshotai/Kimi-K2-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-06,
- "output_cost_per_token": 2e-06,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-R1-0528": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2.5e-07,
- "output_cost_per_token": 2.5e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/Qwen3-235B-A22B": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-06,
- "output_cost_per_token": 2e-06,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-V3-0324": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 4e-07,
- "output_cost_per_token": 4e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/QwQ-32B": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 2e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-R1": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 4e-07,
- "output_cost_per_token": 4e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/deepseek-ai/DeepSeek-V3": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 2e-07,
- "output_cost_per_token": 2e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Llama-3.3-70B-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Llama-3.2-3B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/Qwen/Qwen2.5-72B-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": {
- "max_tokens": 131072,
- "max_input_tokens": 131072,
- "max_output_tokens": 131072,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
- },
- "hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": {
- "max_tokens": 32768,
- "max_input_tokens": 32768,
- "max_output_tokens": 32768,
- "input_cost_per_token": 1.2e-07,
- "output_cost_per_token": 3e-07,
- "litellm_provider": "hyperbolic",
- "mode": "chat",
- "supports_function_calling": true,
- "supports_parallel_function_calling": true,
- "supports_system_messages": true,
- "supports_tool_choice": true
+ "output_cost_per_token": 0.0,
+ "litellm_provider": "voyage",
+ "mode": "embedding"
},
"voyage/voyage-lite-01": {
"max_tokens": 4096,
@@ -17112,51 +16624,5 @@
"supports_vision": true,
"mode": "chat",
"source": "https://platform.moonshot.ai/docs/pricing"
- },
- "recraft/recraftv3": {
- "mode": "image_generation",
- "input_cost_per_image": 0.04,
- "litellm_provider": "recraft",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "source": "https://www.recraft.ai/docs#pricing"
- },
- "recraft/recraftv2": {
- "mode": "image_generation",
- "input_cost_per_image": 0.022,
- "litellm_provider": "recraft",
- "supported_endpoints": [
- "/v1/images/generations"
- ],
- "source": "https://www.recraft.ai/docs#pricing"
- },
- "morph/morph-v3-fast": {
- "max_tokens": 16000,
- "max_input_tokens": 16000,
- "max_output_tokens": 16000,
- "input_cost_per_token": 8e-07,
- "output_cost_per_token": 1.2e-06,
- "litellm_provider": "morph",
- "mode": "chat",
- "supports_function_calling": false,
- "supports_parallel_function_calling": false,
- "supports_vision": false,
- "supports_system_messages": true,
- "supports_tool_choice": false
- },
- "morph/morph-v3-large": {
- "max_tokens": 16000,
- "max_input_tokens": 16000,
- "max_output_tokens": 16000,
- "input_cost_per_token": 9e-07,
- "output_cost_per_token": 1.9e-06,
- "litellm_provider": "morph",
- "mode": "chat",
- "supports_function_calling": false,
- "supports_parallel_function_calling": false,
- "supports_vision": false,
- "supports_system_messages": true,
- "supports_tool_choice": false
}
}
diff --git a/pyproject.toml b/pyproject.toml
index 546133e471..f419b999e3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
-version = "1.74.8"
+version = "1.74.7"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@@ -144,7 +144,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
-version = "1.74.8"
+version = "1.74.7"
version_files = [
"pyproject.toml:^version"
]
diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py
index 79f7f42e55..cc277f7481 100644
--- a/tests/image_gen_tests/test_image_generation.py
+++ b/tests/image_gen_tests/test_image_generation.py
@@ -165,11 +165,6 @@ class TestOpenAIGPTImage1(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
return {"model": "gpt-image-1"}
-class TestRecraftImageGeneration(BaseImageGenTest):
- def get_base_image_generation_call_args(self) -> dict:
- return {"model": "recraft/recraftv3"}
-
-
class TestAzureOpenAIDalle3(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
litellm.set_verbose = True
diff --git a/tests/llm_translation/test_anthropic_text_completion.py b/tests/llm_translation/test_anthropic_text_completion.py
new file mode 100644
index 0000000000..c0aa5b5d88
--- /dev/null
+++ b/tests/llm_translation/test_anthropic_text_completion.py
@@ -0,0 +1,73 @@
+import asyncio
+import os
+import sys
+import traceback
+
+from dotenv import load_dotenv
+
+import litellm.types
+import litellm.types.utils
+from litellm.llms.anthropic.chat import ModelResponseIterator
+
+load_dotenv()
+import io
+import os
+
+sys.path.insert(
+ 0, os.path.abspath("../..")
+) # Adds the parent directory to the system path
+from typing import Optional
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model", ["claude-2", "anthropic/claude-2"])
+@pytest.mark.flaky(retries=6, delay=1)
+async def test_acompletion_claude2(model):
+ try:
+ litellm.set_verbose = True
+ messages = [
+ {
+ "role": "system",
+ "content": "Your goal is generate a joke on the topic user gives.",
+ },
+ {"role": "user", "content": "Generate a 3 liner joke for me"},
+ ]
+ # test without max-tokens
+ response = await litellm.acompletion(model=model, messages=messages)
+ # Add any assertions here to check the response
+ print(response)
+ print(response.usage)
+ print(response.usage.completion_tokens)
+ print(response["usage"]["completion_tokens"])
+ # print("new cost tracking")
+ except litellm.InternalServerError:
+ pytest.skip("model is overloaded.")
+ except Exception as e:
+ pytest.fail(f"Error occurred: {e}")
+
+
+@pytest.mark.asyncio
+async def test_acompletion_claude2_stream():
+ try:
+ litellm.set_verbose = False
+ messages = [
+ {
+ "role": "system",
+ "content": "Your goal is generate a joke on the topic user gives.",
+ },
+ {"role": "user", "content": "Generate a 3 liner joke for me"},
+ ]
+ # test without max-tokens
+ response = await litellm.acompletion(
+ model="anthropic_text/claude-2",
+ messages=messages,
+ stream=True,
+ max_tokens=10,
+ )
+ async for chunk in response:
+ print(chunk)
+ except Exception as e:
+ pytest.fail(f"Error occurred: {e}")
diff --git a/tests/llm_translation/test_hyperbolic.py b/tests/llm_translation/test_hyperbolic.py
deleted file mode 100644
index 38f4dea436..0000000000
--- a/tests/llm_translation/test_hyperbolic.py
+++ /dev/null
@@ -1,119 +0,0 @@
-import os
-import sys
-from datetime import datetime
-from unittest.mock import MagicMock
-
-import pytest
-
-sys.path.insert(
- 0, os.path.abspath("../..")
-) # Adds the parent directory to the system path
-
-import litellm
-from litellm import get_llm_provider
-
-
-def test_get_llm_provider_hyperbolic():
- """Test that hyperbolic/ prefix returns the correct provider"""
- model, provider, _, _ = get_llm_provider(model="hyperbolic/deepseek-v3")
- assert provider == "hyperbolic"
- assert model == "deepseek-v3"
-
-
-def test_hyperbolic_completion_call():
- """Test basic completion call structure for Hyperbolic"""
- # This is primarily a structure test since we don't have actual API keys
- try:
- litellm.set_verbose = True
- response = litellm.completion(
- model="hyperbolic/qwen-2.5-72b",
- messages=[{"role": "user", "content": "Hello!"}],
- mock_response="Hi there!",
- )
- assert response is not None
- except Exception as e:
- # Expected to fail without valid API key, but should recognize the provider
- assert "hyperbolic" in str(e).lower() or "api" in str(e).lower()
-
-
-def test_hyperbolic_config_initialization():
- """Test that HyperbolicChatConfig initializes correctly"""
- from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig
-
- config = HyperbolicChatConfig()
- assert config.custom_llm_provider == "hyperbolic"
-
-
-def test_hyperbolic_get_openai_compatible_provider_info():
- """Test API base and key handling"""
- from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig
-
- config = HyperbolicChatConfig()
-
- # Test default API base
- api_base, api_key = config._get_openai_compatible_provider_info(None, None)
- assert api_base == "https://api.hyperbolic.xyz/v1"
- # api_key may be set from environment, so we don't test for None
-
- # Test custom API base
- custom_base = "https://custom.hyperbolic.com/v1"
- api_base, api_key = config._get_openai_compatible_provider_info(custom_base, "test-key")
- assert api_base == custom_base
- assert api_key == "test-key"
-
-
-def test_hyperbolic_in_provider_lists():
- """Test that hyperbolic is in all relevant provider lists"""
- from litellm.constants import (
- openai_compatible_endpoints,
- openai_compatible_providers,
- openai_text_completion_compatible_providers,
- )
-
- assert "hyperbolic" in openai_compatible_providers
- assert "hyperbolic" in openai_text_completion_compatible_providers
- assert "https://api.hyperbolic.xyz/v1" in openai_compatible_endpoints
-
-
-def test_hyperbolic_models_configuration():
- """Test that Hyperbolic models are properly configured"""
- import json
- import os
-
- # Load model configuration directly from the JSON file
- json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
- with open(json_path, 'r') as f:
- model_data = json.load(f)
-
- # Test a few key models
- test_models = [
- "hyperbolic/deepseek-ai/DeepSeek-V3",
- "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct",
- "hyperbolic/deepseek-ai/DeepSeek-R1",
- ]
-
- for model in test_models:
- assert model in model_data
- model_info = model_data[model]
- assert model_info["litellm_provider"] == "hyperbolic"
- assert model_info["mode"] == "chat"
- assert "max_tokens" in model_info
- assert "input_cost_per_token" in model_info
- assert "output_cost_per_token" in model_info
-
-
-def test_hyperbolic_supported_params():
- """Test that supported OpenAI parameters are correctly configured"""
- from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig
-
- config = HyperbolicChatConfig()
- supported_params = config.get_supported_openai_params("hyperbolic/deepseek-v3")
-
- # Check for essential parameters
- assert "messages" in supported_params
- assert "model" in supported_params
- assert "stream" in supported_params
- assert "temperature" in supported_params
- assert "max_tokens" in supported_params
- assert "tools" in supported_params
- assert "tool_choice" in supported_params
\ No newline at end of file
diff --git a/tests/llm_translation/test_lambda_ai.py b/tests/llm_translation/test_lambda_ai.py
deleted file mode 100644
index 3b73629b4a..0000000000
--- a/tests/llm_translation/test_lambda_ai.py
+++ /dev/null
@@ -1,153 +0,0 @@
-"""
-Tests for Lambda AI provider integration
-"""
-import os
-from unittest import mock
-
-import pytest
-
-import litellm
-from litellm import completion
-from litellm.llms.lambda_ai.chat.transformation import LambdaAIChatConfig
-
-
-def test_lambda_ai_config_initialization():
- """Test LambdaAIChatConfig initializes correctly"""
- config = LambdaAIChatConfig()
- assert config.custom_llm_provider == "lambda_ai"
-
-
-def test_lambda_ai_get_openai_compatible_provider_info():
- """Test Lambda AI provider info retrieval"""
- config = LambdaAIChatConfig()
-
- # Test with default values (no env vars set)
- with mock.patch.dict(os.environ, {}, clear=True):
- api_base, api_key = config._get_openai_compatible_provider_info(None, None)
- assert api_base == "https://api.lambda.ai/v1"
- assert api_key is None
-
- # Test with environment variables
- with mock.patch.dict(os.environ, {"LAMBDA_API_KEY": "test-key", "LAMBDA_API_BASE": "https://custom.lambda.ai/v1"}):
- api_base, api_key = config._get_openai_compatible_provider_info(None, None)
- assert api_base == "https://custom.lambda.ai/v1"
- assert api_key == "test-key"
-
- # Test with explicit parameters (should override env vars)
- with mock.patch.dict(os.environ, {"LAMBDA_API_KEY": "env-key", "LAMBDA_API_BASE": "https://env.lambda.ai/v1"}):
- api_base, api_key = config._get_openai_compatible_provider_info(
- "https://param.lambda.ai/v1", "param-key"
- )
- assert api_base == "https://param.lambda.ai/v1"
- assert api_key == "param-key"
-
-
-def test_get_llm_provider_lambda_ai():
- """Test that get_llm_provider correctly identifies Lambda AI"""
- from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
-
- # Test with lambda_ai/model-name format
- model, provider, api_key, api_base = get_llm_provider("lambda_ai/llama3.1-8b-instruct")
- assert model == "llama3.1-8b-instruct"
- assert provider == "lambda_ai"
-
- # Test with api_base containing Lambda AI endpoint
- model, provider, api_key, api_base = get_llm_provider(
- "llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1"
- )
- assert model == "llama3.1-8b-instruct"
- assert provider == "lambda_ai"
- assert api_base == "https://api.lambda.ai/v1"
-
-
-def test_lambda_ai_in_provider_lists():
- """Test that Lambda AI is registered in all necessary provider lists"""
- assert "lambda_ai" in litellm.openai_compatible_providers
- assert "lambda_ai" in litellm.provider_list
- assert "https://api.lambda.ai/v1" in litellm.openai_compatible_endpoints
-
-
-@pytest.mark.asyncio
-async def test_lambda_ai_completion_call():
- """Test completion call with Lambda AI provider (requires LAMBDA_API_KEY)"""
- # Skip if no API key is available
- if not os.getenv("LAMBDA_API_KEY"):
- pytest.skip("LAMBDA_API_KEY not set")
-
- try:
- response = await litellm.acompletion(
- model="lambda_ai/llama3.1-8b-instruct",
- messages=[{"role": "user", "content": "Hello, this is a test"}],
- max_tokens=10,
- )
- assert response.choices[0].message.content
- assert response.model
- assert response.usage
- except Exception as e:
- # If the API key is invalid or there's a network issue, that's okay
- # The important thing is that the provider was recognized
- if "lambda_ai" not in str(e) and "provider" not in str(e).lower():
- # Re-raise if it's not a provider-related error
- raise
-
-
-def test_lambda_ai_models_configuration():
- """Test that Lambda AI models are configured correctly"""
- from litellm import get_model_info
-
- # Reload model cost map to pick up local changes
- os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
- litellm.model_cost = litellm.get_model_cost_map(url="")
-
- # Clear and repopulate lambda_ai_models list after reloading model_cost
- litellm.lambda_ai_models = []
- litellm.add_known_models()
-
- # Some Lambda AI models to test
- lambda_ai_models = [
- "lambda_ai/deepseek-llama3.3-70b",
- "lambda_ai/hermes3-8b",
- "lambda_ai/llama3.1-8b-instruct",
- "lambda_ai/llama3.2-11b-vision-instruct",
- "lambda_ai/qwen25-coder-32b-instruct",
- ]
-
- for model in lambda_ai_models:
- model_info = get_model_info(model)
- assert model_info is not None, f"Model info not found for {model}"
- assert model_info.get("litellm_provider") == "lambda_ai", f"{model} should have lambda_ai as provider"
- assert model_info.get("mode") == "chat", f"{model} should be in chat mode"
- assert model_info.get("supports_function_calling") is True, f"{model} should support function calling"
- assert model_info.get("supports_system_messages") is True, f"{model} should support system messages"
-
- # Check vision support for vision models
- if "vision" in model:
- assert model_info.get("supports_vision") is True, f"{model} should support vision"
-
-
-def test_lambda_ai_model_list_populated():
- """Test that lambda_ai_models list is populated correctly"""
- # Ensure we're using local model cost map and repopulate models
- os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
- litellm.model_cost = litellm.get_model_cost_map(url="")
-
- # Clear and repopulate all model lists after reloading model_cost
- litellm.lambda_ai_models = []
- litellm.add_known_models()
-
- # This should be populated by the add_known_models function
- assert len(litellm.lambda_ai_models) > 0, "lambda_ai_models list should not be empty"
-
- # Check that all models in the list are Lambda AI models
- for model in litellm.lambda_ai_models:
- assert model.startswith("lambda_ai/"), f"Model {model} should start with 'lambda_ai/'"
-
- # Check some expected models are in the list
- expected_models = [
- "lambda_ai/llama3.1-8b-instruct",
- "lambda_ai/hermes3-405b",
- "lambda_ai/deepseek-v3-0324",
- ]
-
- for model in expected_models:
- assert model in litellm.lambda_ai_models, f"{model} should be in lambda_ai_models list"
\ No newline at end of file
diff --git a/tests/llm_translation/test_morph.py b/tests/llm_translation/test_morph.py
deleted file mode 100644
index 7d2568a0a7..0000000000
--- a/tests/llm_translation/test_morph.py
+++ /dev/null
@@ -1,108 +0,0 @@
-"""Unit tests for Morph provider integration."""
-
-import os
-import sys
-from unittest.mock import patch
-
-sys.path.insert(
- 0, os.path.abspath("../..")
-) # Adds the parent directory to the system path
-
-import litellm
-from litellm import MorphChatConfig, get_llm_provider
-
-# Force model loading
-litellm.add_known_models()
-
-
-def test_morph_config_get_provider_info():
- """Test that MorphChatConfig returns correct provider info."""
- config = MorphChatConfig()
-
- # Test with environment variable
- with patch.dict(os.environ, {"MORPH_API_KEY": "test-key-from-env"}):
- api_base, api_key = config._get_openai_compatible_provider_info(None, None)
- assert api_base == "https://api.morphllm.com/v1"
- assert api_key == "test-key-from-env"
-
- # Test with passed api_key
- api_base, api_key = config._get_openai_compatible_provider_info(None, "direct-key")
- assert api_base == "https://api.morphllm.com/v1"
- assert api_key == "direct-key"
-
- # Test with custom api_base
- api_base, api_key = config._get_openai_compatible_provider_info("https://custom.morph.com", "key")
- assert api_base == "https://custom.morph.com"
- assert api_key == "key"
-
-
-def test_morph_get_llm_provider():
- """Test that get_llm_provider correctly identifies morph models."""
- # Test with morph/model format
- _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-large")
- assert custom_llm_provider == "morph"
-
- _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-fast")
- assert custom_llm_provider == "morph"
-
-
-def test_morph_in_provider_lists():
- """Test that morph is included in all necessary provider lists."""
- import litellm
- from litellm.constants import openai_compatible_providers, openai_compatible_endpoints
-
- # Check morph is in openai_compatible_providers
- assert "morph" in openai_compatible_providers
-
- # Check morph endpoint is in openai_compatible_endpoints
- assert "https://api.morphllm.com/v1" in openai_compatible_endpoints
-
- # Check morph is in provider_list
- assert "morph" in litellm.provider_list
-
- # Check models are in model_list after initialization
- assert all(model in litellm.model_list for model in ["morph/morph-v3-large", "morph/morph-v3-fast"])
-
-
-def test_morph_model_info():
- """Test that morph models have correct configuration."""
- import litellm
- model_info = litellm.get_model_info("morph/morph-v3-large")
-
- assert model_info["litellm_provider"] == "morph"
- assert model_info["mode"] == "chat"
- assert model_info["max_tokens"] == 16000
- assert model_info["max_input_tokens"] == 16000
- assert model_info["max_output_tokens"] == 16000
- assert model_info["input_cost_per_token"] == 9e-07 # $0.9/1M tokens
- assert model_info["output_cost_per_token"] == 1.9e-06 # $1.9/1M tokens
- assert model_info["supports_function_calling"] is False
- assert model_info["supports_vision"] is False
- assert model_info["supports_system_messages"] is True
-
-
-def test_morph_supported_params():
- """Test that MorphChatConfig returns correct supported parameters."""
- config = MorphChatConfig()
- supported_params = config.get_supported_openai_params("morph/morph-v3-large")
-
- expected_params = [
- "messages",
- "model",
- "stream",
- "temperature",
- "max_tokens",
- "tools",
- "tool_choice",
- "response_format",
- ]
-
- assert all(param in supported_params for param in expected_params)
-
-
-def test_morph_custom_llm_provider():
- """Test that morph models are correctly identified."""
- config = MorphChatConfig()
- assert config.custom_llm_provider == "morph"
-
-
diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py
index 3ccdfe485b..79a1dec744 100644
--- a/tests/llm_translation/test_optional_params.py
+++ b/tests/llm_translation/test_optional_params.py
@@ -985,7 +985,7 @@ def test_watsonx_tool_choice():
model="gemini-1.5-pro", custom_llm_provider="watsonx", tool_choice="auto"
)
print(optional_params)
- assert optional_params["tool_choice_option"] == "auto"
+ assert optional_params["tool_choice_options"] == "auto"
def test_watsonx_text_top_k():
diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py
index 9c3ae1e1f4..8b6ed7ae59 100644
--- a/tests/local_testing/test_completion.py
+++ b/tests/local_testing/test_completion.py
@@ -760,7 +760,7 @@ def test_completion_base64(model):
pytest.fail(f"An exception occurred - {str(e)}")
-@pytest.mark.parametrize("model", ["claude-3-sonnet-20240229"])
+@pytest.mark.parametrize("model", ["claude-3-5-sonnet-latest"])
def test_completion_function_plus_image(model):
litellm.set_verbose = True
diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py
index 6851325cc1..807ffc6082 100644
--- a/tests/local_testing/test_exceptions.py
+++ b/tests/local_testing/test_exceptions.py
@@ -422,7 +422,7 @@ def test_anthropic_openai_exception():
old_azure_key = os.environ["ANTHROPIC_API_KEY"]
os.environ.pop("ANTHROPIC_API_KEY")
response = completion(
- model="anthropic/claude-3-sonnet-20240229",
+ model="anthropic/claude-3-5-sonnet-latest",
messages=[{"role": "user", "content": "hello"}],
)
print(f"response: {response}")
@@ -495,6 +495,7 @@ def test_completion_bedrock_invalid_role_exception():
== "litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}"
)
+
@pytest.mark.skip(reason="OpenAI exception changed to a generic error")
def test_content_policy_exceptionimage_generation_openai():
try:
@@ -773,7 +774,15 @@ def test_litellm_predibase_exception():
@pytest.mark.parametrize(
- "provider", ["predibase", "vertex_ai_beta", "anthropic", "databricks", "watsonx", "fireworks_ai"]
+ "provider",
+ [
+ "predibase",
+ "vertex_ai_beta",
+ "anthropic",
+ "databricks",
+ "watsonx",
+ "fireworks_ai",
+ ],
)
def test_exception_mapping(provider):
"""
@@ -826,14 +835,14 @@ def test_fireworks_ai_exception_mapping():
2. Text-based rate limit detection (the main issue fixed)
3. Generic 400 errors that should NOT be rate limits
4. ExceptionCheckers utility function
-
+
Related to: https://github.com/BerriAI/litellm/pull/11455
Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference
"""
import litellm
from litellm.llms.fireworks_ai.common_utils import FireworksAIException
from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
-
+
# Test scenarios covering all important cases
test_scenarios = [
{
@@ -855,57 +864,63 @@ def test_fireworks_ai_exception_mapping():
"expected_exception": litellm.BadRequestError,
},
]
-
+
# Test each scenario
for scenario in test_scenarios:
mock_exception = FireworksAIException(
- status_code=scenario["status_code"],
- message=scenario["message"],
- headers={}
+ status_code=scenario["status_code"], message=scenario["message"], headers={}
)
-
+
try:
response = litellm.completion(
model="fireworks_ai/llama-v3p1-70b-instruct",
messages=[{"role": "user", "content": "Hello"}],
mock_response=mock_exception,
)
- pytest.fail(f"Expected {scenario['expected_exception'].__name__} to be raised")
+ pytest.fail(
+ f"Expected {scenario['expected_exception'].__name__} to be raised"
+ )
except scenario["expected_exception"] as e:
if scenario["expected_exception"] == litellm.RateLimitError:
assert "rate limit" in str(e).lower() or "429" in str(e)
except Exception as e:
- pytest.fail(f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}")
-
+ pytest.fail(
+ f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}"
+ )
+
# Test ExceptionCheckers.is_error_str_rate_limit() method directly
-
+
# Test cases that should return True (rate limit detected)
rate_limit_strings = [
"429 rate limit exceeded",
- "Rate limit exceeded, please try again later",
+ "Rate limit exceeded, please try again later",
"RATE LIMIT ERROR",
"Error 429: rate limit",
'{"error":{"type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}',
"HTTP 429 Too Many Requests",
]
-
+
for error_str in rate_limit_strings:
- assert ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should detect rate limit in: {error_str}"
-
+ assert ExceptionCheckers.is_error_str_rate_limit(
+ error_str
+ ), f"Should detect rate limit in: {error_str}"
+
# Test cases that should return False (not rate limit)
non_rate_limit_strings = [
"400 Bad Request",
- "Authentication failed",
+ "Authentication failed",
"Invalid model specified",
"Context window exceeded",
"Internal server error",
"",
"Some other error message",
]
-
+
for error_str in non_rate_limit_strings:
- assert not ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should NOT detect rate limit in: {error_str}"
-
+ assert not ExceptionCheckers.is_error_str_rate_limit(
+ error_str
+ ), f"Should NOT detect rate limit in: {error_str}"
+
# Test edge cases
assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore
assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore
@@ -1142,6 +1157,7 @@ def test_openai_gateway_timeout_error():
"""
openai_client = OpenAI()
mapped_target = openai_client.chat.completions.with_raw_response # type: ignore
+
def _return_exception(*args, **kwargs):
import datetime
@@ -1175,13 +1191,17 @@ def test_openai_gateway_timeout_error():
setattr(exception, k, v)
raise exception
- try:
+ try:
with patch.object(
mapped_target,
"create",
side_effect=_return_exception,
):
- litellm.completion(model="openai/gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello world"}], client=openai_client)
+ litellm.completion(
+ model="openai/gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello world"}],
+ client=openai_client,
+ )
pytest.fail("Expected to raise Timeout")
except litellm.Timeout as e:
assert e.status_code == 504
@@ -1377,6 +1397,3 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model):
assert exc_info.value.code == "invalid_value"
assert exc_info.value.param is not None
assert exc_info.value.type == "invalid_request_error"
-
-
-
diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py
index a3443499d0..82ed0f2a08 100644
--- a/tests/local_testing/test_function_calling.py
+++ b/tests/local_testing/test_function_calling.py
@@ -791,7 +791,7 @@ async def test_watsonx_tool_choice(sync_mode):
mock_completion.assert_called_once()
print(mock_completion.call_args.kwargs)
json_data = json.loads(mock_completion.call_args.kwargs["data"])
- json_data["tool_choice_option"] == "auto"
+ json_data["tool_choice_options"] == "auto"
except Exception as e:
print(e)
if "The read operation timed out" in str(e):
diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py
index 3445dd5a51..bdc37bc3cc 100644
--- a/tests/local_testing/test_router.py
+++ b/tests/local_testing/test_router.py
@@ -125,12 +125,6 @@ async def test_router_provider_wildcard_routing():
print("response 3 = ", response3)
- response4 = await router.acompletion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": "hello"}],
- )
-
-
@pytest.mark.asyncio()
async def test_router_provider_wildcard_routing_regex():
"""
@@ -1136,7 +1130,7 @@ async def test_aimg_gen_on_router():
"api_base": os.getenv("AZURE_SWEDEN_API_BASE"),
"api_key": os.getenv("AZURE_SWEDEN_API_KEY"),
},
- }
+ },
]
router = Router(model_list=model_list, num_retries=3)
response = await router.aimage_generation(
@@ -2787,4 +2781,4 @@ def test_router_get_model_group_info():
assert model_group_info is not None
assert model_group_info.model_group == "gpt-4"
assert model_group_info.input_cost_per_token > 0
- assert model_group_info.output_cost_per_token > 0
\ No newline at end of file
+ assert model_group_info.output_cost_per_token > 0
diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py
index 5db4094677..924dff707e 100644
--- a/tests/local_testing/test_streaming.py
+++ b/tests/local_testing/test_streaming.py
@@ -642,6 +642,7 @@ def test_completion_ollama_hosted_stream():
"model",
[
# "claude-3-5-haiku-20241022",
+ # "claude-2",
# "mistral/mistral-small-latest",
"openrouter/openai/gpt-4o-mini",
],
@@ -672,7 +673,6 @@ def test_completion_model_stream(model):
pytest.fail(f"Error occurred: {e}")
-
@pytest.mark.parametrize(
"sync_mode",
[True, False],
@@ -889,6 +889,7 @@ async def test_completion_gemini_stream_accumulated_json(sync_mode):
# return
pytest.fail(f"Error occurred: {e}")
+
@pytest.mark.flaky(retries=3, delay=1)
def test_completion_mistral_api_mistral_large_function_call_with_streaming():
litellm.set_verbose = True
diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py
index bcdcc71ee3..e076b66b7d 100644
--- a/tests/test_litellm/llms/azure/test_azure_common_utils.py
+++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py
@@ -12,13 +12,7 @@ sys.path.insert(
) # Adds the parent directory to the system path
import litellm
from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token
-from litellm.secret_managers.get_azure_ad_token_provider import (
- get_azure_ad_token_provider,
-)
from litellm.types.router import GenericLiteLLMParams
-from litellm.types.secret_managers.get_azure_ad_token_provider import (
- AzureCredentialType,
-)
from litellm.types.utils import CallTypes
@@ -1304,7 +1298,7 @@ def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch):
# Verify the debug message was logged
setup_mocks["logger"].debug.assert_any_call(
- "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
+ "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
)
# Verify get_azure_ad_token_provider was called
@@ -1331,7 +1325,7 @@ def test_get_azure_ad_token_with_token_refresh_error(setup_mocks):
# Verify the debug message was logged
setup_mocks["logger"].debug.assert_any_call(
- "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
+ "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
)
# Verify error was logged
@@ -1339,8 +1333,8 @@ def test_get_azure_ad_token_with_token_refresh_error(setup_mocks):
"Azure AD Token Provider could not be used."
)
- # Verify get_azure_ad_token_provider was called twice (once for service principal, once for DefaultAzureCredential)
- assert setup_mocks["token_provider"].call_count == 2
+ # Verify get_azure_ad_token_provider was called
+ setup_mocks["token_provider"].assert_called_once()
# Verify the token is None since the provider raised an error
assert token is None
@@ -1386,101 +1380,3 @@ def test_token_provider_raises_exception(setup_mocks):
# Verify the error was logged
setup_mocks["logger"].error.assert_called()
-
-
-def test_get_azure_ad_token_provider_with_default_azure_credential():
- """
- Test that get_azure_ad_token_provider correctly uses DefaultAzureCredential
- when explicitly specified as the credential type. This verifies that the function
- can dynamically instantiate DefaultAzureCredential and return a working token provider.
- """
- # Mock Azure identity classes
- with patch('azure.identity.DefaultAzureCredential') as mock_default_cred, \
- patch('azure.identity.get_bearer_token_provider') as mock_token_provider:
-
- # Configure mocks
- mock_credential_instance = MagicMock()
- mock_default_cred.return_value = mock_credential_instance
- mock_token_provider.return_value = lambda: "test-default-azure-token"
-
- # Test with DefaultAzureCredential specified explicitly
- token_provider = get_azure_ad_token_provider(
- azure_scope="https://cognitiveservices.azure.com/.default",
- azure_credential=AzureCredentialType.DefaultAzureCredential
- )
-
- # Verify DefaultAzureCredential was instantiated
- mock_default_cred.assert_called_once_with()
-
- # Verify get_bearer_token_provider was called with the right parameters
- mock_token_provider.assert_called_once_with(
- mock_credential_instance,
- "https://cognitiveservices.azure.com/.default"
- )
-
- # Verify the returned token provider works
- token = token_provider()
- assert token == "test-default-azure-token"
-
-
-def test_get_azure_ad_token_fallback_to_default_azure_credential(setup_mocks, monkeypatch):
- """
- Test that get_azure_ad_token falls back to DefaultAzureCredential when the
- service principal method fails but token refresh is enabled. This tests the
- complete fallback flow from service principal to DefaultAzureCredential.
- """
- # Clear environment variables that might interfere
- monkeypatch.delenv("AZURE_USERNAME", raising=False)
- monkeypatch.delenv("AZURE_PASSWORD", raising=False)
- monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False)
- monkeypatch.delenv("AZURE_CLIENT_ID", raising=False)
- monkeypatch.delenv("AZURE_TENANT_ID", raising=False)
-
- # Reset mocks to ensure clean state
- setup_mocks["token_provider"].reset_mock()
-
- # Enable token refresh
- setup_mocks["litellm"].enable_azure_ad_token_refresh = True
-
- # Configure get_azure_ad_token_provider to fail first (service principal)
- # but succeed on second call (DefaultAzureCredential)
- def mock_token_provider_side_effect(*args, **kwargs):
- # If called with azure_credential=DefaultAzureCredential, return a working provider
- if kwargs.get("azure_credential") == AzureCredentialType.DefaultAzureCredential:
- return lambda: "mock-default-azure-credential-token"
- # Otherwise (service principal call), return None to simulate failure
- return None
-
- setup_mocks["token_provider"].side_effect = mock_token_provider_side_effect
-
- # Create test parameters with no other auth methods available
- litellm_params = GenericLiteLLMParams()
-
- # Call the function
- token = get_azure_ad_token(litellm_params)
-
- # Verify the success debug message was logged
- setup_mocks["logger"].debug.assert_any_call(
- "Successfully obtained Azure AD token provider using DefaultAzureCredential"
- )
-
- # Verify get_azure_ad_token_provider was called twice:
- # 1. First with just azure_scope (service principal attempt)
- # 2. Second with azure_credential=DefaultAzureCredential (fallback)
- assert setup_mocks["token_provider"].call_count == 2
-
- # Verify the calls were made with expected parameters
- calls = setup_mocks["token_provider"].call_args_list
-
- # First call should be service principal attempt (no azure_credential)
- first_call_kwargs = calls[0][1]
- assert "azure_scope" in first_call_kwargs
- assert first_call_kwargs.get("azure_credential") is None
-
- # Second call should be DefaultAzureCredential attempt
- second_call_kwargs = calls[1][1]
- assert "azure_scope" in second_call_kwargs
- assert second_call_kwargs.get("azure_credential") == AzureCredentialType.DefaultAzureCredential
-
- # Verify the token is what we expect from our DefaultAzureCredential mock
- assert token == "mock-default-azure-credential-token"
diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py
index c6ae2b9c4e..65413d46b9 100644
--- a/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py
+++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py
@@ -178,14 +178,3 @@ class TestGitHubCopilotAuthenticator:
authenticator._get_device_code.assert_called_once()
authenticator._poll_for_access_token.assert_called_once_with("mock-device-code")
mock_print.assert_called_once()
-
- def test_get_api_base_from_file(self, authenticator):
- """Test retrieving the API base endpoint from a file."""
- mock_api_key_data = json.dumps({
- "token": "mock-api-key",
- "expires_at": (datetime.now() + timedelta(hours=1)).timestamp(),
- "endpoints": {"api": "https://api.enterprise.githubcopilot.com"}
- })
- with patch("builtins.open", mock_open(read_data=mock_api_key_data)):
- api_base = authenticator.get_api_base()
- assert api_base == "https://api.enterprise.githubcopilot.com"
diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py
index d9afbff53a..9672f045df 100644
--- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py
+++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py
@@ -40,8 +40,6 @@ def test_github_copilot_config_get_openai_compatible_provider_info():
mock_api_key = "gh.test-key-123456789"
config.authenticator = MagicMock()
config.authenticator.get_api_key.return_value = mock_api_key
- # Test with dynamic endpoint
- config.authenticator.get_api_base.return_value = "https://api.enterprise.githubcopilot.com"
# Test with default values
model = "github_copilot/gpt-4"
@@ -56,24 +54,10 @@ def test_github_copilot_config_get_openai_compatible_provider_info():
custom_llm_provider="github_copilot",
)
- assert api_base == "https://api.enterprise.githubcopilot.com"
+ assert api_base == "https://api.githubcopilot.com/"
assert dynamic_api_key == mock_api_key
assert custom_llm_provider == "github_copilot"
- # Test fallback to default if no dynamic endpoint
- config.authenticator.get_api_base.return_value = None
- (
- api_base,
- dynamic_api_key,
- custom_llm_provider,
- ) = config._get_openai_compatible_provider_info(
- model=model,
- api_base=None,
- api_key=None,
- custom_llm_provider="github_copilot",
- )
- assert api_base == "https://api.githubcopilot.com/"
-
# Test with authentication failure
config.authenticator.get_api_key.side_effect = GetAPIKeyError(
message="Failed to get API key",
diff --git a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py
deleted file mode 100644
index 4dd610ac86..0000000000
--- a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py
+++ /dev/null
@@ -1,270 +0,0 @@
-import json
-import os
-import sys
-from typing import List, Optional
-from unittest.mock import MagicMock, patch
-
-import httpx
-import pytest
-
-sys.path.insert(
- 0, os.path.abspath("../../../../..")
-) # Adds the parent directory to the system path
-
-from litellm.llms.recraft.image_generation.transformation import (
- RecraftImageGenerationConfig,
-)
-from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
-from litellm.types.utils import ImageObject, ImageResponse
-
-
-class TestRecraftImageGenerationTransformation:
- def setup_method(self):
- """Set up test fixtures before each test method."""
- self.config = RecraftImageGenerationConfig()
- self.model = "recraft-v3"
- self.logging_obj = MagicMock()
-
-
- def test_map_openai_params_supported_params(self):
- """Test that map_openai_params correctly maps supported parameters."""
- non_default_params = {
- "n": 2,
- "response_format": "url",
- "size": "1024x1024",
- "style": "photographic"
- }
- optional_params = {}
-
- result = self.config.map_openai_params(
- non_default_params=non_default_params,
- optional_params=optional_params,
- model=self.model,
- drop_params=False
- )
-
- assert result == non_default_params
-
- def test_map_openai_params_unsupported_param_drop_true(self):
- """Test that map_openai_params drops unsupported parameters when drop_params=True."""
- non_default_params = {
- "n": 2,
- "unsupported_param": "value"
- }
- optional_params = {}
-
- result = self.config.map_openai_params(
- non_default_params=non_default_params,
- optional_params=optional_params,
- model=self.model,
- drop_params=True
- )
-
- assert result == {"n": 2}
- assert "unsupported_param" not in result
-
- def test_map_openai_params_unsupported_param_drop_false(self):
- """Test that map_openai_params raises ValueError for unsupported parameters when drop_params=False."""
- non_default_params = {
- "n": 2,
- "unsupported_param": "value"
- }
- optional_params = {}
-
- with pytest.raises(ValueError) as exc_info:
- self.config.map_openai_params(
- non_default_params=non_default_params,
- optional_params=optional_params,
- model=self.model,
- drop_params=False
- )
-
- assert "unsupported_param" in str(exc_info.value)
- assert "is not supported for model" in str(exc_info.value)
-
- @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str")
- def test_get_complete_url_with_api_base(self, mock_get_secret):
- """Test that get_complete_url returns correct URL when api_base is provided."""
- api_base = "https://custom.api.recraft.ai"
-
- result = self.config.get_complete_url(
- api_base=api_base,
- api_key="test_key",
- model=self.model,
- optional_params={},
- litellm_params={}
- )
-
- expected_url = f"{api_base}/{self.config.IMAGE_GENERATION_ENDPOINT}"
- assert result == expected_url
- mock_get_secret.assert_not_called()
-
- @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str")
- def test_get_complete_url_with_secret_base(self, mock_get_secret):
- """Test that get_complete_url uses secret when api_base is None."""
- mock_get_secret.return_value = "https://secret.api.recraft.ai"
-
- result = self.config.get_complete_url(
- api_base=None,
- api_key="test_key",
- model=self.model,
- optional_params={},
- litellm_params={}
- )
-
- expected_url = f"https://secret.api.recraft.ai/{self.config.IMAGE_GENERATION_ENDPOINT}"
- assert result == expected_url
- mock_get_secret.assert_called_once_with("RECRAFT_API_BASE")
-
- @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str")
- def test_get_complete_url_with_default_base(self, mock_get_secret):
- """Test that get_complete_url uses default base URL when no other options are available."""
- mock_get_secret.return_value = None
-
- result = self.config.get_complete_url(
- api_base=None,
- api_key="test_key",
- model=self.model,
- optional_params={},
- litellm_params={}
- )
-
- expected_url = f"{self.config.DEFAULT_BASE_URL}/{self.config.IMAGE_GENERATION_ENDPOINT}"
- assert result == expected_url
-
- @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str")
- def test_validate_environment_with_api_key(self, mock_get_secret):
- """Test that validate_environment correctly sets authorization header when api_key is provided."""
- headers = {}
- api_key = "test_api_key"
-
- result = self.config.validate_environment(
- headers=headers,
- model=self.model,
- messages=[],
- optional_params={},
- litellm_params={},
- api_key=api_key
- )
-
- assert result["Authorization"] == f"Bearer {api_key}"
- mock_get_secret.assert_not_called()
-
- @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str")
- def test_validate_environment_with_secret_key(self, mock_get_secret):
- """Test that validate_environment uses secret API key when api_key is None."""
- mock_get_secret.return_value = "secret_api_key"
- headers = {}
-
- result = self.config.validate_environment(
- headers=headers,
- model=self.model,
- messages=[],
- optional_params={},
- litellm_params={},
- api_key=None
- )
-
- assert result["Authorization"] == "Bearer secret_api_key"
- mock_get_secret.assert_called_once_with("RECRAFT_API_KEY")
-
- @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str")
- def test_validate_environment_no_api_key_raises_error(self, mock_get_secret):
- """Test that validate_environment raises ValueError when no API key is available."""
- mock_get_secret.return_value = None
- headers = {}
-
- with pytest.raises(ValueError) as exc_info:
- self.config.validate_environment(
- headers=headers,
- model=self.model,
- messages=[],
- optional_params={},
- litellm_params={},
- api_key=None
- )
-
- assert "RECRAFT_API_KEY is not set" in str(exc_info.value)
-
- def test_transform_image_generation_request(self):
- """Test that transform_image_generation_request correctly transforms request parameters."""
- prompt = "A beautiful sunset over mountains"
- optional_params = {
- "n": 2,
- "size": "1024x1024",
- "style": "photographic"
- }
- litellm_params = {}
- headers = {}
-
- result = self.config.transform_image_generation_request(
- model=self.model,
- prompt=prompt,
- optional_params=optional_params,
- litellm_params=litellm_params,
- headers=headers
- )
-
- assert result["prompt"] == prompt
- assert result["model"] == self.model
- assert result["n"] == 2
- assert result["size"] == "1024x1024"
- assert result["style"] == "photographic"
-
- def test_transform_image_generation_response_success(self):
- """Test that transform_image_generation_response correctly transforms successful response."""
- # Mock response data
- response_data = {
- "data": [
- {"url": "https://example.com/image1.jpg", "b64_json": None},
- {"url": None, "b64_json": "base64encodeddata"}
- ]
- }
-
- # Create mock response
- mock_response = MagicMock()
- mock_response.json.return_value = response_data
-
- # Create empty model response
- model_response = ImageResponse(data=[])
-
- result = self.config.transform_image_generation_response(
- model=self.model,
- raw_response=mock_response,
- model_response=model_response,
- logging_obj=self.logging_obj,
- request_data={},
- optional_params={},
- litellm_params={},
- encoding=None
- )
-
- assert len(result.data) == 2
- assert result.data[0].url == "https://example.com/image1.jpg"
- assert result.data[0].b64_json is None
- assert result.data[1].url is None
- assert result.data[1].b64_json == "base64encodeddata"
-
- def test_transform_image_generation_response_json_error(self):
- """Test that transform_image_generation_response raises error when response JSON is invalid."""
- # Create mock response that raises JSON decode error
- mock_response = MagicMock()
- mock_response.json.side_effect = json.JSONDecodeError("Invalid JSON", "", 0)
- mock_response.status_code = 500
- mock_response.headers = {}
-
- model_response = ImageResponse(data=[])
-
- with pytest.raises(Exception) as exc_info:
- self.config.transform_image_generation_response(
- model=self.model,
- raw_response=mock_response,
- model_response=model_response,
- logging_obj=self.logging_obj,
- request_data={},
- optional_params={},
- litellm_params={},
- encoding=None
- )
-
- assert "Error transforming image generation response" in str(exc_info.value)
\ No newline at end of file
diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py
index fd9c366a2f..b9a71e9621 100644
--- a/tests/test_litellm/test_main.py
+++ b/tests/test_litellm/test_main.py
@@ -1097,3 +1097,28 @@ def test_stream_chunk_builder_thinking_blocks():
assert response is not None
assert response.choices[0].message.content is not None
assert response.choices[0].message.thinking_blocks is not None
+
+
+from litellm.llms.openai.openai import OpenAIChatCompletion
+
+
+def throw_retryable_error(*_, **__):
+ raise RuntimeError("BOOM")
+
+
+@pytest.mark.asyncio
+async def test_retrying() -> None:
+ litellm.num_retries = 10
+ with (
+ patch.object(
+ OpenAIChatCompletion,
+ "make_openai_chat_completion_request",
+ side_effect=throw_retryable_error,
+ ) as mock_request,
+ pytest.raises(litellm.InternalServerError, match="LiteLLM Retried: 10 times"),
+ ):
+ await litellm.acompletion(
+ model="gpt-4o-mini",
+ messages=[{"role": "user", "content": "Hello"}],
+ )
+ assert mock_request.call_count >= 10, "Expected retrying to be used"
diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py
index 16b9838d80..0b5f1a9433 100644
--- a/tests/test_openai_endpoints.py
+++ b/tests/test_openai_endpoints.py
@@ -570,7 +570,7 @@ async def test_proxy_all_models():
await chat_completion(
session=session,
key=LITELLM_MASTER_KEY,
- model="anthropic/claude-3-sonnet-20240229",
+ model="anthropic/claude-3-5-sonnet-latest",
)
diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx
index e72b6330c3..402f0b26e9 100644
--- a/ui/litellm-dashboard/src/components/team/team_info.tsx
+++ b/ui/litellm-dashboard/src/components/team/team_info.tsx
@@ -30,7 +30,6 @@ import {
teamMemberUpdateCall,
Member,
teamUpdateCall,
- getGuardrailsList,
} from "@/components/networking";
import { Button, Form, Input, Select, message, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
@@ -148,7 +147,6 @@ const TeamInfoView: React.FC = ({
const [mcpAccessGroups, setMcpAccessGroups] = useState([]);
const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false);
const [copiedStates, setCopiedStates] = useState>({})
- const [guardrailsList, setGuardrailsList] = useState([]);
console.log("userModels in team info", userModels);
@@ -184,23 +182,6 @@ const TeamInfoView: React.FC = ({
}
};
- useEffect(() => {
- const fetchGuardrails = async () => {
- try {
- if (!accessToken) return;
- const response = await getGuardrailsList(accessToken);
- const guardrailNames = response.guardrails.map(
- (g: { guardrail_name: string }) => g.guardrail_name
- );
- setGuardrailsList(guardrailNames);
- } catch (error) {
- console.error("Failed to fetch guardrails:", error);
- }
- };
-
- fetchGuardrails();
- }, [accessToken]);
-
const handleMemberCreate = async (values: any) => {
try {
if (accessToken == null) return;
@@ -692,14 +673,13 @@ const TeamInfoView: React.FC = ({