mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-02 10:21:52 +00:00
Merge branch 'main' into litellm_dev_10_10_2025_p2
This commit is contained in:
Krish Dholakia
+36
-9
@@ -676,18 +676,16 @@ jobs:
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install PostgreSQL
|
||||
command: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install postgresql postgresql-contrib
|
||||
echo 'export PATH=/usr/lib/postgresql/*/bin:$PATH' >> $BASH_ENV
|
||||
- setup_google_dns
|
||||
- run:
|
||||
name: Show git commit hash
|
||||
command: |
|
||||
echo "Git commit hash: $CIRCLE_SHA1"
|
||||
|
||||
- run:
|
||||
name: Install PostgreSQL
|
||||
command: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y postgresql-14 postgresql-contrib-14
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
@@ -2375,6 +2373,25 @@ jobs:
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "assemblyai==0.37.0"
|
||||
- run:
|
||||
name: Install dockerize
|
||||
command: |
|
||||
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
||||
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
||||
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
||||
- run:
|
||||
name: Start PostgreSQL Database
|
||||
command: |
|
||||
docker run -d \
|
||||
--name postgres-db \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=circle_test \
|
||||
-p 5432:5432 \
|
||||
postgres:14
|
||||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
@@ -2385,10 +2402,11 @@ jobs:
|
||||
command: |
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e DATABASE_URL=$CLEAN_STORE_MODEL_IN_DB_DATABASE_URL \
|
||||
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
||||
-e STORE_MODEL_IN_DB="True" \
|
||||
-e LITELLM_MASTER_KEY="sk-1234" \
|
||||
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
@@ -2418,7 +2436,16 @@ jobs:
|
||||
python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
|
||||
no_output_timeout:
|
||||
120m
|
||||
# Clean up first container
|
||||
- run:
|
||||
name: Stop and remove containers
|
||||
command: |
|
||||
docker stop my-app || true
|
||||
docker rm my-app || true
|
||||
docker stop postgres-db || true
|
||||
docker rm postgres-db || true
|
||||
when: always
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
|
||||
proxy_build_from_pip_tests:
|
||||
# Change from docker to machine executor
|
||||
|
||||
+46
-1
@@ -68,8 +68,11 @@ run_grype_scans() {
|
||||
|
||||
# Allowlist of CVEs to be ignored in failure threshold/reporting
|
||||
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
|
||||
# - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
|
||||
ALLOWED_CVES=(
|
||||
"CVE-2025-8869"
|
||||
"GHSA-4xh5-x5gv-qwph"
|
||||
"CVE-2025-8291" # no fix available as of Oct 11, 2025
|
||||
)
|
||||
|
||||
# Build JSON array of allowlisted CVE IDs for jq
|
||||
@@ -77,6 +80,26 @@ run_grype_scans() {
|
||||
|
||||
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
|
||||
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
|
||||
echo ""
|
||||
|
||||
# Show all high-severity vulnerabilities for transparency
|
||||
TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
|
||||
.matches[]
|
||||
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
|
||||
| .vulnerability.id' | wc -l)
|
||||
|
||||
if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
|
||||
echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
|
||||
echo ""
|
||||
echo "All high-severity vulnerabilities (including allowlisted):"
|
||||
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
|
||||
["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
|
||||
(.matches[]
|
||||
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
|
||||
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
|
||||
| @tsv' | column -t -s $'\t'
|
||||
echo ""
|
||||
fi
|
||||
|
||||
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
|
||||
.matches[]
|
||||
@@ -85,8 +108,17 @@ run_grype_scans() {
|
||||
| .vulnerability.id' | wc -l)
|
||||
|
||||
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
|
||||
echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "ERROR: Security Scan Failed"
|
||||
echo "=========================================="
|
||||
echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
|
||||
echo ""
|
||||
echo "These vulnerabilities are NOT in the allowlist and must be addressed."
|
||||
echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
|
||||
echo ""
|
||||
echo "Detailed vulnerability report:"
|
||||
echo ""
|
||||
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
|
||||
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
|
||||
(.matches[]
|
||||
@@ -94,6 +126,19 @@ run_grype_scans() {
|
||||
| select((.vulnerability.id as $id | $allow | index($id) | not))
|
||||
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
|
||||
| @tsv' | column -t -s $'\t'
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Action Required:"
|
||||
echo "=========================================="
|
||||
echo "1. If a fix is available, update the package to the fixed version"
|
||||
echo "2. If the vulnerability is not applicable or has no fix:"
|
||||
echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
|
||||
echo " - Add a comment explaining why it's safe to ignore"
|
||||
echo ""
|
||||
echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
|
||||
echo "Add all relevant IDs to the allowlist if they refer to the same issue."
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
exit 1
|
||||
else
|
||||
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ services:
|
||||
#########################################
|
||||
## Uncomment these lines to start proxy with a config.yaml file ##
|
||||
# volumes:
|
||||
# - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
|
||||
# - ./config.yaml:/app/config.yaml
|
||||
# command:
|
||||
# - "--config=/app/config.yaml"
|
||||
##############################################
|
||||
|
||||
@@ -6,18 +6,27 @@ LiteLLM supports the following models for OCI on-demand GenAI API.
|
||||
|
||||
Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm) to see if the model is available for your region.
|
||||
|
||||
## Supported Models
|
||||
|
||||
### Meta Llama Models
|
||||
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
|
||||
- `meta.llama-4-scout-17b-16e-instruct`
|
||||
- `meta.llama-3.3-70b-instruct`
|
||||
- `meta.llama-3.2-90b-vision-instruct`
|
||||
- `meta.llama-3.1-405b-instruct`
|
||||
|
||||
### xAI Grok Models
|
||||
- `xai.grok-4`
|
||||
- `xai.grok-3`
|
||||
- `xai.grok-3-fast`
|
||||
- `xai.grok-3-mini`
|
||||
- `xai.grok-3-mini-fast`
|
||||
|
||||
### Cohere Models
|
||||
- `cohere.command-latest`
|
||||
- `cohere.command-a-03-2025`
|
||||
- `cohere.command-plus-latest`
|
||||
|
||||
## Authentication
|
||||
|
||||
LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters:
|
||||
@@ -83,3 +92,24 @@ response = completion(
|
||||
for chunk in response:
|
||||
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
|
||||
```
|
||||
|
||||
## Usage Examples by Model Type
|
||||
|
||||
### Using Cohere Models
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
messages = [{"role": "user", "content": "Explain quantum computing"}]
|
||||
response = completion(
|
||||
model="oci/cohere.command-latest",
|
||||
messages=messages,
|
||||
oci_region="us-chicago-1",
|
||||
oci_user=<your_oci_user>,
|
||||
oci_fingerprint=<your_oci_fingerprint>,
|
||||
oci_tenancy=<your_oci_tenancy>,
|
||||
oci_key=<string_with_content_of_oci_key>,
|
||||
oci_compartment_id=<oci_compartment_id>,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
@@ -339,6 +339,72 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
| fine tuned `gpt-3.5-turbo-1106` | `response = completion(model="ft:gpt-3.5-turbo-1106", messages=messages)` |
|
||||
| fine tuned `gpt-3.5-turbo-0613` | `response = completion(model="ft:gpt-3.5-turbo-0613", messages=messages)` |
|
||||
|
||||
## Getting Reasoning Content in `/chat/completions`
|
||||
|
||||
GPT-5 models return reasoning content when called via the Responses API. You can call these models via the `/chat/completions` endpoint by using the `openai/responses/` prefix.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
```python
|
||||
response = litellm.completion(
|
||||
model="openai/responses/gpt-5-mini", # tells litellm to call the model via the Responses API
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
reasoning_effort="low",
|
||||
)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "openai/responses/gpt-5-mini",
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"reasoning_effort": "low"
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
Expected Response:
|
||||
```json
|
||||
{
|
||||
"id": "chatcmpl-6382a222-43c9-40c4-856b-22e105d88075",
|
||||
"created": 1760146746,
|
||||
"model": "gpt-5-mini",
|
||||
"object": "chat.completion",
|
||||
"system_fingerprint": null,
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "Paris",
|
||||
"role": "assistant",
|
||||
"tool_calls": null,
|
||||
"function_call": null,
|
||||
"reasoning_content": "**Identifying the capital**\n\nThe user wants me to think of the capital of France and write it down. That's pretty straightforward: it's Paris. There aren't any safety issues to consider here. I think it would be best to keep it concise, so maybe just \"Paris\" would suffice. I feel confident that I should just stick to that without adding anything else. So, let's write it down!",
|
||||
"provider_specific_fields": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 7,
|
||||
"prompt_tokens": 18,
|
||||
"total_tokens": 25,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": {
|
||||
"audio_tokens": null,
|
||||
"cached_tokens": 0,
|
||||
"text_tokens": null,
|
||||
"image_tokens": null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## OpenAI Chat Completion to Responses API Bridge
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
## **Batch APIs**
|
||||
# Vertex Batch APIs
|
||||
|
||||
Just add the following Vertex env vars to your environment.
|
||||
|
||||
|
||||
@@ -124,10 +124,7 @@ Deploy Gemma models on custom Vertex AI prediction endpoints with OpenAI-compati
|
||||
| Vertex Documentation | [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions) |
|
||||
| Required Parameter | `api_base` - Full prediction endpoint URL |
|
||||
|
||||
### Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="Proxy">
|
||||
**Proxy Usage:**
|
||||
|
||||
**1. Add to config.yaml**
|
||||
|
||||
@@ -160,9 +157,7 @@ curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="sdk" label="SDK">
|
||||
**SDK Usage:**
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
@@ -176,5 +171,59 @@ response = completion(
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
## MedGemma Models (Custom Endpoints)
|
||||
|
||||
Deploy MedGemma models on custom Vertex AI prediction endpoints with OpenAI-compatible format. MedGemma models use the same `vertex_ai/gemma/` route.
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Provider Route | `vertex_ai/gemma/{MODEL_NAME}` |
|
||||
| Vertex Documentation | [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions) |
|
||||
| Required Parameter | `api_base` - Full prediction endpoint URL |
|
||||
|
||||
**Proxy Usage:**
|
||||
|
||||
**1. Add to config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: medgemma-model
|
||||
litellm_params:
|
||||
model: vertex_ai/gemma/medgemma-2b-v1
|
||||
api_base: https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict
|
||||
vertex_project: "my-project-id"
|
||||
vertex_location: "us-central1"
|
||||
```
|
||||
|
||||
**2. Start proxy**
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it**
|
||||
|
||||
```bash
|
||||
curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "medgemma-model",
|
||||
"messages": [{"role": "user", "content": "What are the symptoms of hypertension?"}],
|
||||
"max_tokens": 100
|
||||
}'
|
||||
```
|
||||
|
||||
**SDK Usage:**
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="vertex_ai/gemma/medgemma-2b-v1",
|
||||
messages=[{"role": "user", "content": "What are the symptoms of hypertension?"}],
|
||||
api_base="https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict",
|
||||
vertex_project="my-project-id",
|
||||
vertex_location="us-central1",
|
||||
)
|
||||
```
|
||||
|
||||
@@ -780,6 +780,8 @@ router_settings:
|
||||
| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
|
||||
| USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
|
||||
| WEBHOOK_URL | URL for receiving webhooks from external services
|
||||
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run |
|
||||
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 |
|
||||
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000 |
|
||||
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run
|
||||
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
|
||||
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000
|
||||
| DEFAULT_SHARED_HEALTH_CHECK_TTL | Time-to-live in seconds for cached health check results in shared health check mode. Default is 300 (5 minutes)
|
||||
| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute)
|
||||
@@ -0,0 +1,277 @@
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Setting Tag Budgets
|
||||
|
||||
Track spend and set budgets for your API requests using tags. Tags allow you to categorize and monitor costs across different cost centers, projects, and departments.
|
||||
|
||||
## Pre-Requisites
|
||||
|
||||
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
|
||||
|
||||
## What are Tags?
|
||||
|
||||
Tags are labels you can attach to your LLM requests to track and limit spending by category.
|
||||
|
||||
**Common Use Cases:**
|
||||
- **Cost Center Tracking**: Allocate LLM costs to specific departments or business units (e.g., "engineering", "marketing", "customer-support")
|
||||
- **Project-based Budgeting**: Set budgets for different projects or initiatives (e.g., "project-alpha", "chatbot-v2")
|
||||
- **Customer Attribution**: Track spend per customer or client (e.g., "customer-acme", "customer-techcorp")
|
||||
- **Feature Monitoring**: Monitor costs for specific features (e.g., "feature-chat", "feature-summarization")
|
||||
|
||||
Tags are added to each request in the `metadata` field to track and enforce budget limits.
|
||||
|
||||
## Setting Tag Budgets
|
||||
|
||||
### 1. Create a tag with budget
|
||||
|
||||
Create a tag to represent a cost center, project, or any budget category. Set `max_budget` ($ value allowed) and `budget_duration` (how frequently the budget resets).
|
||||
|
||||
**Example:** Create a tag for your Engineering department with a monthly $500 budget
|
||||
|
||||
#### API
|
||||
|
||||
Create a new tag and set `max_budget` and `budget_duration`
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/tag/new' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"name": "engineering",
|
||||
"description": "Engineering department cost center",
|
||||
"max_budget": 500.0,
|
||||
"budget_duration": "30d"
|
||||
}'
|
||||
```
|
||||
|
||||
**Request Body Parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique name for the tag (e.g., cost center name) |
|
||||
| `description` | string | No | Description of what this tag tracks |
|
||||
| `models` | list[string] | No | Restrict tag to specific models |
|
||||
| `max_budget` | float | No | Maximum budget in USD |
|
||||
| `budget_duration` | string | No | How often budget resets (e.g., "30d", "1d") |
|
||||
| `soft_budget` | float | No | Soft budget limit for warnings |
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "engineering",
|
||||
"description": "Engineering department cost center",
|
||||
"max_budget": 500.0,
|
||||
"budget_duration": "30d",
|
||||
"budget_reset_at": "2025-11-10T00:00:00Z",
|
||||
"created_at": "2025-10-11T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
#### LiteLLM Admin UI
|
||||
|
||||
Navigate to the **Tag Management** page and click **Create New Tag**. Fill in the tag details and set your budget:
|
||||
|
||||
<Image
|
||||
img={require('../../img/tag_budget1.png')}
|
||||
style={{width: '80%', display: 'block', margin: '0'}}
|
||||
/>
|
||||
|
||||
<br />
|
||||
|
||||
|
||||
**Possible values for `budget_duration`:**
|
||||
|
||||
| `budget_duration` | When Budget will reset |
|
||||
| --- | --- |
|
||||
| `budget_duration="1s"` | every 1 second |
|
||||
| `budget_duration="1m"` | every 1 minute |
|
||||
| `budget_duration="1h"` | every 1 hour |
|
||||
| `budget_duration="1d"` | every 1 day |
|
||||
| `budget_duration="7d"` | every 1 week |
|
||||
| `budget_duration="30d"` | every 1 month |
|
||||
|
||||
### 2. Use the tag in your requests
|
||||
|
||||
Add tags to your API requests in the `metadata` field:
|
||||
|
||||
:::info Tags Budgets on API Keys
|
||||
|
||||
Currently, tag budget enforcement is only supported per request. If you'd like to set tags on API keys so all requests automatically inherit the tags budgets, please [create a feature request on GitHub](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeat%5D%3A).
|
||||
|
||||
:::
|
||||
|
||||
<Tabs>
|
||||
|
||||
<TabItem value="openai" label="OpenAI SDK">
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # Your LiteLLM proxy key
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
extra_body={
|
||||
"metadata": {
|
||||
"tags": ["engineering"]
|
||||
}
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="cURL">
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"metadata": {
|
||||
"tags": ["engineering"]
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
### 3. Test It
|
||||
|
||||
Make requests until the budget is exceeded:
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"metadata": {
|
||||
"tags": ["engineering"]
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**When budget is exceeded, you'll see:**
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "Budget has been exceeded! Tag=engineering Current cost: 505.50, Max budget: 500.0",
|
||||
"type": "budget_exceeded",
|
||||
"param": null,
|
||||
"code": "400"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Managing Tags
|
||||
|
||||
### View Tag Information
|
||||
|
||||
Get information about specific tags:
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/tag/info' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"names": ["engineering", "marketing"]
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"engineering": {
|
||||
"name": "engineering",
|
||||
"description": "Engineering department cost center",
|
||||
"spend": 245.50,
|
||||
"max_budget": 500.0,
|
||||
"budget_duration": "30d",
|
||||
"budget_reset_at": "2025-11-10T00:00:00Z",
|
||||
"created_at": "2025-10-11T00:00:00Z",
|
||||
"updated_at": "2025-10-11T12:30:00Z"
|
||||
},
|
||||
"marketing": {
|
||||
"name": "marketing",
|
||||
"description": "Marketing department cost center",
|
||||
"spend": 89.20,
|
||||
"max_budget": 300.0,
|
||||
"budget_duration": "30d",
|
||||
"budget_reset_at": "2025-11-10T00:00:00Z",
|
||||
"created_at": "2025-10-11T00:00:00Z",
|
||||
"updated_at": "2025-10-11T12:30:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Update Tag Budget
|
||||
|
||||
Update an existing tag's budget:
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/tag/update' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"name": "engineering",
|
||||
"max_budget": 750.0,
|
||||
"budget_duration": "30d"
|
||||
}'
|
||||
```
|
||||
|
||||
### Delete Tag
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/tag/delete' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"name": "engineering"
|
||||
}'
|
||||
```
|
||||
|
||||
## Multiple Tags per Request
|
||||
|
||||
You can apply multiple tags to a single request to track costs across different dimensions simultaneously. For example, track both the cost center and the specific project:
|
||||
|
||||
```python
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
extra_body={
|
||||
"metadata": {
|
||||
"tags": ["engineering", "project-alpha", "customer-acme"]
|
||||
}
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"metadata": {
|
||||
"tags": ["engineering", "project-alpha", "customer-acme"]
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Budget Enforcement:** If any tag exceeds its budget, the request will be rejected.
|
||||
@@ -105,7 +105,7 @@ LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual ke
|
||||
Alternatively, use the Anthropic pass-through endpoint:
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
|
||||
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
|
||||
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
|
||||
```
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 305 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 416 KiB |
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: "[Preview] v1.77.7-stable - Claude Sonnet 4.5"
|
||||
title: "v1.77.7-stable - 2.9x Lower Median Latency"
|
||||
slug: "v1-77-7"
|
||||
date: 2025-10-04T10:00:00
|
||||
authors:
|
||||
@@ -15,7 +15,7 @@ authors:
|
||||
title: Backend Performance Engineer
|
||||
url: https://www.linkedin.com/in/alexsander-baptista/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D5603AQGXnziu4kqNCQ/profile-displayphoto-crop_800_800/B56ZkxEcuOKEAI-/0/1757464874550?e=1762387200&v=beta&t=9SNXLsWhx8OnYPAMQ9fqAr02oevDYEAL2vMYg2f9ieg
|
||||
- name: Achintya Srivastava
|
||||
- name: Achintya Rajan
|
||||
title: Fullstack Engineer
|
||||
url: https://www.linkedin.com/in/achintya-rajan/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D5603AQGdkEeyJTdljw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1716271140869?e=1762387200&v=beta&t=9gOoLPeqR2E5z3KSX61EUj3HVZXmgo87vhVuSHeffjc
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
---
|
||||
title: "[Preview] v1.78.0-stable - MCP Gateway: Control Tool Access by Team/Key"
|
||||
slug: "v1-78-0"
|
||||
date: 2025-10-11T10:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
- name: Alexsander Hamir
|
||||
title: Backend Performance Engineer
|
||||
url: https://www.linkedin.com/in/alexsander-baptista/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D5603AQGXnziu4kqNCQ/profile-displayphoto-crop_800_800/B56ZkxEcuOKEAI-/0/1757464874550?e=1762387200&v=beta&t=9SNXLsWhx8OnYPAMQ9fqAr02oevDYEAL2vMYg2f9ieg
|
||||
- name: Achintya Rajan
|
||||
title: Fullstack Engineer
|
||||
url: https://www.linkedin.com/in/achintya-rajan/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D5603AQGdkEeyJTdljw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1716271140869?e=1762387200&v=beta&t=9gOoLPeqR2E5z3KSX61EUj3HVZXmgo87vhVuSHeffjc
|
||||
- name: Sameer Kankute
|
||||
title: Backend Engineer (LLM Translation)
|
||||
url: https://www.linkedin.com/in/sameer-kankute/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1762387200&v=beta&t=0jbuX-f4eSnDxBY3olI6meuYr-LMbObhFmFbRcKF5mY
|
||||
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
``` showLineNumbers title="docker run litellm"
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
``` showLineNumbers title="pip install litellm"
|
||||
pip install litellm==1.78.0
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Key Highlights
|
||||
|
||||
- **MCP Gateway Enhancements** - Fine-grained tool control at team/key level, OpenAPI to MCP server conversion, and per-tool parameter allowlists
|
||||
- **GPT-5 Pro & GPT-Image-1-Mini** - Day 0 support for OpenAI's GPT-5 Pro (400K context) and gpt-image-1-mini image generation
|
||||
- **UI Performance Boost** - Replaces bloated key list calls with lean key aliases endpoint, Turbopack for faster development, and major UI refactors
|
||||
- **EnkryptAI Guardrails** - New guardrail integration for content moderation
|
||||
- **Tag-Based Budgets** - Support for setting budgets based on request tags
|
||||
- **Azure AD & SSO** - Enhanced Azure AD default credentials selection and EntraID app roles support
|
||||
|
||||
---
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
|
||||
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
|
||||
| OpenAI | `gpt-5-pro` | 400K | $15.00 | $120.00 | Responses API, reasoning, vision, function calling, prompt caching, web search |
|
||||
| OpenAI | `gpt-5-pro-2025-10-06` | 400K | $15.00 | $120.00 | Responses API, reasoning, vision, function calling, prompt caching, web search |
|
||||
| OpenAI | `gpt-image-1-mini` | - | $2.00/img | - | Image generation and editing |
|
||||
| OpenAI | `gpt-realtime-mini` | 128K | $0.60 | $2.40 | Realtime audio, function calling |
|
||||
| Azure AI | `azure_ai/Phi-4-mini-reasoning` | 131K | $0.08 | $0.32 | Function calling |
|
||||
| Azure AI | `azure_ai/Phi-4-reasoning` | 32K | $0.125 | $0.50 | Function calling, reasoning |
|
||||
| Azure AI | `azure_ai/MAI-DS-R1` | 128K | $1.35 | $5.40 | Reasoning, function calling |
|
||||
| Bedrock | `au.anthropic.claude-sonnet-4-5-20250929-v1:0` | 200K | $3.30 | $16.50 | Chat, reasoning, vision, function calling, prompt caching |
|
||||
| Bedrock | `global.anthropic.claude-sonnet-4-5-20250929-v1:0` | 200K | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
|
||||
| Bedrock | `global.anthropic.claude-sonnet-4-20250514-v1:0` | 1M | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
|
||||
| Bedrock | `cohere.embed-v4:0` | 128K | $0.12 | - | Embeddings, image input support |
|
||||
| OCI | `oci/cohere.command-latest` | 128K | $1.56 | $1.56 | Function calling |
|
||||
| OCI | `oci/cohere.command-a-03-2025` | 256K | $1.56 | $1.56 | Function calling |
|
||||
| OCI | `oci/cohere.command-plus-latest` | 128K | $1.56 | $1.56 | Function calling |
|
||||
| Together AI | `together_ai/moonshotai/Kimi-K2-Instruct-0905` | 262K | $1.00 | $3.00 | Function calling |
|
||||
| Together AI | `together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct` | 262K | $0.15 | $1.50 | Function calling |
|
||||
| Together AI | `together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking` | 262K | $0.15 | $1.50 | Function calling |
|
||||
| Vertex AI | MedGemma models | Varies | Varies | Varies | Medical-focused Gemma models on custom endpoints |
|
||||
| Watson X | 27 new foundation models | Varies | Varies | Varies | Granite, Llama, Mistral families |
|
||||
|
||||
#### Features
|
||||
|
||||
- **[OpenAI](../../docs/providers/openai)**
|
||||
- Add GPT-5 Pro model configuration and documentation - [PR #15258](https://github.com/BerriAI/litellm/pull/15258)
|
||||
- Add stop parameter to non-supported params for GPT-5 - [PR #15244](https://github.com/BerriAI/litellm/pull/15244)
|
||||
- Day 0 Support, Add gpt-image-1-mini - [PR #15259](https://github.com/BerriAI/litellm/pull/15259)
|
||||
- Add gpt-realtime-mini support - [PR #15283](https://github.com/BerriAI/litellm/pull/15283)
|
||||
- Add gpt-5-pro-2025-10-06 to model costs - [PR #15344](https://github.com/BerriAI/litellm/pull/15344)
|
||||
- Minimal fix: gpt5 models should not go on cooldown when called with temperature!=1 - [PR #15330](https://github.com/BerriAI/litellm/pull/15330)
|
||||
|
||||
- **[Snowflake Cortex](../../docs/providers/snowflake)**
|
||||
- Add function calling support for Snowflake Cortex REST API - [PR #15221](https://github.com/BerriAI/litellm/pull/15221)
|
||||
|
||||
- **[Gemini](../../docs/providers/gemini)**
|
||||
- Fix header forwarding for Gemini/Vertex AI providers in proxy mode - [PR #15231](https://github.com/BerriAI/litellm/pull/15231)
|
||||
|
||||
- **[Azure](../../docs/providers/azure)**
|
||||
- Removed stop param from unsupported azure models - [PR #15229](https://github.com/BerriAI/litellm/pull/15229)
|
||||
- Fix(azure/responses): remove invalid status param from azure call - [PR #15253](https://github.com/BerriAI/litellm/pull/15253)
|
||||
- Add new Azure AI models with pricing details - [PR #15387](https://github.com/BerriAI/litellm/pull/15387)
|
||||
- AzureAD Default credentials - select credential type based on environment - [PR #14470](https://github.com/BerriAI/litellm/pull/14470)
|
||||
|
||||
- **[Bedrock](../../docs/providers/bedrock)**
|
||||
- Add Global Cross-Region Inference - [PR #15210](https://github.com/BerriAI/litellm/pull/15210)
|
||||
- Add Cohere Embed v4 support for AWS Bedrock - [PR #15298](https://github.com/BerriAI/litellm/pull/15298)
|
||||
- Fix(bedrock): include cacheWriteInputTokens in prompt_tokens calculation - [PR #15292](https://github.com/BerriAI/litellm/pull/15292)
|
||||
- Add Bedrock AU Cross-Region Inference for Claude Sonnet 4.5 - [PR #15402](https://github.com/BerriAI/litellm/pull/15402)
|
||||
- Converse → /v1/messages streaming doesn't handle parallel tool calls with Claude models - [PR #15315](https://github.com/BerriAI/litellm/pull/15315)
|
||||
|
||||
- **[Vertex AI](../../docs/providers/vertex)**
|
||||
- Implement Context Caching for Vertex AI provider - [PR #15226](https://github.com/BerriAI/litellm/pull/15226)
|
||||
- Support for Vertex AI Gemma Models on Custom Endpoints - [PR #15397](https://github.com/BerriAI/litellm/pull/15397)
|
||||
- VertexAI - gemma model family support (custom endpoints) - [PR #15419](https://github.com/BerriAI/litellm/pull/15419)
|
||||
- VertexAI Gemma model family streaming support + Added MedGemma - [PR #15427](https://github.com/BerriAI/litellm/pull/15427)
|
||||
|
||||
- **[OCI](../../docs/providers/oci)**
|
||||
- Add OCI Cohere support with tool calling and streaming capabilities - [PR #15365](https://github.com/BerriAI/litellm/pull/15365)
|
||||
|
||||
- **[Watson X](../../docs/providers/watsonx)**
|
||||
- Add Watson X foundation model definitions to model_prices_and_context_window.json - [PR #15219](https://github.com/BerriAI/litellm/pull/15219)
|
||||
- Watsonx - Apply correct prompt templates for openai/gpt-oss model family - [PR #15341](https://github.com/BerriAI/litellm/pull/15341)
|
||||
|
||||
- **[OpenRouter](../../docs/providers/openrouter)**
|
||||
- Fix - (openrouter): move cache_control to content blocks for claude/gemini - [PR #15345](https://github.com/BerriAI/litellm/pull/15345)
|
||||
- Fix - OpenRouter cache_control to only apply to last content block - [PR #15395](https://github.com/BerriAI/litellm/pull/15395)
|
||||
|
||||
- **[Together AI](../../docs/providers/togetherai)**
|
||||
- Add new together models - [PR #15383](https://github.com/BerriAI/litellm/pull/15383)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **General**
|
||||
- Bug fix: gpt-5-chat-latest has incorrect max_input_tokens value - [PR #15116](https://github.com/BerriAI/litellm/pull/15116)
|
||||
- Fix reasoning response ID - [PR #15265](https://github.com/BerriAI/litellm/pull/15265)
|
||||
- Fix issue with parsing assistant messages - [PR #15320](https://github.com/BerriAI/litellm/pull/15320)
|
||||
- Fix litellm_param based costing - [PR #15336](https://github.com/BerriAI/litellm/pull/15336)
|
||||
- Fix lint errors - [PR #15406](https://github.com/BerriAI/litellm/pull/15406)
|
||||
|
||||
---
|
||||
|
||||
## LLM API Endpoints
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Responses API](../../docs/response_api)**
|
||||
- Added streaming support for response api streaming image generation - [PR #15269](https://github.com/BerriAI/litellm/pull/15269)
|
||||
- Add native Responses API support for litellm_proxy provider - [PR #15347](https://github.com/BerriAI/litellm/pull/15347)
|
||||
- Temporarily relax ResponsesAPIResponse parsing to support custom backends (e.g., vLLM) - [PR #15362](https://github.com/BerriAI/litellm/pull/15362)
|
||||
|
||||
- **[Files API](../../docs/files_api)**
|
||||
- Feat(files): add @client decorator to file operations - [PR #15339](https://github.com/BerriAI/litellm/pull/15339)
|
||||
|
||||
- **[/generateContent](../../docs/providers/gemini)**
|
||||
- Fix gemini cli by actually streaming the response - [PR #15264](https://github.com/BerriAI/litellm/pull/15264)
|
||||
|
||||
- **[Azure Passthrough](../../docs/pass_through/azure)**
|
||||
- Azure - passthrough support with router models - [PR #15240](https://github.com/BerriAI/litellm/pull/15240)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **General**
|
||||
- Fix x-litellm-cache-key header not being returned on cache hit - [PR #15348](https://github.com/BerriAI/litellm/pull/15348)
|
||||
|
||||
---
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
#### Features
|
||||
|
||||
- **Proxy CLI Auth**
|
||||
- Proxy CLI - dont store existing key in the URL, store it in the state param - [PR #15290](https://github.com/BerriAI/litellm/pull/15290)
|
||||
|
||||
- **Models + Endpoints**
|
||||
- Make PATCH `/model/{model_id}/update` handle `team_id` consistently with POST `/model/new` - [PR #15297](https://github.com/BerriAI/litellm/pull/15297)
|
||||
- Feature: adds Infinity as a provider in the UI - [PR #15285](https://github.com/BerriAI/litellm/pull/15285)
|
||||
- Fix: model + endpoints page crash when config file contains router_settings.model_group_alias - [PR #15308](https://github.com/BerriAI/litellm/pull/15308)
|
||||
- Models & Endpoints Initial Refactor - [PR #15435](https://github.com/BerriAI/litellm/pull/15435)
|
||||
- Litellm UI API Reference page updates - [PR #15438](https://github.com/BerriAI/litellm/pull/15438)
|
||||
|
||||
- **Teams**
|
||||
- Teams page: new column "Your Role" on the teams table - [PR #15384](https://github.com/BerriAI/litellm/pull/15384)
|
||||
- LiteLLM Dashboard Teams UI refactor - [PR #15418](https://github.com/BerriAI/litellm/pull/15418)
|
||||
|
||||
- **UI Infrastructure**
|
||||
- Added prettier to autoformat frontend - [PR #15215](https://github.com/BerriAI/litellm/pull/15215)
|
||||
- Adds turbopack to the npm run dev command in UI to build faster during development - [PR #15250](https://github.com/BerriAI/litellm/pull/15250)
|
||||
- (perf) fix: Replaces bloated key list calls with lean key aliases endpoint - [PR #15252](https://github.com/BerriAI/litellm/pull/15252)
|
||||
- Potentially fixes a UI spasm issue with an expired cookie - [PR #15309](https://github.com/BerriAI/litellm/pull/15309)
|
||||
- LiteLLM UI Refactor Infrastructure - [PR #15236](https://github.com/BerriAI/litellm/pull/15236)
|
||||
- Enforces removal of unused imports from UI - [PR #15416](https://github.com/BerriAI/litellm/pull/15416)
|
||||
- Fix: usage page >> Model Activity >> spend per day graph: y-axis clipping on large spend values - [PR #15389](https://github.com/BerriAI/litellm/pull/15389)
|
||||
- Updates guardrail provider logos - [PR #15421](https://github.com/BerriAI/litellm/pull/15421)
|
||||
|
||||
- **Admin Settings**
|
||||
- Fix: Router settings do not update despite success message - [PR #15249](https://github.com/BerriAI/litellm/pull/15249)
|
||||
- Fix: Prevents DB from accidentally overriding config file values if they are empty in DB - [PR #15340](https://github.com/BerriAI/litellm/pull/15340)
|
||||
|
||||
- **SSO**
|
||||
- SSO - support EntraID app roles - [PR #15351](https://github.com/BerriAI/litellm/pull/15351)
|
||||
|
||||
---
|
||||
|
||||
## Logging / Guardrail / Prompt Management Integrations
|
||||
|
||||
#### Features
|
||||
|
||||
- **[PostHog](../../docs/observability/posthog)**
|
||||
- Feat: posthog per request api key - [PR #15379](https://github.com/BerriAI/litellm/pull/15379)
|
||||
|
||||
#### Guardrails
|
||||
|
||||
- **[EnkryptAI](../../docs/proxy/guardrails)**
|
||||
- Add EnkryptAI Guardrails on LiteLLM - [PR #15390](https://github.com/BerriAI/litellm/pull/15390)
|
||||
|
||||
---
|
||||
|
||||
## Spend Tracking, Budgets and Rate Limiting
|
||||
|
||||
- **Tag Management**
|
||||
- Tag Management - Add support for setting tag based budgets - [PR #15433](https://github.com/BerriAI/litellm/pull/15433)
|
||||
|
||||
- **Dynamic Rate Limiter v3**
|
||||
- QA/Fixes - Dynamic Rate Limiter v3 - final QA - [PR #15311](https://github.com/BerriAI/litellm/pull/15311)
|
||||
- Fix dynamic Rate limiter v3 - inserting litellm_model_saturation - [PR #15394](https://github.com/BerriAI/litellm/pull/15394)
|
||||
|
||||
- **Shared Health Check**
|
||||
- Implement Shared Health Check State Across Pods - [PR #15380](https://github.com/BerriAI/litellm/pull/15380)
|
||||
|
||||
---
|
||||
|
||||
## MCP Gateway
|
||||
|
||||
- **Tool Control**
|
||||
- MCP Gateway - UI - Select allowed tools for Key, Teams - [PR #15241](https://github.com/BerriAI/litellm/pull/15241)
|
||||
- MCP Gateway - Backend - Allow storing allowed tools by team/key - [PR #15243](https://github.com/BerriAI/litellm/pull/15243)
|
||||
- MCP Gateway - Fine-grained Database Object Storage Control - [PR #15255](https://github.com/BerriAI/litellm/pull/15255)
|
||||
- MCP Gateway - Litellm mcp fixes team control - [PR #15304](https://github.com/BerriAI/litellm/pull/15304)
|
||||
- MCP Gateway - QA/Fixes - Ensure Team/Key level enforcement works for MCPs - [PR #15305](https://github.com/BerriAI/litellm/pull/15305)
|
||||
- Feature: Include server_name in /v1/mcp/server/health endpoint response - [PR #15431](https://github.com/BerriAI/litellm/pull/15431)
|
||||
|
||||
- **OpenAPI Integration**
|
||||
- MCP - support converting OpenAPI specs to MCP servers - [PR #15343](https://github.com/BerriAI/litellm/pull/15343)
|
||||
- MCP - specify allowed params per tool - [PR #15346](https://github.com/BerriAI/litellm/pull/15346)
|
||||
|
||||
- **Configuration**
|
||||
- MCP - support setting CA_BUNDLE_PATH - [PR #15253](https://github.com/BerriAI/litellm/pull/15253)
|
||||
- Fix: Ensure MCP client stays open during tool call - [PR #15391](https://github.com/BerriAI/litellm/pull/15391)
|
||||
- Remove hardcoded "public" schema in migration.sql - [PR #15363](https://github.com/BerriAI/litellm/pull/15363)
|
||||
|
||||
---
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
- **Router Optimizations**
|
||||
- Fix - Router: add model_name index for O(1) deployment lookups - [PR #15113](https://github.com/BerriAI/litellm/pull/15113)
|
||||
- Refactor Utils: extract inner function from client - [PR #15234](https://github.com/BerriAI/litellm/pull/15234)
|
||||
- Fix Networking: remove limitations - [PR #15302](https://github.com/BerriAI/litellm/pull/15302)
|
||||
|
||||
- **Session Management**
|
||||
- Fix - Sessions not being shared - [PR #15388](https://github.com/BerriAI/litellm/pull/15388)
|
||||
- Fix: remove panic from hot path - [PR #15396](https://github.com/BerriAI/litellm/pull/15396)
|
||||
- Fix - shared session parsing and usage issue - [PR #15440](https://github.com/BerriAI/litellm/pull/15440)
|
||||
- Fix: handle closed aiohttp sessions - [PR #15442](https://github.com/BerriAI/litellm/pull/15442)
|
||||
- Fix: prevent session leaks when recreating aiohttp sessions - [PR #15443](https://github.com/BerriAI/litellm/pull/15443)
|
||||
|
||||
- **SSL/TLS Performance**
|
||||
- Perf: optimize SSL/TLS handshake performance with prioritized cipher - [PR #15398](https://github.com/BerriAI/litellm/pull/15398)
|
||||
|
||||
- **Dependencies**
|
||||
- Upgrades tenacity version to 8.5.0 - [PR #15303](https://github.com/BerriAI/litellm/pull/15303)
|
||||
|
||||
- **Data Masking**
|
||||
- Fix - SensitiveDataMasker converts lists to string - [PR #15420](https://github.com/BerriAI/litellm/pull/15420)
|
||||
|
||||
---
|
||||
|
||||
|
||||
## General AI Gateway Improvements
|
||||
|
||||
#### Security
|
||||
|
||||
- **General**
|
||||
- Fix: redact AWS credentials when redact_user_api_key_info enabled - [PR #15321](https://github.com/BerriAI/litellm/pull/15321)
|
||||
|
||||
---
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
- **Provider Documentation**
|
||||
- Update doc: perf update - [PR #15211](https://github.com/BerriAI/litellm/pull/15211)
|
||||
- Add W&B Inference documentation - [PR #15278](https://github.com/BerriAI/litellm/pull/15278)
|
||||
|
||||
- **Deployment**
|
||||
- Deletion of docker-compose buggy comment that cause `config.yaml` based startup fail - [PR #15425](https://github.com/BerriAI/litellm/pull/15425)
|
||||
|
||||
---
|
||||
|
||||
## New Contributors
|
||||
|
||||
* @Gal-bloch made their first contribution in [PR #15219](https://github.com/BerriAI/litellm/pull/15219)
|
||||
* @lcfyi made their first contribution in [PR #15315](https://github.com/BerriAI/litellm/pull/15315)
|
||||
* @ashengstd made their first contribution in [PR #15362](https://github.com/BerriAI/litellm/pull/15362)
|
||||
* @vkolehmainen made their first contribution in [PR #15363](https://github.com/BerriAI/litellm/pull/15363)
|
||||
* @jlan-nl made their first contribution in [PR #15330](https://github.com/BerriAI/litellm/pull/15330)
|
||||
* @BCook98 made their first contribution in [PR #15402](https://github.com/BerriAI/litellm/pull/15402)
|
||||
* @PabloGmz96 made their first contribution in [PR #15425](https://github.com/BerriAI/litellm/pull/15425)
|
||||
|
||||
---
|
||||
|
||||
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.7.rc.1...v1.78.0.rc.1)**
|
||||
|
||||
@@ -189,12 +189,13 @@ const sidebars = {
|
||||
type: "category",
|
||||
label: "Budgets + Rate Limits",
|
||||
items: [
|
||||
"proxy/users",
|
||||
"proxy/team_budgets",
|
||||
"proxy/tag_budgets",
|
||||
"proxy/customers",
|
||||
"proxy/dynamic_rate_limit",
|
||||
"proxy/rate_limit_tiers",
|
||||
"proxy/team_budgets",
|
||||
"proxy/temporary_budget_increase",
|
||||
"proxy/users"
|
||||
],
|
||||
},
|
||||
"proxy/caching",
|
||||
@@ -423,6 +424,7 @@ const sidebars = {
|
||||
items: [
|
||||
"providers/vertex",
|
||||
"providers/vertex_partner",
|
||||
"providers/vertex_self_deployed",
|
||||
"providers/vertex_image",
|
||||
"providers/vertex_batch",
|
||||
]
|
||||
|
||||
Binary file not shown.
Binary file not shown.
+18
@@ -0,0 +1,18 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "LiteLLM_TagTable" (
|
||||
"tag_name" TEXT NOT NULL,
|
||||
"description" TEXT,
|
||||
"models" TEXT[],
|
||||
"model_info" JSONB,
|
||||
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
|
||||
"budget_id" TEXT,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "LiteLLM_TagTable_pkey" PRIMARY KEY ("tag_name")
|
||||
);
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "LiteLLM_TagTable" ADD CONSTRAINT "LiteLLM_TagTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
@@ -25,6 +25,7 @@ model LiteLLM_BudgetTable {
|
||||
organization LiteLLM_OrganizationTable[] // multiple orgs can have the same budget
|
||||
keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
|
||||
end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
|
||||
tags LiteLLM_TagTable[] // multiple tags can have the same budget
|
||||
team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team
|
||||
organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
|
||||
}
|
||||
@@ -245,6 +246,20 @@ model LiteLLM_EndUserTable {
|
||||
blocked Boolean @default(false)
|
||||
}
|
||||
|
||||
// Track tags with budgets and spend
|
||||
model LiteLLM_TagTable {
|
||||
tag_name String @id
|
||||
description String?
|
||||
models String[]
|
||||
model_info Json? // maps model_id to model_name
|
||||
spend Float @default(0.0)
|
||||
budget_id String?
|
||||
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
|
||||
created_at DateTime @default(now()) @map("created_at")
|
||||
created_by String?
|
||||
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
|
||||
}
|
||||
|
||||
// store proxy config.yaml
|
||||
model LiteLLM_Config {
|
||||
param_name String @id
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.2.25"
|
||||
version = "0.2.26"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.2.25"
|
||||
version = "0.2.26"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
||||
@@ -18,13 +18,15 @@ from typing import (
|
||||
cast,
|
||||
)
|
||||
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
|
||||
from litellm import ModelResponse
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
CompletionTransformationBridge,
|
||||
)
|
||||
from litellm.types.llms.openai import Reasoning
|
||||
from litellm.types.llms.openai import ChatCompletionToolParamFunctionChunk, Reasoning
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.responses import ResponseInputImageParam
|
||||
@@ -50,6 +52,47 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def _handle_raw_dict_response_item(
|
||||
self, item: Dict[str, Any], index: int
|
||||
) -> Tuple[Optional[Any], int]:
|
||||
"""
|
||||
Handle raw dict response items from Responses API (e.g., GPT-5 Codex format).
|
||||
|
||||
Args:
|
||||
item: Raw dict response item with 'type' field
|
||||
index: Current choice index
|
||||
|
||||
Returns:
|
||||
Tuple of (Choice object or None, updated index)
|
||||
"""
|
||||
from litellm.types.utils import Choices, Message
|
||||
|
||||
item_type = item.get("type")
|
||||
|
||||
# Ignore reasoning items for now
|
||||
if item_type == "reasoning":
|
||||
return None, index
|
||||
|
||||
# Handle message items with output_text content
|
||||
if item_type == "message":
|
||||
content_list = item.get("content", [])
|
||||
for content_item in content_list:
|
||||
if isinstance(content_item, dict):
|
||||
content_type = content_item.get("type")
|
||||
if content_type == "output_text":
|
||||
response_text = content_item.get("text", "")
|
||||
msg = Message(
|
||||
role=item.get("role", "assistant"),
|
||||
content=response_text if response_text else ""
|
||||
)
|
||||
choice = Choices(
|
||||
message=msg, finish_reason="stop", index=index
|
||||
)
|
||||
return choice, index + 1
|
||||
|
||||
# Unknown or unsupported type
|
||||
return None, index
|
||||
|
||||
def convert_chat_completion_messages_to_responses_api(
|
||||
self, messages: List["AllMessageValues"]
|
||||
) -> Tuple[List[Any], Optional[str]]:
|
||||
@@ -201,6 +244,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
if value is not None:
|
||||
if key == "instructions" and instructions:
|
||||
request_data["instructions"] = instructions
|
||||
elif key == "stream_options" and isinstance(value, dict):
|
||||
request_data["stream_options"] = value.get("include_obfuscation")
|
||||
elif key == "user": # string can't be longer than 64 characters
|
||||
if isinstance(value, str) and len(value) <= 64:
|
||||
request_data["user"] = value
|
||||
else:
|
||||
request_data[key] = value
|
||||
|
||||
@@ -221,7 +269,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> "ModelResponse":
|
||||
"""Transform Responses API response to chat completion response"""
|
||||
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
@@ -240,19 +287,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
|
||||
choices: List[Choices] = []
|
||||
index = 0
|
||||
|
||||
reasoning_content: Optional[str] = None
|
||||
|
||||
for item in raw_response.output:
|
||||
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
pass # ignore for now.
|
||||
|
||||
for content in item.summary:
|
||||
response_text = getattr(content, "text", "")
|
||||
reasoning_content = response_text if response_text else ""
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for content in item.content:
|
||||
response_text = getattr(content, "text", "")
|
||||
msg = Message(
|
||||
role=item.role, content=response_text if response_text else ""
|
||||
role=item.role,
|
||||
content=response_text if response_text else "",
|
||||
reasoning_content=reasoning_content,
|
||||
)
|
||||
|
||||
choices.append(
|
||||
Choices(message=msg, finish_reason="stop", index=index)
|
||||
Choices(
|
||||
message=msg,
|
||||
finish_reason="stop",
|
||||
index=index,
|
||||
)
|
||||
)
|
||||
|
||||
reasoning_content = None # flush reasoning content
|
||||
index += 1
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
msg = Message(
|
||||
@@ -267,12 +330,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
"type": "function",
|
||||
}
|
||||
],
|
||||
reasoning_content=reasoning_content,
|
||||
)
|
||||
|
||||
choices.append(
|
||||
Choices(message=msg, finish_reason="tool_calls", index=index)
|
||||
)
|
||||
reasoning_content = None # flush reasoning content
|
||||
index += 1
|
||||
elif isinstance(item, dict):
|
||||
# Handle raw dict responses (e.g., from GPT-5 Codex)
|
||||
choice, index = self._handle_raw_dict_response_item(item=item, index=index)
|
||||
if choice is not None:
|
||||
choices.append(choice)
|
||||
else:
|
||||
pass # don't fail request if item in list is not supported
|
||||
|
||||
@@ -447,9 +517,25 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
||||
self, tools: List[Dict[str, Any]]
|
||||
) -> List["ALL_RESPONSES_API_TOOL_PARAMS"]:
|
||||
"""Convert chat completion tools to responses API tools format"""
|
||||
responses_tools = []
|
||||
responses_tools: List["ALL_RESPONSES_API_TOOL_PARAMS"] = []
|
||||
for tool in tools:
|
||||
responses_tools.append(tool)
|
||||
# convert function tool from chat completion to responses API format
|
||||
if tool.get("type") == "function":
|
||||
function_tool = cast(
|
||||
ChatCompletionToolParamFunctionChunk, tool.get("function")
|
||||
)
|
||||
responses_tools.append(
|
||||
FunctionToolParam(
|
||||
name=function_tool["name"],
|
||||
parameters=function_tool.get("parameters"),
|
||||
strict=function_tool.get("strict"),
|
||||
type="function",
|
||||
description=function_tool.get("description"),
|
||||
)
|
||||
)
|
||||
else:
|
||||
responses_tools.append(tool) # type: ignore
|
||||
|
||||
return cast(List["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
|
||||
|
||||
def _map_reasoning_effort(self, reasoning_effort: str) -> Optional[Reasoning]:
|
||||
|
||||
@@ -11,11 +11,10 @@ For batching specific details see CustomBatchLogger class
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from litellm._uuid import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
_get_httpx_client,
|
||||
@@ -74,6 +73,8 @@ class PostHogLogger(CustomBatchLogger):
|
||||
)
|
||||
|
||||
api_key, api_url = self._get_credentials_for_request(kwargs)
|
||||
if api_key is None or api_url is None:
|
||||
raise Exception("PostHog credentials not found in kwargs")
|
||||
event_payload = self.create_posthog_event_payload(kwargs)
|
||||
|
||||
headers = {
|
||||
@@ -275,7 +276,7 @@ class PostHogLogger(CustomBatchLogger):
|
||||
|
||||
return self._safe_uuid()
|
||||
|
||||
def _get_credentials_for_request(self, kwargs: Dict[str, Any]) -> tuple[str, str]:
|
||||
def _get_credentials_for_request(self, kwargs: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Get PostHog credentials for this request.
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ class SensitiveDataMasker:
|
||||
masked_data[k] = self._mask_value(str_value)
|
||||
else:
|
||||
masked_data[k] = (
|
||||
v if isinstance(v, (int, float, bool, str)) else str(v)
|
||||
v if isinstance(v, (int, float, bool, str, list)) else str(v)
|
||||
)
|
||||
except Exception:
|
||||
masked_data[k] = "<unable to serialize>"
|
||||
@@ -89,12 +89,14 @@ masker = SensitiveDataMasker()
|
||||
data = {
|
||||
"api_key": "sk-1234567890abcdef",
|
||||
"redis_password": "very_secret_pass",
|
||||
"port": 6379
|
||||
"port": 6379,
|
||||
"tags": ["East US 2", "production", "test"]
|
||||
}
|
||||
masked = masker.mask_dict(data)
|
||||
# Result: {
|
||||
# "api_key": "sk-1****cdef",
|
||||
# "redis_password": "very****pass",
|
||||
# "port": 6379
|
||||
# "port": 6379,
|
||||
# "tags": ["East US 2", "production", "test"]
|
||||
# }
|
||||
"""
|
||||
|
||||
@@ -133,7 +133,6 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
||||
**kwargs,
|
||||
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
|
||||
"""Handle non-Anthropic models asynchronously using the adapter"""
|
||||
|
||||
completion_kwargs = (
|
||||
LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
|
||||
@@ -672,6 +672,10 @@ class BaseAzureLLM(BaseOpenAILLM):
|
||||
) -> dict:
|
||||
litellm_params = litellm_params or GenericLiteLLMParams()
|
||||
|
||||
# Check if api-key is already in headers; if so, use it
|
||||
if "api-key" in headers:
|
||||
return headers
|
||||
|
||||
api_key = (
|
||||
litellm_params.api_key
|
||||
or litellm.api_key
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
from openai.types.responses import ResponseReasoningItem
|
||||
@@ -41,12 +41,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
||||
|
||||
def _handle_reasoning_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Handle reasoning items specifically to filter out status=None using OpenAI's model.
|
||||
Handle reasoning items to filter out the status field.
|
||||
Issue: https://github.com/BerriAI/litellm/issues/13484
|
||||
OpenAI API does not accept ReasoningItem(status=None), so we need to:
|
||||
1. Check if the item is a reasoning type
|
||||
2. Create a ResponseReasoningItem object with the item data
|
||||
3. Convert it back to dict with exclude_none=True to filter None values
|
||||
|
||||
Azure OpenAI API does not accept 'status' field in reasoning input items.
|
||||
"""
|
||||
if item.get("type") == "reasoning":
|
||||
try:
|
||||
@@ -82,6 +80,32 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
||||
}
|
||||
return filtered_item
|
||||
return item
|
||||
|
||||
def _validate_input_param(
|
||||
self, input: Union[str, ResponseInputParam]
|
||||
) -> Union[str, ResponseInputParam]:
|
||||
"""
|
||||
Override parent method to also filter out 'status' field from message items.
|
||||
Azure OpenAI API does not accept 'status' field in input messages.
|
||||
"""
|
||||
from typing import cast
|
||||
|
||||
# First call parent's validation
|
||||
validated_input = super()._validate_input_param(input)
|
||||
|
||||
# Then filter out status from message items
|
||||
if isinstance(validated_input, list):
|
||||
filtered_input: List[Any] = []
|
||||
for item in validated_input:
|
||||
if isinstance(item, dict) and item.get("type") == "message":
|
||||
# Filter out status field from message items
|
||||
filtered_item = {k: v for k, v in item.items() if k != "status"}
|
||||
filtered_input.append(filtered_item)
|
||||
else:
|
||||
filtered_input.append(item)
|
||||
return cast(ResponseInputParam, filtered_input)
|
||||
|
||||
return validated_input
|
||||
|
||||
def transform_responses_api_request(
|
||||
self,
|
||||
|
||||
@@ -210,6 +210,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
|
||||
client=None,
|
||||
aembedding=None,
|
||||
max_retries: Optional[int] = None,
|
||||
shared_session=None,
|
||||
) -> EmbeddingResponse:
|
||||
"""
|
||||
- Separate image url from text
|
||||
@@ -275,6 +276,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
|
||||
else None
|
||||
),
|
||||
aembedding=aembedding,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
text_embedding_responses = response.data
|
||||
|
||||
@@ -440,7 +440,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
||||
"""
|
||||
Abbreviations of regions AWS Bedrock supports for cross region inference
|
||||
"""
|
||||
return ["global", "us", "eu", "apac", "jp"]
|
||||
return ["global", "us", "eu", "apac", "jp", "au"]
|
||||
|
||||
@staticmethod
|
||||
def get_bedrock_route(
|
||||
|
||||
@@ -152,6 +152,16 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
||||
|
||||
# If we don't have a client or it's not a ClientSession, create one
|
||||
if not isinstance(self.client, ClientSession):
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
# Don't return yet - check if the newly created session is valid
|
||||
|
||||
# Check if the session itself is closed
|
||||
if self.client.closed:
|
||||
verbose_logger.debug("Session is closed, creating new session")
|
||||
# Create a new session
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
@@ -169,14 +179,17 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
||||
or session_loop != current_loop
|
||||
or session_loop.is_closed()
|
||||
):
|
||||
# Clean up the old session
|
||||
# Close old session to prevent leaks
|
||||
old_session = self.client
|
||||
try:
|
||||
# Note: not awaiting close() here as it might be from a different loop
|
||||
# The session will be garbage collected
|
||||
pass
|
||||
if not old_session.closed:
|
||||
try:
|
||||
asyncio.create_task(old_session.close())
|
||||
except RuntimeError:
|
||||
# Different event loop - can't schedule task, rely on GC
|
||||
verbose_logger.debug("Old session from different loop, relying on GC")
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error closing old session: {e}")
|
||||
pass
|
||||
|
||||
# Create a new session in the current event loop
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
@@ -193,13 +206,58 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
||||
|
||||
return self.client
|
||||
|
||||
async def _make_aiohttp_request(
|
||||
self,
|
||||
client_session: ClientSession,
|
||||
request: httpx.Request,
|
||||
timeout: dict,
|
||||
proxy: Optional[str],
|
||||
sni_hostname: Optional[str],
|
||||
) -> ClientResponse:
|
||||
"""
|
||||
Helper function to make an aiohttp request with the given parameters.
|
||||
|
||||
Args:
|
||||
client_session: The aiohttp ClientSession to use
|
||||
request: The httpx Request to send
|
||||
timeout: Timeout settings dict with 'connect', 'read', 'pool' keys
|
||||
proxy: Optional proxy URL
|
||||
sni_hostname: Optional SNI hostname for SSL
|
||||
|
||||
Returns:
|
||||
ClientResponse from aiohttp
|
||||
"""
|
||||
from aiohttp import ClientTimeout
|
||||
from yarl import URL as YarlURL
|
||||
|
||||
try:
|
||||
data = request.content
|
||||
except httpx.RequestNotRead:
|
||||
data = request.stream # type: ignore
|
||||
request.headers.pop("transfer-encoding", None) # handled by aiohttp
|
||||
|
||||
response = await client_session.request(
|
||||
method=request.method,
|
||||
url=YarlURL(str(request.url), encoded=True),
|
||||
headers=request.headers,
|
||||
data=data,
|
||||
allow_redirects=False,
|
||||
auto_decompress=False,
|
||||
timeout=ClientTimeout(
|
||||
sock_connect=timeout.get("connect"),
|
||||
sock_read=timeout.get("read"),
|
||||
connect=timeout.get("pool"),
|
||||
),
|
||||
proxy=proxy,
|
||||
server_hostname=sni_hostname,
|
||||
).__aenter__()
|
||||
|
||||
return response
|
||||
|
||||
async def handle_async_request(
|
||||
self,
|
||||
request: httpx.Request,
|
||||
) -> httpx.Response:
|
||||
from aiohttp import ClientTimeout
|
||||
from yarl import URL as YarlURL
|
||||
|
||||
timeout = request.extensions.get("timeout", {})
|
||||
sni_hostname = request.extensions.get("sni_hostname")
|
||||
|
||||
@@ -209,28 +267,38 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
||||
# Resolve proxy settings from environment variables
|
||||
proxy = await self._get_proxy_settings(request)
|
||||
|
||||
with map_aiohttp_exceptions():
|
||||
try:
|
||||
data = request.content
|
||||
except httpx.RequestNotRead:
|
||||
data = request.stream # type: ignore
|
||||
request.headers.pop("transfer-encoding", None) # handled by aiohttp
|
||||
|
||||
response = await client_session.request(
|
||||
method=request.method,
|
||||
url=YarlURL(str(request.url), encoded=True),
|
||||
headers=request.headers,
|
||||
data=data,
|
||||
allow_redirects=False,
|
||||
auto_decompress=False,
|
||||
timeout=ClientTimeout(
|
||||
sock_connect=timeout.get("connect"),
|
||||
sock_read=timeout.get("read"),
|
||||
connect=timeout.get("pool"),
|
||||
),
|
||||
proxy=proxy,
|
||||
server_hostname=sni_hostname,
|
||||
).__aenter__()
|
||||
try:
|
||||
with map_aiohttp_exceptions():
|
||||
response = await self._make_aiohttp_request(
|
||||
client_session=client_session,
|
||||
request=request,
|
||||
timeout=timeout,
|
||||
proxy=proxy,
|
||||
sni_hostname=sni_hostname,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
# Handle the case where session was closed between our check and actual use
|
||||
if "Session is closed" in str(e):
|
||||
verbose_logger.debug(f"Session closed during request, retrying with new session: {e}")
|
||||
# Force creation of a new session
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
client_session = self.client
|
||||
|
||||
# Retry the request with the new session
|
||||
with map_aiohttp_exceptions():
|
||||
response = await self._make_aiohttp_request(
|
||||
client_session=client_session,
|
||||
request=request,
|
||||
timeout=timeout,
|
||||
proxy=proxy,
|
||||
sni_hostname=sni_hostname,
|
||||
)
|
||||
else:
|
||||
# Re-raise if it's a different RuntimeError
|
||||
raise
|
||||
|
||||
return httpx.Response(
|
||||
status_code=response.status,
|
||||
|
||||
@@ -13,13 +13,13 @@ from typing import (
|
||||
cast,
|
||||
)
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
import httpx # type: ignore
|
||||
|
||||
import litellm
|
||||
import litellm.litellm_core_utils
|
||||
import litellm.types
|
||||
import litellm.types.utils
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
@@ -239,7 +239,7 @@ class BaseLLMHTTPHandler:
|
||||
json_mode: bool = False,
|
||||
signed_json_body: Optional[bytes] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
):
|
||||
):
|
||||
if client is None:
|
||||
verbose_logger.debug(
|
||||
f"Creating HTTP client with shared_session: {id(shared_session) if shared_session else None}"
|
||||
@@ -1533,6 +1533,7 @@ class BaseLLMHTTPHandler:
|
||||
data=data,
|
||||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
response = sync_httpx_client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
|
||||
@@ -19,6 +19,13 @@ from litellm.llms.custom_httpx.http_handler import (
|
||||
)
|
||||
from litellm.llms.oci.common_utils import OCIError
|
||||
from litellm.types.llms.oci import (
|
||||
CohereChatRequest,
|
||||
CohereMessage,
|
||||
CohereChatResult,
|
||||
CohereParameterDefinition,
|
||||
CohereStreamChunk,
|
||||
CohereTool,
|
||||
CohereToolCall,
|
||||
OCIChatRequestPayload,
|
||||
OCICompletionPayload,
|
||||
OCICompletionResponse,
|
||||
@@ -37,13 +44,13 @@ from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
LlmProviders,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
CustomStreamWrapper,
|
||||
ModelResponse,
|
||||
Usage,
|
||||
)
|
||||
|
||||
@@ -170,13 +177,16 @@ class OCIChatConfig(BaseConfig):
|
||||
"web_search_options": False,
|
||||
}
|
||||
|
||||
# Cohere and Gemini use the same parameter mapping as GENERIC
|
||||
self.openai_to_oci_cohere_param_map = self.openai_to_oci_generic_param_map.copy()
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
supported_params = []
|
||||
vendor = get_vendor_from_model(model)
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
|
||||
open_ai_to_oci_param_map.pop("tool_choice")
|
||||
open_ai_to_oci_param_map.pop("max_retries")
|
||||
else:
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
|
||||
for key, value in open_ai_to_oci_param_map.items():
|
||||
@@ -195,9 +205,7 @@ class OCIChatConfig(BaseConfig):
|
||||
adapted_params = {}
|
||||
vendor = get_vendor_from_model(model)
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
|
||||
else:
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
|
||||
|
||||
@@ -416,21 +424,130 @@ class OCIChatConfig(BaseConfig):
|
||||
def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict:
|
||||
selected_params = {}
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
|
||||
# remove tool_choice from the map
|
||||
open_ai_to_oci_param_map.pop("tool_choice")
|
||||
# Add default values for Cohere API
|
||||
selected_params = {
|
||||
"maxTokens": 600,
|
||||
"temperature": 1,
|
||||
"topK": 0,
|
||||
"topP": 0.75,
|
||||
"frequencyPenalty": 0
|
||||
}
|
||||
else:
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
|
||||
|
||||
for value in open_ai_to_oci_param_map.values():
|
||||
if value in optional_params:
|
||||
selected_params[value] = optional_params[value]
|
||||
# Map OpenAI params to OCI params
|
||||
for openai_key, oci_key in open_ai_to_oci_param_map.items():
|
||||
if oci_key and openai_key in optional_params:
|
||||
selected_params[oci_key] = optional_params[openai_key] # type: ignore[index]
|
||||
|
||||
# Also check for already-mapped OCI params (for backward compatibility)
|
||||
for oci_value in open_ai_to_oci_param_map.values():
|
||||
if oci_value and oci_value in optional_params and oci_value not in selected_params:
|
||||
selected_params[oci_value] = optional_params[oci_value] # type: ignore[index]
|
||||
|
||||
if "tools" in selected_params:
|
||||
selected_params["tools"] = adapt_tool_definition_to_oci_standard(
|
||||
selected_params["tools"], vendor
|
||||
)
|
||||
if vendor == OCIVendors.COHERE:
|
||||
selected_params["tools"] = self.adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment]
|
||||
selected_params["tools"] # type: ignore[arg-type]
|
||||
)
|
||||
else:
|
||||
selected_params["tools"] = adapt_tool_definition_to_oci_standard( # type: ignore[assignment]
|
||||
selected_params["tools"], vendor # type: ignore[arg-type]
|
||||
)
|
||||
return selected_params
|
||||
|
||||
def adapt_messages_to_cohere_standard(self, messages: List[AllMessageValues]) -> List[CohereMessage]:
|
||||
"""Build chat history for Cohere models."""
|
||||
chat_history = []
|
||||
for msg in messages[:-1]: # All messages except the last one
|
||||
role = msg.get("role")
|
||||
content = msg.get("content")
|
||||
|
||||
if isinstance(content, list):
|
||||
# Extract text from content array
|
||||
text_content = ""
|
||||
for content_item in content:
|
||||
if isinstance(content_item, dict) and content_item.get("type") == "text":
|
||||
text_content += content_item.get("text", "")
|
||||
content = text_content
|
||||
|
||||
# Ensure content is a string
|
||||
if not isinstance(content, str):
|
||||
content = str(content) if content is not None else ""
|
||||
|
||||
# Handle tool calls
|
||||
tool_calls: Optional[List[CohereToolCall]] = None
|
||||
if role == "assistant" and "tool_calls" in msg and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item]
|
||||
tool_calls = []
|
||||
for tool_call in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item]
|
||||
# Parse arguments if they're a JSON string
|
||||
raw_arguments: Any = tool_call.get("function", {}).get("arguments", {})
|
||||
if isinstance(raw_arguments, str):
|
||||
try:
|
||||
arguments: Dict[str, Any] = json.loads(raw_arguments)
|
||||
except json.JSONDecodeError:
|
||||
arguments = {}
|
||||
else:
|
||||
arguments = raw_arguments
|
||||
|
||||
tool_calls.append(CohereToolCall(
|
||||
name=str(tool_call.get("function", {}).get("name", "")),
|
||||
parameters=arguments
|
||||
))
|
||||
|
||||
if role == "user":
|
||||
chat_history.append(CohereMessage(role="USER", message=content))
|
||||
elif role == "assistant":
|
||||
chat_history.append(CohereMessage(role="CHATBOT", message=content, toolCalls=tool_calls))
|
||||
elif role == "tool":
|
||||
# Tool messages need special handling
|
||||
chat_history.append(CohereMessage(
|
||||
role="TOOL",
|
||||
message=content,
|
||||
toolCalls=None # Tool messages don't have tool calls
|
||||
))
|
||||
|
||||
return chat_history
|
||||
|
||||
def adapt_tool_definitions_to_cohere_standard(self, tools: List[Dict[str, Any]]) -> List[CohereTool]:
|
||||
"""Adapt tool definitions to Cohere format."""
|
||||
cohere_tools = []
|
||||
for tool in tools:
|
||||
function_def = tool.get("function", {})
|
||||
parameters = function_def.get("parameters", {}).get("properties", {})
|
||||
required = function_def.get("parameters", {}).get("required", [])
|
||||
|
||||
parameter_definitions = {}
|
||||
for param_name, param_schema in parameters.items():
|
||||
parameter_definitions[param_name] = CohereParameterDefinition(
|
||||
description=param_schema.get("description", ""),
|
||||
type=param_schema.get("type", "string"),
|
||||
isRequired=param_name in required
|
||||
)
|
||||
|
||||
cohere_tools.append(CohereTool(
|
||||
name=function_def.get("name", ""),
|
||||
description=function_def.get("description", ""),
|
||||
parameterDefinitions=parameter_definitions
|
||||
))
|
||||
|
||||
return cohere_tools
|
||||
|
||||
def _extract_text_content(self, content: Any) -> str:
|
||||
"""Extract text content from message content."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
elif isinstance(content, list):
|
||||
text_content = ""
|
||||
for content_item in content:
|
||||
if isinstance(content_item, dict) and content_item.get("type") == "text":
|
||||
text_content += content_item.get("text", "")
|
||||
return text_content
|
||||
return str(content)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
@@ -445,28 +562,47 @@ class OCIChatConfig(BaseConfig):
|
||||
|
||||
vendor = get_vendor_from_model(model)
|
||||
|
||||
if vendor == OCIVendors.COHERE:
|
||||
oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND")
|
||||
if oci_serving_mode not in ["ON_DEMAND", "DEDICATED"]:
|
||||
raise Exception(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
"kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'"
|
||||
)
|
||||
|
||||
if oci_serving_mode == "DEDICATED":
|
||||
servingMode = OCIServingMode(
|
||||
servingType="DEDICATED",
|
||||
endpointId=model,
|
||||
)
|
||||
else:
|
||||
oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND")
|
||||
if oci_serving_mode not in ["ON_DEMAND", "DEDICATED"]:
|
||||
raise Exception(
|
||||
"kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'"
|
||||
)
|
||||
servingMode = OCIServingMode(
|
||||
servingType="ON_DEMAND",
|
||||
modelId=model,
|
||||
)
|
||||
|
||||
if oci_serving_mode == "DEDICATED":
|
||||
servingMode = OCIServingMode(
|
||||
servingType="DEDICATED",
|
||||
endpointId=model,
|
||||
)
|
||||
else:
|
||||
servingMode = OCIServingMode(
|
||||
servingType="ON_DEMAND",
|
||||
modelId=model,
|
||||
)
|
||||
# Build request based on vendor type
|
||||
if vendor == OCIVendors.COHERE:
|
||||
# For Cohere, we need to use the specific Cohere format
|
||||
# Extract the last user message as the main message
|
||||
user_messages = [msg for msg in messages if msg.get("role") == "user"]
|
||||
if not user_messages:
|
||||
raise Exception("No user message found for Cohere model")
|
||||
|
||||
|
||||
# Create Cohere-specific chat request
|
||||
chat_request = CohereChatRequest(
|
||||
apiFormat="COHERE",
|
||||
message=self._extract_text_content(user_messages[-1]["content"]),
|
||||
chatHistory=self.adapt_messages_to_cohere_standard(messages),
|
||||
**self._get_optional_params(OCIVendors.COHERE, optional_params)
|
||||
)
|
||||
|
||||
data = OCICompletionPayload(
|
||||
compartmentId=oci_compartment_id,
|
||||
servingMode=servingMode,
|
||||
chatRequest=chat_request
|
||||
)
|
||||
else:
|
||||
# Use generic format for other vendors
|
||||
data = OCICompletionPayload(
|
||||
compartmentId=oci_compartment_id,
|
||||
servingMode=servingMode,
|
||||
@@ -479,6 +615,111 @@ class OCIChatConfig(BaseConfig):
|
||||
|
||||
return data.model_dump(exclude_none=True)
|
||||
|
||||
def _handle_cohere_response(
|
||||
self,
|
||||
json_response: dict,
|
||||
model: str,
|
||||
model_response: ModelResponse
|
||||
) -> ModelResponse:
|
||||
"""Handle Cohere-specific response format."""
|
||||
cohere_response = CohereChatResult(**json_response)
|
||||
# Cohere response format (uses camelCase)
|
||||
model_id = model
|
||||
|
||||
# Set basic response info
|
||||
model_response.model = model_id
|
||||
model_response.created = int(datetime.datetime.now().timestamp())
|
||||
|
||||
# Extract the response text
|
||||
response_text = cohere_response.chatResponse.text
|
||||
oci_finish_reason = cohere_response.chatResponse.finishReason
|
||||
|
||||
# Map finish reason
|
||||
if oci_finish_reason == "COMPLETE":
|
||||
finish_reason = "stop"
|
||||
elif oci_finish_reason == "MAX_TOKENS":
|
||||
finish_reason = "length"
|
||||
else:
|
||||
finish_reason = "stop"
|
||||
|
||||
# Handle tool calls
|
||||
tool_calls: Optional[List[Dict[str, Any]]] = None
|
||||
if cohere_response.chatResponse.toolCalls:
|
||||
tool_calls = []
|
||||
for tool_call in cohere_response.chatResponse.toolCalls:
|
||||
tool_calls.append({
|
||||
"id": f"call_{len(tool_calls)}", # Generate a simple ID
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tool_call.name,
|
||||
"arguments": json.dumps(tool_call.parameters)
|
||||
}
|
||||
})
|
||||
|
||||
# Create choice
|
||||
from litellm.types.utils import Choices
|
||||
choice = Choices(
|
||||
index=0,
|
||||
message={
|
||||
"role": "assistant",
|
||||
"content": response_text,
|
||||
"tool_calls": tool_calls
|
||||
},
|
||||
finish_reason=finish_reason
|
||||
)
|
||||
model_response.choices = [choice]
|
||||
|
||||
# Extract usage info
|
||||
usage_info = cohere_response.chatResponse.usage
|
||||
from litellm.types.utils import Usage
|
||||
model_response.usage = Usage( # type: ignore[attr-defined]
|
||||
prompt_tokens=usage_info.promptTokens, # type: ignore[union-attr]
|
||||
completion_tokens=usage_info.completionTokens, # type: ignore[union-attr]
|
||||
total_tokens=usage_info.totalTokens # type: ignore[union-attr]
|
||||
)
|
||||
|
||||
return model_response
|
||||
|
||||
def _handle_generic_response(
|
||||
self,
|
||||
json: dict,
|
||||
model: str,
|
||||
model_response: ModelResponse,
|
||||
raw_response: httpx.Response
|
||||
) -> ModelResponse:
|
||||
"""Handle generic OCI response format."""
|
||||
try:
|
||||
completion_response = OCICompletionResponse(**json)
|
||||
except TypeError as e:
|
||||
raise OCIError(
|
||||
message=f"Response cannot be casted to OCICompletionResponse: {str(e)}",
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
iso_str = completion_response.chatResponse.timeCreated
|
||||
dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00"))
|
||||
model_response.created = int(dt.timestamp())
|
||||
|
||||
model_response.model = completion_response.modelId
|
||||
|
||||
message = model_response.choices[0].message # type: ignore
|
||||
response_message = completion_response.chatResponse.choices[0].message
|
||||
if response_message.content and response_message.content[0].type == "TEXT":
|
||||
message.content = response_message.content[0].text
|
||||
if response_message.toolCalls:
|
||||
message.tool_calls = adapt_tools_to_openai_standard(
|
||||
response_message.toolCalls
|
||||
)
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=completion_response.chatResponse.usage.promptTokens,
|
||||
completion_tokens=completion_response.chatResponse.usage.completionTokens,
|
||||
total_tokens=completion_response.chatResponse.usage.totalTokens,
|
||||
)
|
||||
model_response.usage = usage # type: ignore
|
||||
|
||||
return model_response
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
@@ -509,46 +750,13 @@ class OCIChatConfig(BaseConfig):
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
try:
|
||||
completion_response = OCICompletionResponse(**json)
|
||||
except TypeError as e:
|
||||
raise OCIError(
|
||||
message=f"Response cannot be casted to OCICompletionResponse: {str(e)}",
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
vendor = get_vendor_from_model(model)
|
||||
|
||||
# Handle response based on vendor type
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
model_response = self._handle_cohere_response(json, model, model_response)
|
||||
else:
|
||||
iso_str = completion_response.chatResponse.timeCreated
|
||||
dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00"))
|
||||
model_response.created = int(dt.timestamp())
|
||||
|
||||
model_response.model = completion_response.modelId
|
||||
|
||||
message = model_response.choices[0].message # type: ignore
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
response_message = completion_response.chatResponse.choices[0].message
|
||||
if response_message.content and response_message.content[0].type == "TEXT":
|
||||
message.content = response_message.content[0].text
|
||||
if response_message.toolCalls:
|
||||
message.tool_calls = adapt_tools_to_openai_standard(
|
||||
response_message.toolCalls
|
||||
)
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=completion_response.chatResponse.usage.promptTokens,
|
||||
completion_tokens=completion_response.chatResponse.usage.completionTokens,
|
||||
total_tokens=completion_response.chatResponse.usage.totalTokens,
|
||||
)
|
||||
model_response.usage = usage # type: ignore
|
||||
model_response = self._handle_generic_response(json, model, model_response, raw_response)
|
||||
|
||||
model_response._hidden_params["additional_headers"] = raw_response.headers
|
||||
|
||||
@@ -818,26 +1026,21 @@ def adapt_messages_to_generic_oci_standard(
|
||||
|
||||
def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors):
|
||||
new_tools = []
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
for tool in tools:
|
||||
if tool["type"] != "function":
|
||||
raise Exception("OCI only supports function tools")
|
||||
|
||||
tool_function = tool.get("function")
|
||||
if not isinstance(tool_function, dict):
|
||||
raise Exception("Prop `function` is not a dictionary")
|
||||
|
||||
new_tool = OCIToolDefinition(
|
||||
type="FUNCTION",
|
||||
name=tool_function.get("name"),
|
||||
description=tool_function.get("description", ""),
|
||||
parameters=tool_function.get("parameters", {}),
|
||||
)
|
||||
else:
|
||||
for tool in tools:
|
||||
if tool["type"] != "function":
|
||||
raise Exception("OCI only supports function tools")
|
||||
|
||||
tool_function = tool.get("function")
|
||||
if not isinstance(tool_function, dict):
|
||||
raise Exception("Prop `function` is not a dictionary")
|
||||
|
||||
new_tool = OCIToolDefinition(
|
||||
type="FUNCTION",
|
||||
name=tool_function.get("name"),
|
||||
description=tool_function.get("description", ""),
|
||||
parameters=tool_function.get("parameters", {}),
|
||||
)
|
||||
new_tools.append(new_tool)
|
||||
new_tools.append(new_tool)
|
||||
|
||||
return new_tools
|
||||
|
||||
@@ -877,6 +1080,58 @@ class OCIStreamWrapper(CustomStreamWrapper):
|
||||
if not chunk.startswith("data:"):
|
||||
raise ValueError(f"Chunk does not start with 'data:': {chunk}")
|
||||
dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON
|
||||
|
||||
# Check if this is a Cohere stream chunk
|
||||
if "apiFormat" in dict_chunk and dict_chunk.get("apiFormat") == "COHERE":
|
||||
return self._handle_cohere_stream_chunk(dict_chunk)
|
||||
else:
|
||||
return self._handle_generic_stream_chunk(dict_chunk)
|
||||
|
||||
def _handle_cohere_stream_chunk(self, dict_chunk: dict):
|
||||
"""Handle Cohere-specific streaming chunks."""
|
||||
try:
|
||||
typed_chunk = CohereStreamChunk(**dict_chunk)
|
||||
except TypeError as e:
|
||||
raise ValueError(f"Chunk cannot be casted to CohereStreamChunk: {str(e)}")
|
||||
|
||||
if typed_chunk.index is None:
|
||||
typed_chunk.index = 0
|
||||
|
||||
# Extract text content
|
||||
text = typed_chunk.text or ""
|
||||
|
||||
# Map finish reason to standard format
|
||||
finish_reason = typed_chunk.finishReason
|
||||
if finish_reason == "COMPLETE":
|
||||
finish_reason = "stop"
|
||||
elif finish_reason == "MAX_TOKENS":
|
||||
finish_reason = "length"
|
||||
elif finish_reason is None:
|
||||
finish_reason = None
|
||||
else:
|
||||
finish_reason = "stop"
|
||||
|
||||
# For Cohere, we don't have tool calls in the streaming format
|
||||
tool_calls = None
|
||||
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=typed_chunk.index if typed_chunk.index else 0,
|
||||
delta=Delta(
|
||||
content=text,
|
||||
tool_calls=tool_calls,
|
||||
provider_specific_fields=None,
|
||||
thinking_blocks=None,
|
||||
reasoning_content=None,
|
||||
),
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
def _handle_generic_stream_chunk(self, dict_chunk: dict):
|
||||
"""Handle generic OCI streaming chunks."""
|
||||
try:
|
||||
typed_chunk = OCIStreamChunk(**dict_chunk)
|
||||
except TypeError as e:
|
||||
|
||||
@@ -1125,6 +1125,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
||||
api_base: Optional[str] = None,
|
||||
client: Optional[AsyncOpenAI] = None,
|
||||
max_retries=None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
):
|
||||
try:
|
||||
openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore
|
||||
@@ -1134,6 +1135,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
headers, response = await self.make_openai_embedding_request(
|
||||
openai_aclient=openai_aclient,
|
||||
@@ -1197,6 +1199,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
||||
client=None,
|
||||
aembedding=None,
|
||||
max_retries: Optional[int] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> EmbeddingResponse:
|
||||
super().embedding()
|
||||
try:
|
||||
@@ -1223,6 +1226,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
max_retries=max_retries,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
openai_client: OpenAI = self._get_openai_client( # type: ignore
|
||||
|
||||
@@ -161,6 +161,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
||||
) -> ResponsesAPIResponse:
|
||||
"""No transform applied since outputs are in OpenAI spec already"""
|
||||
try:
|
||||
logging_obj.post_call(
|
||||
original_response=raw_response.text,
|
||||
additional_args={"complete_input_dict": {}},
|
||||
)
|
||||
raw_response_json = raw_response.json()
|
||||
raw_response_json["created_at"] = _safe_convert_created_field(
|
||||
raw_response_json["created_at"]
|
||||
@@ -169,7 +173,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
||||
raise OpenAIError(
|
||||
message=raw_response.text, status_code=raw_response.status_code
|
||||
)
|
||||
return ResponsesAPIResponse.model_construct(**raw_response_json)
|
||||
try:
|
||||
return ResponsesAPIResponse(**raw_response_json)
|
||||
except Exception:
|
||||
verbose_logger.debug(
|
||||
f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
|
||||
)
|
||||
return ResponsesAPIResponse.model_construct(**raw_response_json)
|
||||
|
||||
def validate_environment(
|
||||
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
|
||||
|
||||
@@ -29,6 +29,38 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def should_fake_stream(
|
||||
self,
|
||||
model: Optional[str],
|
||||
stream: Optional[bool],
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Vertex AI Gemma models do not support streaming.
|
||||
Return True to enable fake streaming on the client side.
|
||||
"""
|
||||
return True
|
||||
|
||||
def _handle_fake_stream_response(
|
||||
self,
|
||||
model_response: ModelResponse,
|
||||
stream: bool,
|
||||
) -> Union[ModelResponse, Any]:
|
||||
"""
|
||||
Helper method to return fake stream iterator if streaming is requested.
|
||||
|
||||
Args:
|
||||
model_response: The completed model response
|
||||
stream: Whether streaming was requested
|
||||
|
||||
Returns:
|
||||
MockResponseIterator if stream=True, otherwise the model_response
|
||||
"""
|
||||
if stream:
|
||||
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
|
||||
return MockResponseIterator(model_response=model_response)
|
||||
return model_response
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
@@ -52,41 +84,10 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Remove 'model' from the request as it's not needed in the instance
|
||||
openai_request.pop("model", None)
|
||||
|
||||
# Wrap in Vertex Gemma format
|
||||
return {
|
||||
"instances": [
|
||||
{
|
||||
"@requestFormat": "chatCompletions",
|
||||
**openai_request,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async def async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Async version of transform_request.
|
||||
"""
|
||||
# Get the base OpenAI request from parent class
|
||||
openai_request = await super().async_transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Remove 'model' from the request as it's not needed in the instance
|
||||
# Remove params not needed/supported by Vertex Gemma
|
||||
openai_request.pop("model", None)
|
||||
openai_request.pop("stream", None) # Streaming not supported, will be faked client-side
|
||||
openai_request.pop("stream_options", None) # Stream options not supported
|
||||
|
||||
# Wrap in Vertex Gemma format
|
||||
return {
|
||||
@@ -137,16 +138,8 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
):
|
||||
"""
|
||||
Make completion request to Vertex Gemma endpoint.
|
||||
Supports both sync and async requests.
|
||||
Supports both sync and async requests with fake streaming.
|
||||
"""
|
||||
# Handle streaming
|
||||
stream = optional_params.get("stream", False)
|
||||
if stream:
|
||||
raise BaseLLMException(
|
||||
status_code=400,
|
||||
message="Streaming is not yet supported for Vertex AI Gemma models",
|
||||
)
|
||||
|
||||
if acompletion:
|
||||
return self._async_completion(
|
||||
model=model,
|
||||
@@ -194,6 +187,9 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
# Check if streaming is requested (will be faked)
|
||||
stream = optional_params.get("stream", False)
|
||||
|
||||
# Transform the request using parent class methods
|
||||
request_data = self.transform_request(
|
||||
model=model,
|
||||
@@ -260,7 +256,8 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
)
|
||||
|
||||
return model_response
|
||||
# Return fake stream iterator if streaming was requested
|
||||
return self._handle_fake_stream_response(model_response=model_response, stream=stream)
|
||||
|
||||
async def _async_completion(
|
||||
self,
|
||||
@@ -277,9 +274,13 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
encoding: Any,
|
||||
):
|
||||
"""Asynchronous completion request"""
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
# Check if streaming is requested (will be faked)
|
||||
stream = optional_params.get("stream", False)
|
||||
|
||||
# Transform the request using parent class async methods
|
||||
request_data = await self.async_transform_request(
|
||||
model=model,
|
||||
@@ -306,7 +307,9 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
)
|
||||
|
||||
# Make the HTTP request
|
||||
http_handler = AsyncHTTPHandler(concurrent_limit=1)
|
||||
http_handler = get_async_httpx_client(
|
||||
llm_provider=LlmProviders.VERTEX_AI,
|
||||
)
|
||||
response = await http_handler.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
@@ -346,5 +349,6 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
)
|
||||
|
||||
return model_response
|
||||
# Return fake stream iterator if streaming was requested
|
||||
return self._handle_fake_stream_response(model_response=model_response, stream=stream)
|
||||
|
||||
|
||||
@@ -3996,6 +3996,7 @@ def embedding( # noqa: PLR0915
|
||||
"""
|
||||
azure = kwargs.get("azure", None)
|
||||
client = kwargs.pop("client", None)
|
||||
shared_session = kwargs.get("shared_session", None)
|
||||
max_retries = kwargs.get("max_retries", None)
|
||||
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
||||
mock_response: Optional[List[float]] = kwargs.get("mock_response", None) # type: ignore
|
||||
@@ -4185,6 +4186,7 @@ def embedding( # noqa: PLR0915
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
max_retries=max_retries,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
elif custom_llm_provider == "databricks":
|
||||
api_base = api_base or litellm.api_base or get_secret("DATABRICKS_API_BASE") # type: ignore
|
||||
|
||||
@@ -866,6 +866,36 @@
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0
|
||||
},
|
||||
"au.anthropic.claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
"cache_read_input_token_cost": 3.3e-07,
|
||||
"input_cost_per_token": 3.3e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 6.6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.475e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 8.25e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6.6e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 200000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.65e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 346
|
||||
},
|
||||
"azure/ada": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "azure",
|
||||
@@ -13044,34 +13074,6 @@
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gpt-5-codex": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1e-05,
|
||||
"supported_endpoints": [
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"gpt-5-2025-08-07": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
"cache_read_input_token_cost_flex": 6.25e-08,
|
||||
@@ -16633,6 +16635,42 @@
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-latest": {
|
||||
"input_cost_per_token": 1.56e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-06,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-a-03-2025": {
|
||||
"input_cost_per_token": 1.56e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 256000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-06,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-plus-latest": {
|
||||
"input_cost_per_token": 1.56e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-06,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"ollama/codegeex4": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "ollama",
|
||||
@@ -19644,6 +19682,22 @@
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"together_ai/baai/bge-base-en-v1.5": {
|
||||
"input_cost_per_token": 8e-09,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 768
|
||||
},
|
||||
"together_ai/BAAI/bge-base-en-v1.5": {
|
||||
"input_cost_per_token": 8e-09,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 768
|
||||
},
|
||||
"together-ai-up-to-4b": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
|
||||
@@ -1380,6 +1380,7 @@ class MCPServerManager:
|
||||
if not server:
|
||||
return {
|
||||
"server_id": server_id,
|
||||
"server_name": None,
|
||||
"status": "unknown",
|
||||
"error": "Server not found",
|
||||
"last_health_check": datetime.now().isoformat(),
|
||||
@@ -1394,6 +1395,7 @@ class MCPServerManager:
|
||||
|
||||
return {
|
||||
"server_id": server_id,
|
||||
"server_name": server.name,
|
||||
"status": "healthy",
|
||||
"tools_count": len(tools),
|
||||
"last_health_check": datetime.now().isoformat(),
|
||||
@@ -1406,6 +1408,7 @@ class MCPServerManager:
|
||||
|
||||
return {
|
||||
"server_id": server_id,
|
||||
"server_name": server.name,
|
||||
"status": "unhealthy",
|
||||
"last_health_check": datetime.now().isoformat(),
|
||||
"response_time_ms": round(response_time, 2),
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
import json
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from mcp.types import Tool as MCPToolSDKTool
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.proxy.types_utils.utils import get_instance_fn
|
||||
from litellm.types.mcp_server.tool_registry import MCPTool
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mcp.types import Tool as MCPToolSDKTool
|
||||
else:
|
||||
try:
|
||||
from mcp.types import Tool as MCPToolSDKTool
|
||||
except ImportError:
|
||||
MCPToolSDKTool = None # type: ignore
|
||||
|
||||
|
||||
class MCPToolRegistry:
|
||||
"""
|
||||
@@ -55,7 +61,11 @@ class MCPToolRegistry:
|
||||
|
||||
def convert_tools_to_mcp_sdk_tool_type(
|
||||
self, tools: List[MCPTool]
|
||||
) -> List[MCPToolSDKTool]:
|
||||
) -> List["MCPToolSDKTool"]:
|
||||
if MCPToolSDKTool is None:
|
||||
raise ImportError(
|
||||
"MCP SDK is not installed. Please install it with: pip install 'litellm[proxy]'"
|
||||
)
|
||||
return [
|
||||
MCPToolSDKTool(
|
||||
name=tool.name,
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
-1
File diff suppressed because one or more lines are too long
-1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
-1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[1979],{25886:function(e,n,t){Promise.resolve().then(t.bind(t,37492))},37492:function(e,n,t){"use strict";t.r(n);var r=t(57437),l=t(44696),s=t(39760);n.default=()=>{let{token:e,accessToken:n,userRole:t,userId:a,premiumUser:i}=(0,s.Z)();return(0,r.jsx)(l.Z,{accessToken:n,token:e,userRole:t,userID:a,premiumUser:i})}},39760:function(e,n,t){"use strict";var r=t(2265),l=t(99376),s=t(14474),a=t(3914);n.Z=()=>{var e,n,t,i,o,u,c;let d=(0,l.useRouter)(),m="undefined"!=typeof document?(0,a.e)("token"):null;(0,r.useEffect)(()=>{m||d.replace("/sso/key/generate")},[m,d]);let f=(0,r.useMemo)(()=>{if(!m)return null;try{return(0,s.o)(m)}catch(e){return(0,a.b)(),d.replace("/sso/key/generate"),null}},[m,d]);return{token:m,accessToken:null!==(e=null==f?void 0:f.key)&&void 0!==e?e:null,userId:null!==(n=null==f?void 0:f.user_id)&&void 0!==n?n:null,userEmail:null!==(t=null==f?void 0:f.user_email)&&void 0!==t?t:null,userRole:function(e){if(!e)return"Undefined Role";switch(e.toLowerCase()){case"app_owner":case"demo_app_owner":return"App Owner";case"app_admin":case"proxy_admin":return"Admin";case"proxy_admin_viewer":return"Admin Viewer";case"org_admin":return"Org Admin";case"internal_user":return"Internal User";case"internal_user_viewer":case"internal_viewer":return"Internal Viewer";case"app_user":return"App User";default:return"Unknown Role"}}(null!==(i=null==f?void 0:f.user_role)&&void 0!==i?i:null),premiumUser:null!==(o=null==f?void 0:f.premium_user)&&void 0!==o?o:null,disabledPersonalKeyCreation:null!==(u=null==f?void 0:f.disabled_non_admin_personal_key_creation)&&void 0!==u?u:null,showSSOBanner:(null==f?void 0:f.login_method)==="username_password"}}},39789:function(e,n,t){"use strict";t.d(n,{Z:function(){return i}});var r=t(57437),l=t(2265),s=t(21487),a=t(84264),i=e=>{let{value:n,onValueChange:t,label:i="Select Time Range",className:o="",showTimeRange:u=!0}=e,[c,d]=(0,l.useState)(!1),m=(0,l.useRef)(null),f=(0,l.useCallback)(e=>{d(!0),setTimeout(()=>d(!1),1500),t(e),requestIdleCallback(()=>{if(e.from){let n;let r={...e},l=new Date(e.from);n=new Date(e.to?e.to:e.from),l.toDateString(),n.toDateString(),l.setHours(0,0,0,0),n.setHours(23,59,59,999),r.from=l,r.to=n,t(r)}},{timeout:100})},[t]),h=(0,l.useCallback)((e,n)=>{if(!e||!n)return"";let t=e=>e.toLocaleString("en-US",{month:"short",day:"numeric",hour:"2-digit",minute:"2-digit",hour12:!0,timeZoneName:"short"});if(e.toDateString()!==n.toDateString())return"".concat(t(e)," - ").concat(t(n));{let t=e.toLocaleDateString("en-US",{month:"short",day:"numeric",year:"numeric"}),r=e.toLocaleTimeString("en-US",{hour:"2-digit",minute:"2-digit",hour12:!0}),l=n.toLocaleTimeString("en-US",{hour:"2-digit",minute:"2-digit",hour12:!0,timeZoneName:"short"});return"".concat(t,": ").concat(r," - ").concat(l)}},[]);return(0,r.jsxs)("div",{className:o,children:[i&&(0,r.jsx)(a.Z,{className:"mb-2",children:i}),(0,r.jsxs)("div",{className:"relative w-fit",children:[(0,r.jsx)("div",{ref:m,children:(0,r.jsx)(s.Z,{enableSelect:!0,value:n,onValueChange:f,placeholder:"Select date range",enableClear:!1,style:{zIndex:100}})}),c&&(0,r.jsx)("div",{className:"absolute top-1/2 animate-pulse",style:{left:"calc(100% + 8px)",transform:"translateY(-50%)",zIndex:110},children:(0,r.jsxs)("div",{className:"flex items-center gap-1 text-green-600 text-sm font-medium bg-white px-2 py-1 rounded-full border border-green-200 shadow-sm whitespace-nowrap",children:[(0,r.jsx)("div",{className:"w-3 h-3 bg-green-500 text-white rounded-full flex items-center justify-center text-xs",children:"✓"}),(0,r.jsx)("span",{className:"text-xs",children:"Selected"})]})})]}),u&&n.from&&n.to&&(0,r.jsx)(a.Z,{className:"mt-2 text-xs text-gray-500",children:h(n.from,n.to)})]})}}},function(e){e.O(0,[9820,1491,1526,2926,9678,7281,2344,1487,2662,8049,4696,2971,2117,1744],function(){return e(e.s=25886)}),_N_E=e.O()}]);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user