tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-11 19:48:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into litellm_dev_08_16_2025_p2

Browse Source
This commit is contained in:
Krish Dholakia
2025-08-23 11:30:51 -07:00
committed by GitHub
parent 385bb15219 06a2915e9f
commit ef66bb87b3
317 changed files with 13057 additions and 4016 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+9 -9
View File
@@ -95,7 +95,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -220,7 +220,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -327,7 +327,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -602,7 +602,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -1522,7 +1522,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@@ -1679,7 +1679,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
# Run pytest and generate JUnit XML report
- run:
name: Install dockerize
@@ -1819,7 +1819,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@@ -2399,7 +2399,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
pip install "openai==1.99.5"
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
@@ -2790,7 +2790,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.99.5"
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
.circleci/requirements.txt
+1 -1
View File
@@ -1,5 +1,5 @@
# used by CI/CD testing
openai==1.99.5
openai==1.100.1
python-dotenv
tiktoken
importlib_metadata
.github/workflows/test-linting.yml
+5 -4
View File
@@ -22,11 +22,8 @@ jobs:
- name: Install dependencies
run: |
pip install openai==1.99.5
poetry install --with dev
pip install openai==1.99.5
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
@@ -40,6 +37,10 @@ jobs:
poetry run ruff check .
cd ..
- name: Print OpenAI version
run: |
poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm
.gitignore
+2 -1
View File
@@ -94,4 +94,5 @@ test.py
litellm_config.yaml
.cursor
.vscode/launch.json
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
Dockerfile
+2 -2
View File
@@ -65,8 +65,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate
cookbook/liteLLM_Baseten.ipynb
Vendored
+61 -152
View File
@@ -6,19 +6,21 @@
"id": "gZx-wHJapG5w"
},
"source": [
"# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
"# LiteLLM with Baseten Model APIs\n",
"\n",
"* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
"* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
"This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
"\n",
"\n",
"## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
"Example call\n",
"## Example Usage\n",
"```python\n",
"model = \"q841o8w\" # baseten model version ID\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"```"
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
" max_tokens=1000,\n",
" temperature=0.7\n",
")\n",
"```\n",
"\n",
"## Setup"
]
},
{
@@ -29,20 +31,25 @@
},
"outputs": [],
"source": [
"!pip install litellm==0.1.399\n",
"!pip install baseten urllib3"
"%pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {
"id": "VEukLhDzo4vw"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion"
"from litellm import completion\n",
"\n",
"# Set your Baseten API key\n",
"os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
"\n",
"# Test message\n",
"messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
]
},
{
@@ -51,19 +58,31 @@
"id": "4STYM2OHFNlc"
},
"source": [
"## Setup"
"## Example 1: Basic Completion\n",
"\n",
"Simple completion with the GPT-OSS 120B model"
]
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": null,
"metadata": {
"id": "DorpLxw1FHbC"
},
"outputs": [],
"source": [
"os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
"messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
"print(\"=== Basic Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=messages,\n",
" max_tokens=1000,\n",
" temperature=0.7,\n",
" top_p=0.9,\n",
" presence_penalty=0.1,\n",
" frequency_penalty=0.1,\n",
")\n",
"print(f\"Response: {response.choices[0].message.content}\")\n",
"print(f\"Usage: {response.usage}\")"
]
},
{
@@ -72,13 +91,14 @@
"id": "syF3dTdKFSQQ"
},
"source": [
"## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"### Pass Your Baseten model `Version ID` as `model`"
"## Example 2: Streaming Completion\n",
"\n",
"Streaming completion with usage statistics"
]
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -86,137 +106,26 @@
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
" 'created': 1692135883.699066,\n",
" 'model': 'qvv0xeq'}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"model = \"qvv0xeq\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7n21UroEGCGa"
},
"source": [
"## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uLVWFH899lAF",
"outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
" 'created': 1692135900.2806294,\n",
" 'model': 'q841o8w'}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"q841o8w\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "6-TFwmPAGPXq"
},
"source": [
"## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gbeYZOrUE_Bp",
"outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
" 'created': 1692135914.7472186,\n",
" 'model': '31dxrj3'}"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"31dxrj3\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
"print(\"=== Streaming Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
" stream=True,\n",
" max_tokens=500,\n",
" temperature=0.8,\n",
" stream_options={\n",
" \"include_usage\": True,\n",
" \"continuous_usage_stats\": True\n",
" },\n",
")\n",
"\n",
"print(\"Streaming response:\")\n",
"for chunk in response:\n",
" if chunk.choices and chunk.choices[0].delta.content:\n",
" print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
"print(\"\\n\")"
]
}
],
@@ -234,4 +143,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}
deploy/charts/litellm-helm/Chart.yaml
+1 -1
View File
@@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.4
version: 0.4.5
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
deploy/charts/litellm-helm/README.md
+2 -2
View File
@@ -24,7 +24,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
@@ -135,7 +135,7 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
```bash
kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"
deploy/charts/litellm-helm/templates/deployment.yaml
+7
View File
@@ -71,7 +71,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
deploy/charts/litellm-helm/templates/migrations-job.yaml
+11
View File
@@ -49,7 +49,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
@@ -73,6 +80,10 @@ spec:
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}
deploy/charts/litellm-helm/templates/secret-masterkey.yaml
+1 -1
View File
@@ -1,5 +1,5 @@
{{- if not .Values.masterkeySecretName }}
{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
apiVersion: v1
kind: Secret
metadata:
deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml
+7 -1
View File
@@ -2,13 +2,19 @@ suite: test masterkey secret
templates:
- secret-masterkey.yaml
tests:
- it: should create a secret if masterkeySecretName is not set
- it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
asserts:
- isKind:
of: Secret
- matchRegex:
path: data.masterkey
pattern: ^c2st
# Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
# but stored as base64 encoded in Kubernetes secret (requirement).
# "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
- it: should not create a secret if masterkeySecretName is set
template: secret-masterkey.yaml
set:
deploy/charts/litellm-helm/values.yaml
+6
View File
@@ -161,6 +161,8 @@ db:
name: postgres
usernameKey: username
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
@@ -206,6 +208,10 @@ migrationJob:
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
annotations: {}
ttlSecondsAfterFinished: 120
resources: {}
# requests:
# cpu: 100m
# memory: 100Mi
extraContainers: []
# Hook configuration
docker/Dockerfile.database
+2 -2
View File
@@ -57,8 +57,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
docker/Dockerfile.non_root
+5 -3
View File
@@ -47,8 +47,8 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \
@@ -70,7 +70,9 @@ RUN mkdir -p /nonexistent /.npm && \
chown -R nobody:nogroup /app && \
chown -R nobody:nogroup /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
# Get paths for directories that need write access at runtime
docker/build_from_pip/requirements.txt
+1
View File
@@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
openai==1.99.9
ddtrace==2.19.0 # for advanced DD tracing / profiling
docker/install_auto_router.sh
Executable
+3
View File
@@ -0,0 +1,3 @@
#!/bin/bash
pip install semantic_router==0.1.11 --no-deps
pip install aurelio-sdk==0.0.19
docs/my-website/docs/completion/document_understanding.md
+66
View File
@@ -10,6 +10,7 @@ Works for:
- Bedrock Models
- Anthropic API Models
- OpenAI API Models
- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
## Quick Start
@@ -279,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## Mistral Example
Here is a sample payload for using the Mistral model for document understanding:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm.utils import completion
# pdf file_id received from files endpoint
file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
# model
model = "mistral/mistral-large-latest"
file_content = [
{"type": "text", "text": "What's this file about?"},
{
"type": "file",
"file": {
"file_id": file_id,
}
},
]
response = completion(
model=model,
messages=[{"role": "user", "content": file_content}],
)
assert response is not None
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "mistral/mistral-large-latest",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is the content of the file?"
},
{
"type": "file",
"file": {
"file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
}
}
]
}
]
}
```
</TabItem>
</Tabs>
## Checking if a model supports pdf input
<Tabs>
docs/my-website/docs/providers/baseten.md
+98 -15
View File
@@ -1,23 +1,106 @@
# Baseten
LiteLLM supports any Text-Gen-Interface models on Baseten.
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
[Here's a tutorial on deploying a huggingface TGI model (Llama2, CodeLlama, WizardCoder, Falcon, etc.) on Baseten](https://truss.baseten.co/examples/performance/tgi-server)
# Baseten
LiteLLM supports both Baseten Model APIs and dedicated deployments with automatic routing.
## API Types
### Model API (Default)
- **URL**: `https://inference.baseten.co/v1`
- **Format**: `baseten/<model-name>` (e.g., `baseten/openai/gpt-oss-120b`)
- **Best for**: Quick access to popular models
### Dedicated Deployments
- **URL**: `https://model-{id}.api.baseten.co/environments/production/sync/v1`
- **Format**: `baseten/{8-digit-alphanumeric-code}` (e.g., `baseten/abcd1234`)
- **Best for**: Custom models, latency SLAs
:::tip
**Automatic Routing**: LiteLLM detects the type based on model format:
- 8-digit alphanumeric codes → Dedicated deployment
- All other formats → Model API
:::
## Quick Start
### API KEYS
```python
import os
os.environ["BASETEN_API_KEY"] = ""
import os
from litellm import completion
os.environ['BASETEN_API_KEY'] = "your-api-key"
# Model API (default)
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Hello!"}]
)
# Dedicated deployment (8-digit ID)
response = completion(
model="baseten/abcd1234",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### Baseten Models
Baseten provides infrastructure to deploy and serve ML models https://www.baseten.co/. Use liteLLM to easily call models deployed on Baseten.
## Examples
Example Baseten Usage - Note: liteLLM supports all models deployed on Baseten
### Basic Usage
```python
# Model API
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Explain quantum computing"}],
max_tokens=500,
temperature=0.7
)
Usage: Pass `model=baseten/<Model ID>`
# Dedicated deployment
response = completion(
model="baseten/abcd1234",
messages=[{"role": "user", "content": "Explain quantum computing"}],
max_tokens=500,
temperature=0.7
)
```
| Model Name | Function Call | Required OS Variables |
|------------------|--------------------------------------------|------------------------------------|
| Falcon 7B | `completion(model='baseten/qvv0xeq', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
| Wizard LM | `completion(model='baseten/q841o8w', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
| MPT 7B Base | `completion(model='baseten/31dxrj3', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
### Streaming (Model API only)
```python
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Write a poem"}],
stream=True,
stream_options={"include_usage": True}
)
for chunk in response:
if chunk.choices and chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
## Usage with LiteLLM Proxy
1. **Config**:
```yaml
model_list:
- model_name: baseten-model
litellm_params:
model: baseten/openai/gpt-oss-120b
api_key: your-baseten-api-key
```
2. **Request**:
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="baseten-model",
messages=[{"role": "user", "content": "Hello!"}]
)
```
docs/my-website/docs/providers/deepinfra.md
+140
View File
@@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# DeepInfra
https://deepinfra.com/
@@ -7,6 +10,11 @@ https://deepinfra.com/
:::
## Table of Contents
- [API Key](#api-key)
- [Chat Models](#chat-models)
- [Rerank Endpoint](#rerank-endpoint)
## API Key
```python
@@ -53,3 +61,135 @@ for chunk in response:
| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` |
| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` |
| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` |
## Rerank Endpoint
LiteLLM provides a Cohere API compatible `/rerank` endpoint for DeepInfra rerank models.
### Supported Rerank Models
| Model Name | Description |
|------------|-------------|
| `deepinfra/Qwen/Qwen3-Reranker-0.6B` | Lightweight rerank model (0.6B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-4B` | Medium rerank model (4B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-8B` | Large rerank model (8B parameters) |
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import rerank
import os
os.environ["DEEPINFRA_API_KEY"] = "your-api-key"
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add to config.yaml
```yaml
model_list:
- model_name: Qwen/Qwen3-Reranker-0.6B
litellm_params:
model: deepinfra/Qwen/Qwen3-Reranker-0.6B
api_key: os.environ/DEEPINFRA_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000/
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/rerank' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "Qwen/Qwen3-Reranker-0.6B",
"query": "What is the capital of France?",
"documents": [
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
}'
```
</TabItem>
</Tabs>
### Supported Cohere Rerank API Params
| Param | Type | Description |
| ------------------ | ----------- | ----------------------------------------------- |
| `query` | `str` | The query to rerank the documents against |
| `documents` | `list[str]` | The documents to rerank |
### Provider-specific parameters
Pass any deepinfra specific parameters as a keyword argument to the rerank function, e.g.
```
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
],
my_custom_param="my_custom_value", # any other deepinfra specific parameters
)
```
### Response Format
```json
{
"id": "request-id",
"results": [
{
"index": 0,
"relevance_score": 0.9975274205207825
},
{
"index": 1,
"relevance_score": 0.011687257327139378
}
],
"meta": {
"billed_units": {
"total_tokens": 427
},
"tokens": {
"input_tokens": 427,
"output_tokens": 0
}
}
}
```
docs/my-website/docs/providers/litellm_proxy.md
+16 -1
View File
@@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
| Description | LiteLLM Proxy is an OpenAI-compatible gateway that allows you to interact with multiple LLM providers through a unified API. Simply use the `litellm_proxy/` prefix before the model name to route your requests through the proxy. |
| Provider Route on LiteLLM | `litellm_proxy/` (add this prefix to the model name, to route any requests to litellm_proxy - e.g. `litellm_proxy/your-model-name`) |
| Setup LiteLLM Gateway | [LiteLLM Gateway ↗](../simple_proxy) |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/rerank` |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/images/edits`, `/rerank` |
@@ -111,6 +111,21 @@ response = litellm.image_generation(
)
```
## Image Edit
```python
import litellm
with open("your-image.png", "rb") as f:
response = litellm.image_edit(
model="litellm_proxy/gpt-image-1",
prompt="Make this image a watercolor painting",
image=[f],
api_base="your-litellm-proxy-url",
api_key="your-litellm-proxy-api-key",
)
```
## Audio Transcription
```python
docs/my-website/docs/providers/vertex_partner.md
+87
View File
@@ -14,6 +14,7 @@ import TabItem from '@theme/TabItem';
| Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) |
| Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
| Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
## Vertex AI - Anthropic (Claude)
@@ -571,6 +572,92 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## VertexAI Qwen API
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/qwen/{MODEL}` |
| Vertex Documentation | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
**LiteLLM Supports all Vertex AI Qwen Models.** Ensure you use the `vertex_ai/qwen/` prefix for all Vertex AI Qwen models.
| Model Name | Usage |
|------------------|------------------------------|
| vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | `completion('vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas', messages)` |
| vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas | `completion('vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas', messages)` |
#### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
model = "qwen/qwen3-coder-480b-a35b-instruct-maas"
vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
response = completion(
model="vertex_ai/" + model,
messages=[{"role": "user", "content": "hi"}],
vertex_ai_project=vertex_ai_project,
vertex_ai_location=vertex_ai_location,
)
print("\nModel Response", response)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: vertex-qwen
litellm_params:
model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
- model_name: vertex-qwen
litellm_params:
model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-west-1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "vertex-qwen", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
## Model Garden
:::tip
docs/my-website/docs/proxy/config_settings.md
+5
View File
@@ -335,12 +335,16 @@ router_settings:
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
| AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
| AWS_PROFILE_NAME | AWS CLI profile name to be used
| AWS_REGION | AWS region for service interactions (takes precedence over AWS_DEFAULT_REGION)
| AWS_REGION_NAME | Default AWS region for service interactions
| AWS_ROLE_ARN | ARN of the AWS IAM role to assume for authentication
| AWS_ROLE_NAME | Role name for AWS IAM usage
| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
| AWS_SESSION_NAME | Name for AWS session
| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
| AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
| AZURE_API_VERSION | Version of the Azure API being used
| AZURE_AUTHORITY_HOST | Azure authority host URL
| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
@@ -349,6 +353,7 @@ router_settings:
| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow
docs/my-website/docs/proxy/team_budgets.md
+6
View File
@@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
# Setting Team Budgets
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
Track spend, set budgets for your Internal Team
## Setting Monthly Team Budgets
docs/my-website/docs/proxy/users.md
+8
View File
@@ -58,6 +58,9 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
> **Prerequisite:**
> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
:::
@@ -793,6 +796,11 @@ Expected Response:
Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.
Changes:
- This moves to using async_increment instead of async_set_cache when updating current requests/tokens.
- The in-memory cache is synced with redis every 0.01s, to avoid calling redis for every request.
docs/my-website/docs/rerank.md
+2 -1
View File
@@ -118,4 +118,5 @@ curl http://0.0.0.0:4000/rerank \
| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
| Infinity| [Usage](../docs/providers/infinity) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |
docs/my-website/docs/response_api.md
+10 -2
View File
@@ -803,10 +803,18 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you
1. Enable storing request / response content in the database
Set `store_prompts_in_spend_logs: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the database.
Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify.
```yaml showLineNumbers title="config.yaml with Session Continuity"
litellm_settings:
callbacks: ["s3_v2"]
cold_storage_custom_logger: s3_v2
s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets
s3_bucket_name: litellm-logs # AWS Bucket Name for S3
s3_region_name: us-west-2
```yaml
general_settings:
store_prompts_in_cold_storage: true
store_prompts_in_spend_logs: true
```
docs/my-website/img/release_notes/faster_caching_calls.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 270 KiB

docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 472 KiB

docs/my-website/package-lock.json
Generated
+3 -3
View File
@@ -12933,9 +12933,9 @@
}
},
"node_modules/mermaid": {
"version": "11.9.0",
"resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.9.0.tgz",
"integrity": "sha512-YdPXn9slEwO0omQfQIsW6vS84weVQftIyyTGAZCwM//MGhPzL1+l6vO6bkf0wnP4tHigH1alZ5Ooy3HXI2gOag==",
"version": "11.10.0",
"resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.10.0.tgz",
"integrity": "sha512-oQsFzPBy9xlpnGxUqLbVY8pvknLlsNIJ0NWwi8SUJjhbP1IT0E0o1lfhU4iYV3ubpy+xkzkaOyDUQMn06vQElQ==",
"dependencies": {
"@braintree/sanitize-url": "^7.0.4",
"@iconify/utils": "^2.1.33",
docs/my-website/package.json
+2 -1
View File
@@ -49,6 +49,7 @@
},
"overrides": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4"
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0"
}
}
docs/my-website/release_notes/v1.75.5-stable/index.md
+44 -3
View File
@@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.75.5-stable"
title: "v1.75.5-stable - Redis latency improvements"
slug: "v1-75-5"
date: 2025-08-10T10:00:00
authors:
@@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.5.rc.1
ghcr.io/berriai/litellm:v1.75.5-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.5.post1
pip install litellm==1.75.5.post2
```
</TabItem>
@@ -43,8 +43,49 @@ pip install litellm==1.75.5.post1
---
## Key Highlights
- **Redis - Latency Improvements** - Reduces P99 latency by 50% with Redis enabled.
- **Responses API Session Management** - Support for managing responses API sessions with images.
- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure.
- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform.
### Risk of Upgrade
If you build the proxy from the pip package, you should hold off on upgrading. This version makes `prisma migrate deploy` our default for managing the DB. This is safer, as it doesn't reset the DB, but it requires a manual `prisma generate` step.
Users of our Docker image, are **not** affected by this change.
---
## Redis Latency Improvements
<Image
img={require('../../img/release_notes/faster_caching_calls.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
This release adds in-memory caching for Redis requests, enabling faster response times in high-traffic. Now, LiteLLM instances will check their in-memory cache for a cache hit, before checking Redis. This reduces caching-related latency from 100ms for LLM API calls to sub-1ms, on cache hits.
---
## Responses API Session Management w/ Images
<Image
img={require('../../img/release_notes/responses_api_session_mgt_images.jpg')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
LiteLLM now supports session management for Responses API requests with images. This is great for use-cases like chatbots, that are using the Responses API to track the state of a conversation. LiteLLM session management works across **ALL** LLM API's (including Anthropic, Bedrock, OpenAI, etc). LiteLLM session management works by storing the request and response content in an s3 bucket, you can specify.
---
## New Models / Updated Models
#### New Model Support
docs/my-website/release_notes/v1.75.8/index.md
+231
View File
@@ -0,0 +1,231 @@
---
title: "[PRE-RELEASE]v1.75.8"
slug: "v1-75-8"
date: 2025-08-16T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.8
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.8
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Team Member Rate Limits** - Individual rate limiting for team members with JWT authentication support.
- **Performance Improvements** - New experimental HTTP handler flag for 100+ RPS improvement on OpenAI calls.
- **GPT-5 Model Family Support** - Full support for OpenAI's GPT-5 models with `reasoning_effort` parameter and Azure OpenAI integration.
- **Azure AI Flux Image Generation** - Support for Azure AI's Flux image generation models.
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
| Azure AI | `azure_ai/FLUX-1.1-pro` | - | - | $40/image | Image generation |
| Azure AI | `azure_ai/FLUX.1-Kontext-pro` | - | - | $40/image | Image generation |
| Vertex AI | `vertex_ai/deepseek-ai/deepseek-r1-0528-maas` | 65k | $1.35 | $5.4 | Chat completions + reasoning |
| OpenRouter | `openrouter/deepseek/deepseek-chat-v3-0324` | 65k | $0.14 | $0.28 | Chat completions |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Added `reasoning_effort` parameter support for GPT-5 model family - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/providers/openai#openai-chat-completion-models)
- Support for `reasoning` parameter in Responses API - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/response_api)
- **[Azure OpenAI](../../docs/providers/azure/azure)**
- GPT-5 support with max_tokens and `reasoning` parameter - [PR #13510](https://github.com/BerriAI/litellm/pull/13510), [Get Started](../../docs/providers/azure/azure#gpt-5-models)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Streaming support for bedrock gpt-oss model family - [PR #13346](https://github.com/BerriAI/litellm/pull/13346), [Get Started](../../docs/providers/bedrock#openai-gpt-oss)
- `/messages` endpoint compatibility with `bedrock/converse/<model>` - [PR #13627](https://github.com/BerriAI/litellm/pull/13627)
- Cache point support for assistant and tool messages - [PR #13640](https://github.com/BerriAI/litellm/pull/13640)
- **[Azure AI](../../docs/providers/azure)**
- New Azure AI Flux Image Generation provider - [PR #13592](https://github.com/BerriAI/litellm/pull/13592), [Get Started](../../docs/providers/azure_ai_img)
- Fixed Content-Type header for image generation - [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
- **[CometAPI](../../docs/providers/comet)**
- New provider support with chat completions and streaming - [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
- **[SambaNova](../../docs/providers/sambanova)**
- Added embedding model support - [PR #13308](https://github.com/BerriAI/litellm/pull/13308), [Get Started](../../docs/providers/sambanova#sambanova---embeddings)
- **[Vertex AI](../../docs/providers/vertex)**
- Added `/countTokens` endpoint support for Gemini CLI integration - [PR #13545](https://github.com/BerriAI/litellm/pull/13545)
- Token counter support for VertexAI models - [PR #13558](https://github.com/BerriAI/litellm/pull/13558)
- **[hosted_vllm](../../docs/providers/vllm)**
- Added `reasoning_effort` parameter support - [PR #13620](https://github.com/BerriAI/litellm/pull/13620), [Get Started](../../docs/providers/vllm#reasoning-effort)
#### Bugs
- **[OCI](../../docs/providers/oci)**
- Fixed streaming issues - [PR #13437](https://github.com/BerriAI/litellm/pull/13437)
- **[Ollama](../../docs/providers/ollama)**
- Fixed GPT-OSS streaming with 'thinking' field - [PR #13375](https://github.com/BerriAI/litellm/pull/13375)
- **[VolcEngine](../../docs/providers/volcengine)**
- Fixed thinking disabled parameter handling - [PR #13598](https://github.com/BerriAI/litellm/pull/13598)
- **[Streaming](../../docs/completion/stream)**
- Consistent 'finish_reason' chunk indexing - [PR #13560](https://github.com/BerriAI/litellm/pull/13560)
---
## LLM API Endpoints
#### Features
- **[/messages](../../docs/anthropic/messages)**
- Tool use arguments properly returned for non-anthropic models - [PR #13638](https://github.com/BerriAI/litellm/pull/13638)
#### Bugs
- **[Real-time API](../../docs/realtime)**
- Fixed endpoint for no intent scenarios - [PR #13476](https://github.com/BerriAI/litellm/pull/13476)
- **[Responses API](../../docs/response_api)**
- Fixed `stream=True` + `background=True` with Responses API - [PR #13654](https://github.com/BerriAI/litellm/pull/13654)
---
## [MCP Gateway](../../docs/mcp)
#### Features
- **Access Control & Configuration**
- Enhanced MCPServerManager with access groups and description support - [PR #13549](https://github.com/BerriAI/litellm/pull/13549)
#### Bugs
- **Authentication**
- Fixed MCP gateway key authentication - [PR #13630](https://github.com/BerriAI/litellm/pull/13630)
[Read More](../../docs/mcp)
---
## Management Endpoints / UI
#### Features
- **Team Management**
- Team Member Rate Limits implementation - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
- JWT authentication support for team member rate limits - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
- Show team member TPM/RPM limits in UI - [PR #13662](https://github.com/BerriAI/litellm/pull/13662)
- Allow editing team member RPM/TPM limits - [PR #13669](https://github.com/BerriAI/litellm/pull/13669)
- Allow unsetting TPM and RPM in Teams Settings - [PR #13430](https://github.com/BerriAI/litellm/pull/13430)
- Team Member Permissions Page access column changes - [PR #13145](https://github.com/BerriAI/litellm/pull/13145)
- **Key Management**
- Display errors from backend on the UI Keys page - [PR #13435](https://github.com/BerriAI/litellm/pull/13435)
- Added confirmation modal before deleting keys - [PR #13655](https://github.com/BerriAI/litellm/pull/13655)
- Support for `user` parameter in LiteLLM SDK to Proxy communication - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
- **UI Improvements**
- Fixed internal users table overflow - [PR #12736](https://github.com/BerriAI/litellm/pull/12736)
- Enhanced chart readability with short-form notation for large numbers - [PR #12370](https://github.com/BerriAI/litellm/pull/12370)
- Fixed image overflow in LiteLLM model display - [PR #13639](https://github.com/BerriAI/litellm/pull/13639)
- Removed ambiguous network response errors - [PR #13582](https://github.com/BerriAI/litellm/pull/13582)
- **Credentials**
- Added CredentialDeleteModal component and integration with CredentialsPanel - [PR #13550](https://github.com/BerriAI/litellm/pull/13550)
- **Admin & Permissions**
- Allow routes for admin viewer - [PR #13588](https://github.com/BerriAI/litellm/pull/13588)
#### Bugs
- **SCIM Integration**
- Fixed SCIM Team Memberships metadata handling - [PR #13553](https://github.com/BerriAI/litellm/pull/13553)
- **Authentication**
- Fixed incorrect key info endpoint - [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
---
## Logging / Guardrail Integrations
#### Features
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
- Added key/team logging for Langfuse OTEL Logger - [PR #13512](https://github.com/BerriAI/litellm/pull/13512)
- Fixed LangfuseOtelSpanAttributes constants to match expected values - [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
- **[MLflow](../../docs/proxy/logging#mlflow)**
- Updated MLflow logger usage span attributes - [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
#### Bugs
- **Security**
- Hide sensitive data in `/model/info` - azure entra client_secret - [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
- Fixed trivy/secrets false positives - [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **HTTP Performance**
- New 'EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER' flag for +100 RPS improvement on OpenAI calls - [PR #13625](https://github.com/BerriAI/litellm/pull/13625)
- **Database Monitoring**
- Added DB metrics to Prometheus - [PR #13626](https://github.com/BerriAI/litellm/pull/13626)
- **Error Handling**
- Added safe divide by 0 protection to prevent crashes - [PR #13624](https://github.com/BerriAI/litellm/pull/13624)
#### Bugs
- **Dependencies**
- Updated boto3 to 1.36.0 and aioboto3 to 13.4.0 - [PR #13665](https://github.com/BerriAI/litellm/pull/13665)
---
## General Proxy Improvements
#### Features
- **Database**
- Removed redundant `use_prisma_migrate` flag - now default - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
- **LLM Translation**
- Added model ID check - [PR #13507](https://github.com/BerriAI/litellm/pull/13507)
- Refactored Anthropic configurations and added support for `anthropic_beta` headers - [PR #13590](https://github.com/BerriAI/litellm/pull/13590)
---
## New Contributors
* @TensorNull made their first contribution in [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
* @MajorD00m made their first contribution in [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
* @VerunicaM made their first contribution in [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
* @huangyafei made their first contribution in [PR #13607](https://github.com/BerriAI/litellm/pull/13607)
* @TomeHirata made their first contribution in [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
* @willfinnigan made their first contribution in [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
* @dcbark01 made their first contribution in [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
* @javacruft made their first contribution in [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.75.5-stable.rc-draft...v1.75.8-nightly)**
litellm-proxy-extras/litellm_proxy_extras/utils.py
+1 -1
View File
@@ -298,7 +298,7 @@ class ProxyExtrasDBManager:
and "database schema is not empty" in e.stderr
):
logger.info(
"Database schema is not empty, creating baseline migration"
"Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system"
)
ProxyExtrasDBManager._create_baseline_migration(schema_path)
logger.info(
litellm/__init__.py
+227 -227
View File
@@ -146,6 +146,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"vector_store_pre_call_hook",
"dotprompt",
]
configured_cold_storage_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
_known_custom_logger_compatible_callbacks: List = list(
get_args(_custom_logger_compatible_callbacks_literal)
@@ -467,79 +468,80 @@ BEDROCK_CONVERSE_MODELS = [
]
####### COMPLETION MODELS ###################
open_ai_chat_completion_models: List = []
open_ai_text_completion_models: List = []
cohere_models: List = []
cohere_chat_models: List = []
mistral_chat_models: List = []
text_completion_codestral_models: List = []
anthropic_models: List = []
openrouter_models: List = []
datarobot_models: List = []
vertex_language_models: List = []
vertex_vision_models: List = []
vertex_chat_models: List = []
vertex_code_chat_models: List = []
vertex_ai_image_models: List = []
vertex_text_models: List = []
vertex_code_text_models: List = []
vertex_embedding_models: List = []
vertex_anthropic_models: List = []
vertex_llama3_models: List = []
vertex_deepseek_models: List = []
vertex_ai_ai21_models: List = []
vertex_mistral_models: List = []
ai21_models: List = []
ai21_chat_models: List = []
nlp_cloud_models: List = []
aleph_alpha_models: List = []
bedrock_models: List = []
bedrock_converse_models: List = BEDROCK_CONVERSE_MODELS
fireworks_ai_models: List = []
fireworks_ai_embedding_models: List = []
deepinfra_models: List = []
perplexity_models: List = []
watsonx_models: List = []
gemini_models: List = []
xai_models: List = []
deepseek_models: List = []
azure_ai_models: List = []
jina_ai_models: List = []
voyage_models: List = []
infinity_models: List = []
databricks_models: List = []
cloudflare_models: List = []
codestral_models: List = []
friendliai_models: List = []
featherless_ai_models: List = []
palm_models: List = []
groq_models: List = []
azure_models: List = []
azure_text_models: List = []
anyscale_models: List = []
cerebras_models: List = []
galadriel_models: List = []
sambanova_models: List = []
sambanova_embedding_models: List = []
novita_models: List = []
assemblyai_models: List = []
snowflake_models: List = []
gradient_ai_models: List = []
llama_models: List = []
nscale_models: List = []
nebius_models: List = []
nebius_embedding_models: List = []
deepgram_models: List = []
elevenlabs_models: List = []
dashscope_models: List = []
moonshot_models: List = []
v0_models: List = []
morph_models: List = []
lambda_ai_models: List = []
hyperbolic_models: List = []
recraft_models: List = []
cometapi_models: List = []
oci_models: List = []
from typing import Set
open_ai_chat_completion_models: Set = set()
open_ai_text_completion_models: Set = set()
cohere_models: Set = set()
cohere_chat_models: Set = set()
mistral_chat_models: Set = set()
text_completion_codestral_models: Set = set()
anthropic_models: Set = set()
openrouter_models: Set = set()
datarobot_models: Set = set()
vertex_language_models: Set = set()
vertex_vision_models: Set = set()
vertex_chat_models: Set = set()
vertex_code_chat_models: Set = set()
vertex_ai_image_models: Set = set()
vertex_text_models: Set = set()
vertex_code_text_models: Set = set()
vertex_embedding_models: Set = set()
vertex_anthropic_models: Set = set()
vertex_llama3_models: Set = set()
vertex_deepseek_models: Set = set()
vertex_ai_ai21_models: Set = set()
vertex_mistral_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
aleph_alpha_models: Set = set()
bedrock_models: Set = set()
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
fireworks_ai_models: Set = set()
fireworks_ai_embedding_models: Set = set()
deepinfra_models: Set = set()
perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
deepseek_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
infinity_models: Set = set()
databricks_models: Set = set()
cloudflare_models: Set = set()
codestral_models: Set = set()
friendliai_models: Set = set()
featherless_ai_models: Set = set()
palm_models: Set = set()
groq_models: Set = set()
azure_models: Set = set()
azure_text_models: Set = set()
anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
assemblyai_models: Set = set()
snowflake_models: Set = set()
gradient_ai_models: Set = set()
llama_models: Set = set()
nscale_models: Set = set()
nebius_models: Set = set()
nebius_embedding_models: Set = set()
deepgram_models: Set = set()
elevenlabs_models: Set = set()
dashscope_models: Set = set()
moonshot_models: Set = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
hyperbolic_models: Set = set()
recraft_models: Set = set()
cometapi_models: Set = set()
oci_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@@ -580,166 +582,166 @@ def add_known_models():
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(
key
):
open_ai_chat_completion_models.append(key)
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
open_ai_text_completion_models.append(key)
open_ai_text_completion_models.add(key)
elif value.get("litellm_provider") == "azure_text":
azure_text_models.append(key)
azure_text_models.add(key)
elif value.get("litellm_provider") == "cohere":
cohere_models.append(key)
cohere_models.add(key)
elif value.get("litellm_provider") == "cohere_chat":
cohere_chat_models.append(key)
cohere_chat_models.add(key)
elif value.get("litellm_provider") == "mistral":
mistral_chat_models.append(key)
mistral_chat_models.add(key)
elif value.get("litellm_provider") == "anthropic":
anthropic_models.append(key)
anthropic_models.add(key)
elif value.get("litellm_provider") == "empower":
empower_models.append(key)
empower_models.add(key)
elif value.get("litellm_provider") == "openrouter":
openrouter_models.append(key)
openrouter_models.add(key)
elif value.get("litellm_provider") == "datarobot":
datarobot_models.append(key)
datarobot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-text-models":
vertex_text_models.append(key)
vertex_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
vertex_code_text_models.append(key)
vertex_code_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-language-models":
vertex_language_models.append(key)
vertex_language_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-vision-models":
vertex_vision_models.append(key)
vertex_vision_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-chat-models":
vertex_chat_models.append(key)
vertex_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
vertex_code_chat_models.append(key)
vertex_code_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
vertex_embedding_models.append(key)
vertex_embedding_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
key = key.replace("vertex_ai/", "")
vertex_anthropic_models.append(key)
vertex_anthropic_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-llama_models":
key = key.replace("vertex_ai/", "")
vertex_llama3_models.append(key)
vertex_llama3_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
key = key.replace("vertex_ai/", "")
vertex_deepseek_models.append(key)
vertex_deepseek_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
key = key.replace("vertex_ai/", "")
vertex_mistral_models.append(key)
vertex_mistral_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
key = key.replace("vertex_ai/", "")
vertex_ai_ai21_models.append(key)
vertex_ai_ai21_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-image-models":
key = key.replace("vertex_ai/", "")
vertex_ai_image_models.append(key)
vertex_ai_image_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.append(key)
ai21_chat_models.add(key)
else:
ai21_models.append(key)
ai21_models.add(key)
elif value.get("litellm_provider") == "nlp_cloud":
nlp_cloud_models.append(key)
nlp_cloud_models.add(key)
elif value.get("litellm_provider") == "aleph_alpha":
aleph_alpha_models.append(key)
aleph_alpha_models.add(key)
elif value.get(
"litellm_provider"
) == "bedrock" and not is_bedrock_pricing_only_model(key):
bedrock_models.append(key)
bedrock_models.add(key)
elif value.get("litellm_provider") == "bedrock_converse":
bedrock_converse_models.append(key)
bedrock_converse_models.add(key)
elif value.get("litellm_provider") == "deepinfra":
deepinfra_models.append(key)
deepinfra_models.add(key)
elif value.get("litellm_provider") == "perplexity":
perplexity_models.append(key)
perplexity_models.add(key)
elif value.get("litellm_provider") == "watsonx":
watsonx_models.append(key)
watsonx_models.add(key)
elif value.get("litellm_provider") == "gemini":
gemini_models.append(key)
gemini_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.append(key)
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key:
fireworks_ai_embedding_models.append(key)
fireworks_ai_embedding_models.add(key)
elif value.get("litellm_provider") == "text-completion-codestral":
text_completion_codestral_models.append(key)
text_completion_codestral_models.add(key)
elif value.get("litellm_provider") == "xai":
xai_models.append(key)
xai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.append(key)
deepseek_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
llama_models.append(key)
llama_models.add(key)
elif value.get("litellm_provider") == "nscale":
nscale_models.append(key)
nscale_models.add(key)
elif value.get("litellm_provider") == "azure_ai":
azure_ai_models.append(key)
azure_ai_models.add(key)
elif value.get("litellm_provider") == "voyage":
voyage_models.append(key)
voyage_models.add(key)
elif value.get("litellm_provider") == "infinity":
infinity_models.append(key)
infinity_models.add(key)
elif value.get("litellm_provider") == "databricks":
databricks_models.append(key)
databricks_models.add(key)
elif value.get("litellm_provider") == "cloudflare":
cloudflare_models.append(key)
cloudflare_models.add(key)
elif value.get("litellm_provider") == "codestral":
codestral_models.append(key)
codestral_models.add(key)
elif value.get("litellm_provider") == "friendliai":
friendliai_models.append(key)
friendliai_models.add(key)
elif value.get("litellm_provider") == "palm":
palm_models.append(key)
palm_models.add(key)
elif value.get("litellm_provider") == "groq":
groq_models.append(key)
groq_models.add(key)
elif value.get("litellm_provider") == "azure":
azure_models.append(key)
azure_models.add(key)
elif value.get("litellm_provider") == "anyscale":
anyscale_models.append(key)
anyscale_models.add(key)
elif value.get("litellm_provider") == "cerebras":
cerebras_models.append(key)
cerebras_models.add(key)
elif value.get("litellm_provider") == "galadriel":
galadriel_models.append(key)
galadriel_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.append(key)
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
sambanova_embedding_models.append(key)
sambanova_embedding_models.add(key)
elif value.get("litellm_provider") == "novita":
novita_models.append(key)
novita_models.add(key)
elif value.get("litellm_provider") == "nebius-chat-models":
nebius_models.append(key)
nebius_models.add(key)
elif value.get("litellm_provider") == "nebius-embedding-models":
nebius_embedding_models.append(key)
nebius_embedding_models.add(key)
elif value.get("litellm_provider") == "assemblyai":
assemblyai_models.append(key)
assemblyai_models.add(key)
elif value.get("litellm_provider") == "jina_ai":
jina_ai_models.append(key)
jina_ai_models.add(key)
elif value.get("litellm_provider") == "snowflake":
snowflake_models.append(key)
snowflake_models.add(key)
elif value.get("litellm_provider") == "gradient_ai":
gradient_ai_models.append(key)
gradient_ai_models.add(key)
elif value.get("litellm_provider") == "featherless_ai":
featherless_ai_models.append(key)
featherless_ai_models.add(key)
elif value.get("litellm_provider") == "deepgram":
deepgram_models.append(key)
deepgram_models.add(key)
elif value.get("litellm_provider") == "elevenlabs":
elevenlabs_models.append(key)
elevenlabs_models.add(key)
elif value.get("litellm_provider") == "dashscope":
dashscope_models.append(key)
dashscope_models.add(key)
elif value.get("litellm_provider") == "moonshot":
moonshot_models.append(key)
moonshot_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.append(key)
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.append(key)
morph_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.append(key)
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "hyperbolic":
hyperbolic_models.append(key)
hyperbolic_models.add(key)
elif value.get("litellm_provider") == "recraft":
recraft_models.append(key)
recraft_models.add(key)
elif value.get("litellm_provider") == "cometapi":
cometapi_models.append(key)
cometapi_models.add(key)
elif value.get("litellm_provider") == "oci":
oci_models.append(key)
oci_models.add(key)
add_known_models()
@@ -769,68 +771,68 @@ ollama_models = ["llama2"]
maritalk_models = ["maritalk"]
model_list = (
model_list = list(
open_ai_chat_completion_models
+ open_ai_text_completion_models
+ cohere_models
+ cohere_chat_models
+ anthropic_models
+ replicate_models
+ openrouter_models
+ datarobot_models
+ huggingface_models
+ vertex_chat_models
+ vertex_text_models
+ ai21_models
+ ai21_chat_models
+ together_ai_models
+ baseten_models
+ aleph_alpha_models
+ nlp_cloud_models
+ ollama_models
+ bedrock_models
+ deepinfra_models
+ perplexity_models
+ maritalk_models
+ vertex_language_models
+ watsonx_models
+ gemini_models
+ text_completion_codestral_models
+ xai_models
+ deepseek_models
+ azure_ai_models
+ voyage_models
+ infinity_models
+ databricks_models
+ cloudflare_models
+ codestral_models
+ friendliai_models
+ palm_models
+ groq_models
+ azure_models
+ anyscale_models
+ cerebras_models
+ galadriel_models
+ sambanova_models
+ azure_text_models
+ novita_models
+ assemblyai_models
+ jina_ai_models
+ snowflake_models
+ gradient_ai_models
+ llama_models
+ featherless_ai_models
+ nscale_models
+ deepgram_models
+ elevenlabs_models
+ dashscope_models
+ moonshot_models
+ v0_models
+ morph_models
+ lambda_ai_models
+ recraft_models
+ cometapi_models
+ oci_models
| open_ai_text_completion_models
| cohere_models
| cohere_chat_models
| anthropic_models
| set(replicate_models)
| openrouter_models
| datarobot_models
| set(huggingface_models)
| vertex_chat_models
| vertex_text_models
| ai21_models
| ai21_chat_models
| set(together_ai_models)
| set(baseten_models)
| aleph_alpha_models
| nlp_cloud_models
| set(ollama_models)
| bedrock_models
| deepinfra_models
| perplexity_models
| set(maritalk_models)
| vertex_language_models
| watsonx_models
| gemini_models
| text_completion_codestral_models
| xai_models
| deepseek_models
| azure_ai_models
| voyage_models
| infinity_models
| databricks_models
| cloudflare_models
| codestral_models
| friendliai_models
| palm_models
| groq_models
| azure_models
| anyscale_models
| cerebras_models
| galadriel_models
| sambanova_models
| azure_text_models
| novita_models
| assemblyai_models
| jina_ai_models
| snowflake_models
| gradient_ai_models
| llama_models
| featherless_ai_models
| nscale_models
| deepgram_models
| elevenlabs_models
| dashscope_models
| moonshot_models
| v0_models
| morph_models
| lambda_ai_models
| recraft_models
| cometapi_models
| oci_models
)
model_list_set = set(model_list)
@@ -839,9 +841,9 @@ provider_list: List[Union[LlmProviders, str]] = list(LlmProviders)
models_by_provider: dict = {
"openai": open_ai_chat_completion_models + open_ai_text_completion_models,
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
"text-completion-openai": open_ai_text_completion_models,
"cohere": cohere_models + cohere_chat_models,
"cohere": cohere_models | cohere_chat_models,
"cohere_chat": cohere_chat_models,
"anthropic": anthropic_models,
"replicate": replicate_models,
@@ -850,14 +852,9 @@ models_by_provider: dict = {
"baseten": baseten_models,
"openrouter": openrouter_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
+ vertex_text_models
+ vertex_anthropic_models
+ vertex_vision_models
+ vertex_language_models
+ vertex_deepseek_models,
"vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
"ai21": ai21_models,
"bedrock": bedrock_models + bedrock_converse_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
"ollama_chat": ollama_models,
@@ -866,7 +863,7 @@ models_by_provider: dict = {
"maritalk": maritalk_models,
"watsonx": watsonx_models,
"gemini": gemini_models,
"fireworks_ai": fireworks_ai_models + fireworks_ai_embedding_models,
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
"aleph_alpha": aleph_alpha_models,
"text-completion-codestral": text_completion_codestral_models,
"xai": xai_models,
@@ -882,14 +879,14 @@ models_by_provider: dict = {
"friendliai": friendliai_models,
"palm": palm_models,
"groq": groq_models,
"azure": azure_models + azure_text_models,
"azure": azure_models | azure_text_models,
"azure_text": azure_text_models,
"anyscale": anyscale_models,
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"sambanova": sambanova_models + sambanova_embedding_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models + nebius_embedding_models,
"nebius": nebius_models | nebius_embedding_models,
"assemblyai": assemblyai_models,
"jina_ai": jina_ai_models,
"snowflake": snowflake_models,
@@ -936,12 +933,12 @@ longer_context_model_fallback_dict: dict = {
all_embedding_models = (
open_ai_embedding_models
+ cohere_embedding_models
+ bedrock_embedding_models
+ vertex_embedding_models
+ fireworks_ai_embedding_models
+ nebius_embedding_models
+ sambanova_embedding_models
| set(cohere_embedding_models)
| set(bedrock_embedding_models)
| vertex_embedding_models
| fireworks_ai_embedding_models
| nebius_embedding_models
| sambanova_embedding_models
)
####### IMAGE GENERATION MODELS ###################
@@ -1039,6 +1036,7 @@ from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config
from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig
from .llms.infinity.rerank.transformation import InfinityRerankConfig
from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig
from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.meta_llama.chat.transformation import LlamaAPIConfig
@@ -1150,6 +1148,7 @@ from .llms.topaz.image_variations.transformation import TopazImageVariationConfi
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig
from .llms.groq.chat.transformation import GroqChatConfig
from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig
from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig
from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig
from .llms.azure_ai.chat.transformation import AzureAIStudioConfig
from .llms.mistral.chat.transformation import MistralConfig
@@ -1193,6 +1192,7 @@ nvidiaNimEmbeddingConfig = NvidiaNimEmbeddingConfig()
from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig
from .llms.cerebras.chat import CerebrasConfig
from .llms.baseten.chat import BasetenConfig
from .llms.sambanova.chat import SambanovaConfig
from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig
from .llms.ai21.chat.transformation import AI21ChatConfig
litellm/caching/caching.py
+1 -3
View File
@@ -774,11 +774,9 @@ class Cache:
"""
Internal method to check if the cache type supports async get/set operations
Only S3 Cache Does NOT support async operations
All cache types now support async operations
"""
if self.type and self.type == LiteLLMCacheType.S3:
return False
return True
litellm/caching/caching_handler.py
+1 -7
View File
@@ -599,7 +599,7 @@ class LLMCachingHandler:
cached_result = await litellm.cache.async_get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
)
else: # for s3 caching. [NOT RECOMMENDED IN PROD - this will slow down responses since boto3 is sync]
else: # fallback for caches that don't support async
cached_result = litellm.cache.get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
)
@@ -806,12 +806,6 @@ class LLMCachingHandler:
result, dynamic_cache_object=self.dual_cache, **new_kwargs
)
)
elif isinstance(litellm.cache.cache, S3Cache):
threading.Thread(
target=litellm.cache.add_cache,
args=(result,),
kwargs=new_kwargs,
).start()
else:
asyncio.create_task(
litellm.cache.async_add_cache(
litellm/caching/s3_cache.py
+36 -10
View File
@@ -1,17 +1,17 @@
"""
S3 Cache implementation
WARNING: DO NOT USE THIS IN PRODUCTION - This is not ASYNC
Has 4 methods:
- set_cache
- get_cache
- async_set_cache
- async_get_cache
- async_set_cache (uses run_in_executor)
- async_get_cache (uses run_in_executor)
"""
import ast
import asyncio
import json
from functools import partial
from typing import Optional
from litellm._logging import print_verbose, verbose_logger
@@ -55,20 +55,24 @@ class S3Cache(BaseCache):
**kwargs,
)
def _to_s3_key(self, key: str) -> str:
"""Convert cache key to S3 key"""
return self.key_prefix + key.replace(":", "/")
def set_cache(self, key, value, **kwargs):
try:
print_verbose(f"LiteLLM SET Cache - S3. Key={key}. Value={value}")
ttl = kwargs.get("ttl", None)
# Convert value to JSON before storing in S3
serialized_value = json.dumps(value)
key = self.key_prefix + key
key = self._to_s3_key(key)
if ttl is not None:
cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}"
import datetime
# Calculate expiration time
expiration_time = datetime.datetime.now() + ttl
expiration_time = datetime.datetime.now() + datetime.timedelta(seconds=ttl)
# Upload the data to S3 with the calculated expiration time
self.s3_client.put_object(
@@ -94,17 +98,26 @@ class S3Cache(BaseCache):
ContentDisposition=f'inline; filename="{key}.json"',
)
except Exception as e:
# NON blocking - notify users S3 is throwing an exception
print_verbose(f"S3 Caching: set_cache() - Got exception from S3: {e}")
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
"""
Asynchronously set cache using run_in_executor to avoid blocking the event loop.
Compatible with Python 3.8+.
"""
try:
verbose_logger.debug(f"Set ASYNC S3 Cache: Key={key}. Value={value}")
loop = asyncio.get_event_loop()
func = partial(self.set_cache, key, value, **kwargs)
await loop.run_in_executor(None, func)
except Exception as e:
verbose_logger.error(f"S3 Caching: async_set_cache() - Got exception from S3: {e}")
def get_cache(self, key, **kwargs):
import botocore
try:
key = self.key_prefix + key
key = self._to_s3_key(key)
print_verbose(f"Get S3 Cache: key: {key}")
# Download the data from S3
@@ -138,13 +151,26 @@ class S3Cache(BaseCache):
return None
except Exception as e:
# NON blocking - notify users S3 is throwing an exception
verbose_logger.error(
f"S3 Caching: get_cache() - Got exception from S3: {e}"
)
async def async_get_cache(self, key, **kwargs):
return self.get_cache(key=key, **kwargs)
"""
Asynchronously get cache using run_in_executor to avoid blocking the event loop.
Compatible with Python 3.8+.
"""
try:
verbose_logger.debug(f"Get ASYNC S3 Cache: key: {key}")
loop = asyncio.get_event_loop()
func = partial(self.get_cache, key, **kwargs)
result = await loop.run_in_executor(None, func)
return result
except Exception as e:
verbose_logger.error(
f"S3 Caching: async_get_cache() - Got exception from S3: {e}"
)
return None
def flush_cache(self):
pass
litellm/constants.py
+31 -25
View File
@@ -1,6 +1,9 @@
import os
from typing import List, Literal
AZURE_DEFAULT_RESPONSES_API_VERSION = str(
os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")
)
ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
@@ -248,6 +251,7 @@ LITELLM_CHAT_PROVIDERS = [
"groq",
"nvidia_nim",
"cerebras",
"baseten",
"ai21_chat",
"volcengine",
"codestral",
@@ -424,6 +428,7 @@ openai_compatible_providers: List = [
"groq",
"nvidia_nim",
"cerebras",
"baseten",
"sambanova",
"ai21_chat",
"ai21",
@@ -480,7 +485,7 @@ _openai_like_providers: List = [
"watsonx",
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
# well supported replicate llms
replicate_models: List = [
replicate_models: set = set([
# llama replicate supported LLMs
"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
@@ -493,9 +498,9 @@ replicate_models: List = [
# Others
"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
]
])
clarifai_models: List = [
clarifai_models: set = set([
"clarifai/meta.Llama-3.Llama-3-8B-Instruct",
"clarifai/gcp.generate.gemma-1_1-7b-it",
"clarifai/mistralai.completion.mixtral-8x22B",
@@ -559,10 +564,10 @@ clarifai_models: List = [
"clarifai/gcp.generate.gemini-1_5-pro",
"clarifai/gcp.generate.imagen-2",
"clarifai/salesforce.blip.general-english-image-caption-blip-2",
]
])
huggingface_models: List = [
huggingface_models: set = set([
"meta-llama/Llama-2-7b-hf",
"meta-llama/Llama-2-7b-chat-hf",
"meta-llama/Llama-2-13b-hf",
@@ -575,13 +580,13 @@ huggingface_models: List = [
"meta-llama/Llama-2-13b-chat",
"meta-llama/Llama-2-70b",
"meta-llama/Llama-2-70b-chat",
] # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = [
]) # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = set([
"empower/empower-functions",
"empower/empower-functions-small",
]
])
together_ai_models: List = [
together_ai_models: set = set([
# llama llms - chat
"togethercomputer/llama-2-70b-chat",
# llama llms - language / instruct
@@ -609,16 +614,17 @@ together_ai_models: List = [
"Austism/chronos-hermes-13b",
"upstage/SOLAR-0-70b-16bit",
"WizardLM/WizardLM-70B-V1.0",
] # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
])
# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
baseten_models: List = [
baseten_models: set = set([
"qvv0xeq",
"q841o8w",
"31dxrj3",
] # FALCON 7B # WizardLM # Mosaic ML
]) # FALCON 7B # WizardLM # Mosaic ML
featherless_ai_models: List = [
featherless_ai_models: set = set([
"featherless-ai/Qwerky-72B",
"featherless-ai/Qwerky-QwQ-32B",
"Qwen/Qwen2.5-72B-Instruct",
@@ -628,9 +634,9 @@ featherless_ai_models: List = [
"mistralai/Mistral-Small-24B-Instruct-2501",
"mistralai/Mistral-Nemo-Instruct-2407",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
]
])
nebius_models: List = [
nebius_models: set = set([
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-30B-A3B-fast",
"Qwen/Qwen3-32B",
@@ -643,9 +649,9 @@ nebius_models: List = [
"meta-llama/Llama-3.3-70B-Instruct-fast",
"Qwen/Qwen2.5-32B-Instruct-fast",
"Qwen/Qwen2.5-Coder-32B-Instruct-fast",
]
])
dashscope_models: List = [
dashscope_models: set = set([
"qwen-turbo",
"qwen-plus",
"qwen-max",
@@ -656,13 +662,13 @@ dashscope_models: List = [
"qwen3-235b-a22b",
"qwen3-32b",
"qwen3-30b-a3b",
]
])
nebius_embedding_models: List = [
nebius_embedding_models: set = set([
"BAAI/bge-en-icl",
"BAAI/bge-multilingual-gemma2",
"intfloat/e5-mistral-7b-instruct",
]
])
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
@@ -676,8 +682,8 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"deepseek_r1",
]
open_ai_embedding_models: List = ["text-embedding-ada-002"]
cohere_embedding_models: List = [
open_ai_embedding_models: set = set(["text-embedding-ada-002"])
cohere_embedding_models: set = set([
"embed-v4.0",
"embed-english-v3.0",
"embed-english-light-v3.0",
@@ -685,12 +691,12 @@ cohere_embedding_models: List = [
"embed-english-v2.0",
"embed-english-light-v2.0",
"embed-multilingual-v2.0",
]
bedrock_embedding_models: List = [
])
bedrock_embedding_models: set = set([
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
]
])
known_tokenizer_config = {
"mistralai/Mistral-7B-Instruct-v0.1": {
litellm/images/main.py
+1 -1
View File
@@ -369,6 +369,7 @@ def image_generation( # noqa: PLR0915
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider == LlmProviders.LITELLM_PROXY.value
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
@@ -444,7 +445,6 @@ def image_generation( # noqa: PLR0915
elif custom_llm_provider in (
litellm.LlmProviders.RECRAFT,
litellm.LlmProviders.GEMINI,
):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
litellm/integrations/braintrust_logging.py
+11 -11
View File
@@ -19,10 +19,6 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.utils import print_verbose
global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
global_braintrust_sync_http_handler = HTTPHandler()
API_BASE = "https://api.braintrustdata.com/v1"
@@ -52,6 +48,10 @@ class BraintrustLogger(CustomLogger):
self._project_id_cache: Dict[
str, str
] = {} # Cache mapping project names to IDs
self.global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.global_braintrust_sync_http_handler = HTTPHandler()
def validate_environment(self, api_key: Optional[str]):
"""
@@ -76,7 +76,7 @@ class BraintrustLogger(CustomLogger):
return self._project_id_cache[project_name]
try:
response = global_braintrust_sync_http_handler.post(
response = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project",
headers=self.headers,
json={"name": project_name},
@@ -96,7 +96,7 @@ class BraintrustLogger(CustomLogger):
return self._project_id_cache[project_name]
try:
response = await global_braintrust_http_handler.post(
response = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project/register",
headers=self.headers,
json={"name": project_name},
@@ -146,7 +146,7 @@ class BraintrustLogger(CustomLogger):
return metadata
async def create_default_project_and_experiment(self):
project = await global_braintrust_http_handler.post(
project = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
@@ -155,7 +155,7 @@ class BraintrustLogger(CustomLogger):
self.default_project_id = project_dict["id"]
def create_sync_default_project_and_experiment(self):
project = global_braintrust_sync_http_handler.post(
project = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
@@ -291,9 +291,9 @@ class BraintrustLogger(CustomLogger):
try:
print_verbose(
f"global_braintrust_sync_http_handler.post: {global_braintrust_sync_http_handler.post}"
f"self.global_braintrust_sync_http_handler.post: {self.global_braintrust_sync_http_handler.post}"
)
global_braintrust_sync_http_handler.post(
self.global_braintrust_sync_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
json={"events": [request_data]},
headers=self.headers,
@@ -446,7 +446,7 @@ class BraintrustLogger(CustomLogger):
request_data["metrics"] = metrics
try:
await global_braintrust_http_handler.post(
await self.global_braintrust_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
json={"events": [request_data]},
headers=self.headers,
litellm/integrations/datadog/datadog_llm_obs.py
+97 -4
View File
@@ -27,7 +27,12 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.types.integrations.datadog_llm_obs import *
from litellm.types.utils import CallTypes, StandardLoggingPayload
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
@@ -102,6 +107,24 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
verbose_logger.exception(
f"DataDogLLMObs: Error logging success event - {str(e)}"
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:
verbose_logger.debug(
f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}"
)
payload = self.create_llm_obs_payload(
kwargs, start_time, end_time
)
verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
self.log_queue.append(payload)
if len(self.log_queue) >= self.batch_size:
await self.async_send_batch()
except Exception as e:
verbose_logger.exception(
f"DataDogLLMObs: Error logging failure event - {str(e)}"
)
async def async_send_batch(self):
try:
@@ -174,11 +197,14 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
call_type=standard_logging_payload.get("call_type")
))
error_info = self._assemble_error_info(standard_logging_payload)
meta = Meta(
kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")),
input=input_meta,
output=output_meta,
metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload),
error=error_info,
)
# Calculate metrics (you may need to adjust these based on available data)
@@ -199,11 +225,31 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
start_ns=int(start_time.timestamp() * 1e9),
duration=int((end_time - start_time).total_seconds() * 1e9),
metrics=metrics,
status="error" if error_info else "ok",
tags=[
self._get_datadog_tags(standard_logging_object=standard_logging_payload)
],
)
def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
"""
Assemble error information for failure cases according to DD LLM Obs API spec
"""
# Handle error information for failure cases according to DD LLM Obs API spec
error_info: Optional[DDLLMObsError] = None
if standard_logging_payload.get("status") == "failure":
# Try to get structured error information first
error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information")
if error_information:
error_info = DDLLMObsError(
message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error",
type=error_information.get("error_class"),
stack=error_information.get("traceback")
)
return error_info
def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
"""
Get the time to first token in seconds
@@ -232,8 +278,20 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
for now this handles logging /chat/completions responses
"""
if response_obj is None:
return []
if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]:
return [response_obj["choices"][0]["message"]]
try:
# Safely extract message from response_obj, handle failure cases
if isinstance(response_obj, dict) and "choices" in response_obj:
choices = response_obj["choices"]
if choices and len(choices) > 0 and "message" in choices[0]:
return [choices[0]["message"]]
return []
except (KeyError, IndexError, TypeError):
# In case of any error accessing the response structure, return empty list
return []
return []
def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
@@ -350,11 +408,11 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
def _get_dd_llm_obs_payload_metadata(
self, standard_logging_payload: StandardLoggingPayload
) -> Dict:
) -> Dict[str, Any]:
"""
Fields to track in DD LLM Observability metadata from litellm standard logging payload
"""
_metadata = {
_metadata: Dict[str, Any] = {
"model_name": standard_logging_payload.get("model", "unknown"),
"model_provider": standard_logging_payload.get(
"custom_llm_provider", "unknown"
@@ -364,9 +422,44 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
"cache_hit": standard_logging_payload.get("cache_hit", "unknown"),
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
"guardrail_information": standard_logging_payload.get("guardrail_information", None),
}
#########################################################
# Add latency metrics to metadata
#########################################################
latency_metrics = self._get_latency_metrics(standard_logging_payload)
_metadata.update({"latency_metrics": dict(latency_metrics)})
_standard_logging_metadata: dict = (
dict(standard_logging_payload.get("metadata", {})) or {}
)
_metadata.update(_standard_logging_metadata)
return _metadata
def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics:
"""
Get the latency metrics from the standard logging payload
"""
latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics()
# Add latency metrics to metadata
# Time to first token (convert from seconds to milliseconds for consistency)
time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload)
if time_to_first_token_seconds > 0:
latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000
# LiteLLM overhead time
hidden_params = standard_logging_payload.get("hidden_params", {})
litellm_overhead_ms = hidden_params.get("litellm_overhead_time_ms")
if litellm_overhead_ms is not None:
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information")
if guardrail_info is not None:
_guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration")
if _guardrail_duration_seconds is not None:
# Convert from seconds to milliseconds for consistency
latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000
return latency_metrics
litellm/integrations/mlflow.py
+6 -5
View File
@@ -60,10 +60,7 @@ class MlflowLogger(CustomLogger):
inputs = self._construct_input(kwargs)
input_messages = inputs.get("messages", [])
output_messages = [
c.message.model_dump(exclude_none=True)
for c in getattr(response_obj, "choices", [])
]
output_messages = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])]
if messages := [*input_messages, *output_messages]:
set_span_chat_messages(span, messages)
if tools := inputs.get("tools"):
@@ -168,6 +165,10 @@ class MlflowLogger(CustomLogger):
for key in ["functions", "tools", "stream", "tool_choice", "user"]:
if value := kwargs.get("optional_params", {}).pop(key, None):
inputs[key] = value
if prediction := kwargs.get("prediction"):
inputs["prediction"] = prediction
return inputs
def _extract_attributes(self, kwargs):
@@ -232,7 +233,6 @@ class MlflowLogger(CustomLogger):
"""
import mlflow
call_type = kwargs.get("call_type", "completion")
span_name = f"litellm-{call_type}"
span_type = self._get_span_type(call_type)
@@ -260,6 +260,7 @@ class MlflowLogger(CustomLogger):
tags=self._transform_tag_list_to_dict(attributes.get("request_tags", [])),
start_time_ns=start_time_ns,
)
def _transform_tag_list_to_dict(self, tag_list: list) -> dict:
return {tag: "" for tag in tag_list}
litellm/litellm_core_utils/get_llm_provider_logic.py
+10
View File
@@ -196,6 +196,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.cerebras.ai/v1":
custom_llm_provider = "cerebras"
dynamic_api_key = get_secret_str("CEREBRAS_API_KEY")
elif endpoint == "https://inference.baseten.co/v1":
custom_llm_provider = "baseten"
dynamic_api_key = get_secret_str("BASETEN_API_KEY")
elif endpoint == "https://api.sambanova.ai/v1":
custom_llm_provider = "sambanova"
dynamic_api_key = get_secret_str("SAMBANOVA_API_KEY")
@@ -478,6 +481,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("CEREBRAS_API_KEY")
elif custom_llm_provider == "baseten":
# Use BasetenConfig to determine the appropriate API base URL
if api_base is None:
api_base = litellm.BasetenConfig.get_api_base_for_model(model)
else:
api_base = api_base or get_secret("BASETEN_API_BASE") or "https://inference.baseten.co/v1"
dynamic_api_key = api_key or get_secret_str("BASETEN_API_KEY")
elif custom_llm_provider == "sambanova":
api_base = (
api_base
litellm/litellm_core_utils/get_supported_openai_params.py
+2
View File
@@ -78,6 +78,8 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.nvidiaNimEmbeddingConfig.get_supported_openai_params()
elif custom_llm_provider == "cerebras":
return litellm.CerebrasConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "baseten":
return litellm.BasetenConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "xai":
return litellm.XAIChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "ai21_chat" or custom_llm_provider == "ai21":
litellm/litellm_core_utils/litellm_logging.py
+1 -10
View File
@@ -4106,18 +4106,9 @@ class StandardLoggingPayloadSetup:
"""
# Generate object key in same format as S3Logger
from litellm.integrations.s3 import get_s3_object_key
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
# Only generate object key if cold storage is configured
try:
configured_cold_storage_logger = (
ColdStorageHandler._get_configured_cold_storage_custom_logger()
)
except Exception as e:
verbose_logger.debug(f"Cold storage custom logger unavailable: {e}")
return None
if configured_cold_storage_logger is None:
if litellm.configured_cold_storage_logger is None:
return None
try:
litellm/litellm_core_utils/llm_cost_calc/utils.py
+29 -15
View File
@@ -113,15 +113,20 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]:
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]:
"""
Return prompt cost for a given model and usage.
Return prompt cost, completion cost, and cache costs for a given model and usage.
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
then we use the corresponding threshold cost.
then we use the corresponding threshold cost for all token types.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token"))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token"))
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost"))
cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost"))
## CHECK IF ABOVE THRESHOLD
threshold: Optional[float] = None
@@ -141,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl
f"output_cost_per_token_above_{threshold_str}_tokens",
completion_base_cost,
))
# Apply tiered pricing to cache costs
cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens"
if cache_creation_tiered_key in model_info:
cache_creation_cost = cast(float, _get_cost_per_unit(
model_info, cache_creation_tiered_key, cache_creation_cost
))
if cache_read_tiered_key in model_info:
cache_read_cost = cast(float, _get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
))
break
except (IndexError, ValueError):
continue
except Exception:
continue
return prompt_base_cost, completion_base_cost
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
def calculate_cost_component(
@@ -262,28 +282,22 @@ def generic_cost_per_token(
if text_tokens == 0:
text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens
prompt_base_cost, completion_base_cost = _get_token_base_cost(
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost(
model_info=model_info, usage=usage
)
prompt_cost = float(text_tokens) * prompt_base_cost
### CACHE READ COST
prompt_cost += calculate_cost_component(
model_info, "cache_read_input_token_cost", cache_hit_tokens
)
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST
prompt_cost += calculate_cost_component(
model_info,
"cache_creation_input_token_cost",
usage._cache_creation_input_tokens,
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost
### CHARACTER COST
litellm/litellm_core_utils/logging_utils.py
+116 -3
View File
@@ -1,5 +1,6 @@
import asyncio
import functools
import time
from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional, Union
@@ -11,15 +12,19 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm import ModelResponse as _ModelResponse
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObject,
)
LiteLLMModelResponse = _ModelResponse
Span = Union[_Span, Any]
else:
LiteLLMModelResponse = Any
LiteLLMLoggingObject = Any
Span = Any
import litellm
@@ -28,9 +33,52 @@ import litellm
Helper utils used for logging callbacks
"""
# Global service logger instance to avoid recreating it
_service_logger = None
def _get_service_logger():
"""Get or create the global ServiceLogging instance"""
global _service_logger
if _service_logger is None:
from litellm._service_logger import ServiceLogging
_service_logger = ServiceLogging()
return _service_logger
def _get_parent_otel_span_from_logging_obj(
logging_obj: Optional[LiteLLMLoggingObject] = None,
) -> Optional[Span]:
"""
Extract the parent OTEL span from the logging object using existing helper.
Args:
logging_obj: The LiteLLM logging object containing model call details
Returns:
The parent OTEL span if found, None otherwise
"""
try:
if logging_obj is None or not hasattr(logging_obj, "model_call_details"):
return None
# Reuse existing function by passing model_call_details as kwargs
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
return _get_parent_otel_span_from_kwargs(logging_obj.model_call_details)
except Exception as e:
verbose_logger.exception(
f"Error in _get_parent_otel_span_from_logging_obj: {str(e)}"
)
return None
def convert_litellm_response_object_to_str(
response_obj: Union[Any, LiteLLMModelResponse]
response_obj: Union[Any, LiteLLMModelResponse],
) -> Optional[str]:
"""
Get the string of the response object from LiteLLM
@@ -125,37 +173,102 @@ def track_llm_api_timing():
"""
Decorator to track LLM API call timing for both sync and async functions.
The logging_obj is expected to be passed as an argument to the decorated function.
Logs timing using ServiceLogging similar to Redis cache.
"""
def decorator(func):
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now()
start_time_float = time.time()
logging_obj = kwargs.get("logging_obj", None)
# Extract parent OTEL span from logging object
parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)
try:
result = await func(*args, **kwargs)
return result
finally:
end_time = datetime.now()
end_time_float = time.time()
duration = end_time_float - start_time_float
# Set duration in model call details
_set_duration_in_model_call_details(
logging_obj=kwargs.get("logging_obj", None),
logging_obj=logging_obj,
start_time=start_time,
end_time=end_time,
)
# Log timing using ServiceLogging (like Redis cache)
try:
from litellm.types.services import ServiceTypes
service_logger = _get_service_logger()
# Get function name for call_type
call_type = f"{func.__name__} <- track_llm_api_timing"
# Create async task for service logging (similar to Redis cache pattern)
asyncio.create_task(
service_logger.async_service_success_hook(
service=ServiceTypes.LITELLM,
duration=duration,
call_type=call_type,
start_time=start_time_float,
end_time=end_time_float,
parent_otel_span=parent_otel_span,
)
)
except Exception as e:
verbose_logger.debug(f"Error in service logging: {str(e)}")
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
start_time = datetime.now()
start_time_float = time.time()
logging_obj = kwargs.get("logging_obj", None)
# Extract parent OTEL span from logging object
parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)
try:
result = func(*args, **kwargs)
return result
finally:
end_time = datetime.now()
end_time_float = time.time()
duration = end_time_float - start_time_float
# Set duration in model call details
_set_duration_in_model_call_details(
logging_obj=kwargs.get("logging_obj", None),
logging_obj=logging_obj,
start_time=start_time,
end_time=end_time,
)
# Log timing using ServiceLogging (like Redis cache)
try:
from litellm.types.services import ServiceTypes
service_logger = _get_service_logger()
# Get function name for call_type
call_type = f"{func.__name__} <- track_llm_api_timing"
# Use sync service logging for sync functions
service_logger.service_success_hook(
service=ServiceTypes.LITELLM,
duration=duration,
call_type=call_type,
start_time=start_time_float,
end_time=end_time_float,
parent_otel_span=parent_otel_span,
)
except Exception as e:
verbose_logger.debug(f"Error in service logging: {str(e)}")
# Check if the function is async or sync
if asyncio.iscoroutinefunction(func):
return async_wrapper
litellm/litellm_core_utils/prompt_templates/common_utils.py
+5
View File
@@ -18,6 +18,7 @@ from typing import (
cast,
)
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@@ -453,6 +454,10 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
filename, file_content, content_type = file_data
elif len(file_data) == 4:
filename, file_content, content_type, file_headers = file_data
elif isinstance(file_data, InMemoryFile):
filename = file_data.name
file_content = file_data
content_type = file_data.content_type
else:
file_content = file_data
# Convert content to bytes
litellm/litellm_core_utils/prompt_templates/factory.py
+44 -7
View File
@@ -3193,9 +3193,30 @@ class BedrockConverseMessagesProcessor:
## MERGE CONSECUTIVE TOOL CALL MESSAGES ##
tool_content: List[BedrockContentBlock] = []
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
current_message = messages[msg_i]
tool_call_result = _convert_to_bedrock_tool_call_result(current_message)
tool_content.append(tool_call_result)
# Check if we need to add a separate cachePoint block
has_cache_control = False
# Check for message-level cache_control
if current_message.get("cache_control", None) is not None:
has_cache_control = True
# Check for content-level cache_control in list content
elif isinstance(current_message.get("content"), list):
for content_element in current_message["content"]:
if (isinstance(content_element, dict) and
content_element.get("cache_control", None) is not None):
has_cache_control = True
break
# Add a separate cachePoint block if cache_control is present
if has_cache_control:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
tool_content.append(cache_point_block)
msg_i += 1
if tool_content:
# if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
@@ -3275,13 +3296,29 @@ class BedrockConverseMessagesProcessor:
image_url=image_url
)
assistants_parts.append(assistants_part)
# Add cache point block for assistant content elements
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
OpenAIMessageContentListBlock, element
),
block_type="content_block",
)
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
assistant_content.extend(assistants_parts)
elif _assistant_content is not None and isinstance(
_assistant_content, str
):
assistant_content.append(
BedrockContentBlock(text=_assistant_content)
elif _assistant_content is not None and isinstance(_assistant_content, str):
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
)
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(
litellm/litellm_core_utils/sensitive_data_masker.py
+6 -1
View File
@@ -33,7 +33,12 @@ class SensitiveDataMasker:
value_str = str(value)
masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
# Handle the case where visible_suffix is 0 to avoid showing the entire string
if self.visible_suffix == 0:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}"
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str) -> bool:
key_lower = str(key).lower()
litellm/litellm_core_utils/streaming_handler.py
+6
View File
@@ -1747,6 +1747,11 @@ class CustomStreamWrapper:
if is_empty:
continue
print_verbose(f"final returned processed chunk: {processed_chunk}")
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
processed_chunk._hidden_params["usage"] = usage
return processed_chunk
raise StopAsyncIteration
else: # temporary patch for non-aiohttp async calls
@@ -1790,6 +1795,7 @@ class CustomStreamWrapper:
messages=self.messages,
logging_obj=self.logging_obj,
)
response = self.model_response_creator()
if complete_streaming_response is not None:
setattr(
litellm/llms/azure/chat/gpt_5_transformation.py
+2 -1
View File
@@ -10,6 +10,7 @@ from .gpt_transformation import AzureOpenAIConfig
class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
"""Azure specific handling for gpt-5 models."""
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
@@ -23,7 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
def get_supported_openai_params(self, model: str) -> List[str]:
return OpenAIGPT5Config.get_supported_openai_params(self, model=model)
def map_openai_params(
self,
non_default_params: dict,
litellm/llms/azure/common_utils.py
+43 -26
View File
@@ -365,14 +365,16 @@ def get_azure_ad_token(
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
except ValueError:
verbose_logger.debug("Azure AD Token Provider could not be used.")
#########################################################
# If litellm.enable_azure_ad_token_refresh is True and no other token provider is available,
# try to get DefaultAzureCredential provider
#########################################################
if azure_ad_token_provider is None and azure_ad_token is None:
azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider(
scope=scope,
azure_ad_token_provider = (
BaseAzureLLM._try_get_default_azure_credential_provider(
scope=scope,
)
)
# Execute the token provider to get the token if available
@@ -403,27 +405,27 @@ class BaseAzureLLM(BaseOpenAILLM):
) -> Optional[Callable[[], str]]:
"""
Try to get DefaultAzureCredential provider
Args:
scope: Azure scope for the token
Returns:
Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
"""
from litellm.types.secret_managers.get_azure_ad_token_provider import (
AzureCredentialType,
)
verbose_logger.debug(
"Attempting to use DefaultAzureCredential for Azure Auth"
)
verbose_logger.debug("Attempting to use DefaultAzureCredential for Azure Auth")
try:
azure_ad_token_provider = get_azure_ad_token_provider(
azure_scope=scope,
azure_credential=AzureCredentialType.DefaultAzureCredential,
)
verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
verbose_logger.debug(
"Successfully obtained Azure AD token provider using DefaultAzureCredential"
)
return azure_ad_token_provider
except Exception as e:
verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}")
@@ -656,17 +658,17 @@ class BaseAzureLLM(BaseOpenAILLM):
else:
client = AzureOpenAI(**azure_client_params) # type: ignore
return client
@staticmethod
def _base_validate_azure_environment(
headers: dict, litellm_params: Optional[GenericLiteLLMParams]
headers: dict, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
# If api-key is already in headers, preserve it
if "api-key" in headers:
return headers
api_key = (
litellm_params.api_key
or litellm.api_key
@@ -686,13 +688,24 @@ class BaseAzureLLM(BaseOpenAILLM):
headers["Authorization"] = f"Bearer {azure_ad_token}"
return headers
@staticmethod
def _get_base_azure_url(
api_base: Optional[str],
litellm_params: Optional[Union[GenericLiteLLMParams, Dict[str, Any]]],
route: Literal["/openai/responses", "/openai/vector_stores"]
route: Literal["/openai/responses", "/openai/vector_stores"],
default_api_version: Optional[Union[str, Literal["latest", "preview"]]] = None,
) -> str:
"""
Get the base Azure URL for the given route and API version.
Args:
api_base: The base URL of the Azure API.
litellm_params: The litellm parameters.
route: The route to the API.
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
@@ -702,7 +715,10 @@ class BaseAzureLLM(BaseOpenAILLM):
# Extract api_version or use default
litellm_params = litellm_params or {}
api_version = cast(Optional[str], litellm_params.get("api_version"))
api_version = (
cast(Optional[str], litellm_params.get("api_version"))
or default_api_version
)
# Create a new dictionary with existing params
query_params = dict(original_url.params)
@@ -710,27 +726,28 @@ class BaseAzureLLM(BaseOpenAILLM):
# Add api_version if needed
if "api-version" not in query_params and api_version:
query_params["api-version"] = api_version
# Add the path to the base URL
if route not in api_base:
new_url = _add_path_to_api_base(
api_base=api_base, ending_path=route
)
new_url = _add_path_to_api_base(api_base=api_base, ending_path=route)
else:
new_url = api_base
if BaseAzureLLM._is_azure_v1_api_version(api_version):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url = httpx.URL(new_url)
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
new_url = str(
parsed_url.copy_with(
path=parsed_url.path.replace("/openai", "/openai/v1")
)
)
# Use the new query_params dictionary
final_url = httpx.URL(new_url).copy_with(params=query_params)
return str(final_url)
@staticmethod
def _is_azure_v1_api_version(api_version: Optional[str]) -> bool:
if api_version is None:
litellm/llms/azure/responses/transformation.py
+11 -1
View File
@@ -6,6 +6,7 @@ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfi
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -16,6 +17,10 @@ else:
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.AZURE
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
@@ -70,8 +75,13 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
return BaseAzureLLM._get_base_azure_url(
api_base=api_base, litellm_params=litellm_params, route="/openai/responses"
api_base=api_base,
litellm_params=litellm_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
#########################################################
litellm/llms/base_llm/responses/transformation.py
+6
View File
@@ -12,6 +12,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -29,6 +30,11 @@ class BaseResponsesAPIConfig(ABC):
def __init__(self):
pass
@property
@abstractmethod
def custom_llm_provider(self) -> LlmProviders:
pass
@classmethod
def get_config(cls):
return {
litellm/llms/baseten.py
-172
View File
@@ -1,172 +0,0 @@
import json
import time
from typing import Callable
import litellm
from litellm.types.utils import ModelResponse, Usage
class BasetenError(Exception):
def __init__(self, status_code, message):
self.status_code = status_code
self.message = message
super().__init__(
self.message
) # Call the base class constructor with the parameters it needs
def validate_environment(api_key):
headers = {
"accept": "application/json",
"content-type": "application/json",
}
if api_key:
headers["Authorization"] = f"Api-Key {api_key}"
return headers
def completion(
model: str,
messages: list,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
api_key,
logging_obj,
optional_params: dict,
litellm_params=None,
logger_fn=None,
):
headers = validate_environment(api_key)
completion_url_fragment_1 = "https://app.baseten.co/models/"
completion_url_fragment_2 = "/predict"
model = model
prompt = ""
for message in messages:
if "role" in message:
if message["role"] == "user":
prompt += f"{message['content']}"
else:
prompt += f"{message['content']}"
else:
prompt += f"{message['content']}"
data = {
"inputs": prompt,
"prompt": prompt,
"parameters": optional_params,
"stream": (
True
if "stream" in optional_params and optional_params["stream"] is True
else False
),
}
## LOGGING
logging_obj.pre_call(
input=prompt,
api_key=api_key,
additional_args={"complete_input_dict": data},
)
## COMPLETION CALL
response = litellm.module_level_client.post(
completion_url_fragment_1 + model + completion_url_fragment_2,
headers=headers,
data=json.dumps(data),
stream=(
True
if "stream" in optional_params and optional_params["stream"] is True
else False
),
)
if "text/event-stream" in response.headers["Content-Type"] or (
"stream" in optional_params and optional_params["stream"] is True
):
return response.iter_lines()
else:
## LOGGING
logging_obj.post_call(
input=prompt,
api_key=api_key,
original_response=response.text,
additional_args={"complete_input_dict": data},
)
print_verbose(f"raw model_response: {response.text}")
## RESPONSE OBJECT
completion_response = response.json()
if "error" in completion_response:
raise BasetenError(
message=completion_response["error"],
status_code=response.status_code,
)
else:
if "model_output" in completion_response:
if (
isinstance(completion_response["model_output"], dict)
and "data" in completion_response["model_output"]
and isinstance(completion_response["model_output"]["data"], list)
):
model_response.choices[0].message.content = completion_response[ # type: ignore
"model_output"
][
"data"
][
0
]
elif isinstance(completion_response["model_output"], str):
model_response.choices[0].message.content = completion_response[ # type: ignore
"model_output"
]
elif "completion" in completion_response and isinstance(
completion_response["completion"], str
):
model_response.choices[0].message.content = completion_response[ # type: ignore
"completion"
]
elif isinstance(completion_response, list) and len(completion_response) > 0:
if "generated_text" not in completion_response:
raise BasetenError(
message=f"Unable to parse response. Original response: {response.text}",
status_code=response.status_code,
)
model_response.choices[0].message.content = completion_response[0][ # type: ignore
"generated_text"
]
## GETTING LOGPROBS
if (
"details" in completion_response[0]
and "tokens" in completion_response[0]["details"]
):
model_response.choices[0].finish_reason = completion_response[0][
"details"
]["finish_reason"]
sum_logprob = 0
for token in completion_response[0]["details"]["tokens"]:
sum_logprob += token["logprob"]
model_response.choices[0].logprobs = sum_logprob # type: ignore
else:
raise BasetenError(
message=f"Unable to parse response. Original response: {response.text}",
status_code=response.status_code,
)
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
prompt_tokens = len(encoding.encode(prompt))
completion_tokens = len(
encoding.encode(model_response["choices"][0]["message"]["content"])
)
model_response.created = int(time.time())
model_response.model = model
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
setattr(model_response, "usage", usage)
return model_response
def embedding():
# logic for parsing in - calling - parsing out model embedding calls
pass
litellm/llms/baseten/chat.py
+118
View File
@@ -0,0 +1,118 @@
from typing import Optional
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
class BasetenConfig(OpenAIGPTConfig):
"""
Reference: https://inference.baseten.co/v1
Below are the parameters:
"""
max_tokens: Optional[int] = None
response_format: Optional[dict] = None
seed: Optional[int] = None
stream: Optional[bool] = None
top_p: Optional[int] = None
tool_choice: Optional[str] = None
tools: Optional[list] = None
user: Optional[str] = None
presence_penalty: Optional[int] = None
frequency_penalty: Optional[int] = None
stream_options: Optional[dict] = None
def __init__(
self,
max_tokens: Optional[int] = None,
response_format: Optional[dict] = None,
seed: Optional[int] = None,
stop: Optional[list] = None,
stream: Optional[bool] = None,
temperature: Optional[float] = None,
top_p: Optional[int] = None,
tool_choice: Optional[str] = None,
tools: Optional[list] = None,
user: Optional[str] = None,
presence_penalty: Optional[int] = None,
frequency_penalty: Optional[int] = None,
stream_options: Optional[dict] = None,
) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@classmethod
def get_config(cls):
return super().get_config()
def get_supported_openai_params(self, model: str) -> list:
"""
Get the supported OpenAI params for the given model
"""
return [
"max_tokens",
"max_completion_tokens",
"response_format",
"seed",
"stop",
"stream",
"temperature",
"top_p",
"tool_choice",
"tools",
"user",
"presence_penalty",
"frequency_penalty",
"stream_options",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_openai_params = self.get_supported_openai_params(model=model)
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params
def _get_openai_compatible_provider_info(self, api_base: str, api_key: str) -> tuple:
"""
Get the OpenAI compatible provider info for Baseten
"""
# Default to Model API
default_api_base = "https://inference.baseten.co/v1"
default_api_key = api_key or "BASETEN_API_KEY"
return default_api_base, default_api_key
@staticmethod
def is_dedicated_deployment(model: str) -> bool:
"""
Check if the model is a dedicated deployment (8-digit alphanumeric code)
"""
# Remove 'baseten/' prefix if present
model_id = model.replace("baseten/", "")
# Check if it's an 8-digit alphanumeric code
import re
return bool(re.match(r'^[a-zA-Z0-9]{8}$', model_id))
@staticmethod
def get_api_base_for_model(model: str) -> str:
"""
Get the appropriate API base URL for the given model
"""
if BasetenConfig.is_dedicated_deployment(model):
# Extract the model ID (remove 'baseten/' prefix if present)
model_id = model.replace("baseten/", "")
return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1"
else:
# Use Model API
return "https://inference.baseten.co/v1"
litellm/llms/bedrock/base_aws_llm.py
+164 -15
View File
@@ -179,15 +179,32 @@ class BaseAWSLLM:
aws_sts_endpoint=aws_sts_endpoint,
)
elif aws_role_name is not None:
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
credentials, _cache_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
)
# Check if we're in IRSA and trying to assume the same role we already have
current_role_arn = os.getenv("AWS_ROLE_ARN")
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
# In IRSA environments, we should skip role assumption if we're already running as the target role
# This is true when:
# 1. We have AWS_ROLE_ARN set (current role)
# 2. We have AWS_WEB_IDENTITY_TOKEN_FILE set (IRSA environment)
# 3. The current role matches the requested role
if (current_role_arn and web_identity_token_file and
current_role_arn == aws_role_name):
verbose_logger.debug("Using IRSA same-role optimization: calling _auth_with_env_vars")
# We're already running as this role via IRSA, no need to assume it again
# Use the default boto3 credentials (which will use the IRSA credentials)
credentials, _cache_ttl = self._auth_with_env_vars()
else:
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
credentials, _cache_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
)
elif aws_profile_name is not None: ### CHECK SESSION ###
credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name)
@@ -446,6 +463,92 @@ class BaseAWSLLM:
iam_creds = session.get_credentials()
return iam_creds, self._get_default_ttl_for_boto3_credentials()
def _handle_irsa_cross_account(self, irsa_role_arn: str, aws_role_name: str,
aws_session_name: str, region: str, web_identity_token_file: str) -> dict:
"""Handle cross-account role assumption for IRSA."""
import boto3
verbose_logger.debug("Cross-account role assumption detected")
# Read the web identity token
with open(web_identity_token_file, 'r') as f:
web_identity_token = f.read().strip()
# Create an STS client without credentials
with tracer.trace("boto3.client(sts) for manual IRSA"):
sts_client = boto3.client('sts', region_name=region)
# Manually assume the IRSA role with the session name
verbose_logger.debug(f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}")
irsa_response = sts_client.assume_role_with_web_identity(
RoleArn=irsa_role_arn,
RoleSessionName=aws_session_name,
WebIdentityToken=web_identity_token
)
# Extract the credentials from the IRSA assumption
irsa_creds = irsa_response["Credentials"]
# Create a new STS client with the IRSA credentials
with tracer.trace("boto3.client(sts) with manual IRSA credentials"):
sts_client_with_creds = boto3.client(
'sts',
region_name=region,
aws_access_key_id=irsa_creds["AccessKeyId"],
aws_secret_access_key=irsa_creds["SecretAccessKey"],
aws_session_token=irsa_creds["SessionToken"]
)
# Get current caller identity for debugging
try:
caller_identity = sts_client_with_creds.get_caller_identity()
verbose_logger.debug(f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}")
except Exception as e:
verbose_logger.debug(f"Failed to get caller identity: {e}")
# Now assume the target role
verbose_logger.debug(f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}")
return sts_client_with_creds.assume_role(
RoleArn=aws_role_name, RoleSessionName=aws_session_name
)
def _handle_irsa_same_account(self, aws_role_name: str, aws_session_name: str, region: str) -> dict:
"""Handle same-account role assumption for IRSA."""
import boto3
verbose_logger.debug("Same account role assumption, using automatic IRSA")
with tracer.trace("boto3.client(sts) with automatic IRSA"):
sts_client = boto3.client("sts", region_name=region)
# Get current caller identity for debugging
try:
caller_identity = sts_client.get_caller_identity()
verbose_logger.debug(f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}")
except Exception as e:
verbose_logger.debug(f"Failed to get caller identity: {e}")
# Assume the role
verbose_logger.debug(f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}")
return sts_client.assume_role(
RoleArn=aws_role_name, RoleSessionName=aws_session_name
)
def _extract_credentials_and_ttl(self, sts_response: dict) -> Tuple[Credentials, Optional[int]]:
"""Extract credentials and TTL from STS response."""
from botocore.credentials import Credentials
sts_credentials = sts_response["Credentials"]
credentials = Credentials(
access_key=sts_credentials["AccessKeyId"],
secret_key=sts_credentials["SecretAccessKey"],
token=sts_credentials["SessionToken"],
)
expiration_time = sts_credentials["Expiration"]
ttl = int((expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds())
return credentials, ttl
@tracer.wrap()
def _auth_with_aws_role(
self,
@@ -460,12 +563,58 @@ class BaseAWSLLM:
import boto3
from botocore.credentials import Credentials
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client(
"sts",
aws_access_key_id=aws_access_key_id, # [OPTIONAL]
aws_secret_access_key=aws_secret_access_key, # [OPTIONAL]
)
# Check if we're in an EKS/IRSA environment
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
irsa_role_arn = os.getenv("AWS_ROLE_ARN")
# If we have IRSA environment variables and no explicit credentials,
# we need to use the web identity token flow
if (web_identity_token_file and irsa_role_arn and
aws_access_key_id is None and aws_secret_access_key is None):
# For cross-account role assumption with specific session names,
# we need to manually assume the IRSA role first with the correct session name
verbose_logger.debug(f"IRSA detected: using web identity token from {web_identity_token_file}")
try:
# Get region from environment
region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1"
# Check if we need to do cross-account role assumption
if aws_role_name != irsa_role_arn:
sts_response = self._handle_irsa_cross_account(
irsa_role_arn, aws_role_name, aws_session_name, region, web_identity_token_file
)
else:
sts_response = self._handle_irsa_same_account(
aws_role_name, aws_session_name, region
)
return self._extract_credentials_and_ttl(sts_response)
except Exception as e:
verbose_logger.debug(f"Failed to assume role via IRSA: {e}")
if "AccessDenied" in str(e) and "is not authorized to perform: sts:AssumeRole" in str(e):
# Provide a more helpful error message for trust policy issues
verbose_logger.error(
f"Access denied when trying to assume role {aws_role_name}. "
f"Please ensure the trust policy of {aws_role_name} allows "
f"the current role to assume it. Current identity: check logs with verbose mode."
)
# Re-raise the exception instead of falling through
raise
# In EKS/IRSA environments, use ambient credentials (no explicit keys needed)
# This allows the web identity token to work automatically
if aws_access_key_id is None and aws_secret_access_key is None:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client("sts")
else:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client(
"sts",
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
)
sts_response = sts_client.assume_role(
RoleArn=aws_role_name, RoleSessionName=aws_session_name
litellm/llms/custom_httpx/aiohttp_transport.py
+25 -3
View File
@@ -3,7 +3,7 @@ import contextlib
import os
import typing
import urllib.request
from typing import Callable, Dict, Union
from typing import Callable, Dict, Optional, Union
import aiohttp
import aiohttp.client_exceptions
@@ -115,6 +115,12 @@ class AiohttpTransport(httpx.AsyncBaseTransport):
) -> None:
self.client = client
#########################################################
# Class variables for proxy settings
#########################################################
self.proxy: Optional[str] = None
self.checked_proxy_env_settings: bool = False
async def aclose(self) -> None:
if isinstance(self.client, ClientSession):
await self.client.close()
@@ -249,7 +255,22 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
def _proxy_from_env(self, url: httpx.URL) -> typing.Optional[str]:
"""Return proxy URL from env for the given request URL."""
"""
Return proxy URL from env for the given request URL
Only check the proxy env settings once, this is a costly operation for CPU % usage
."""
#########################################################
# Check if we've already checked the proxy env settings
#########################################################
if self.checked_proxy_env_settings is True:
return self.proxy
#########################################################
# set self.checked_proxy_env_settings to True
#########################################################
self.checked_proxy_env_settings = True
proxies = urllib.request.getproxies()
if urllib.request.proxy_bypass(url.host):
return None
@@ -257,4 +278,5 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
proxy = proxies.get(url.scheme) or proxies.get("all")
if proxy and "://" not in proxy:
proxy = f"http://{proxy}"
return proxy
self.proxy = proxy
return self.proxy
litellm/llms/custom_httpx/http_handler.py
+13 -13
View File
@@ -40,7 +40,9 @@ headers = {
_DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0)
def get_ssl_configuration(ssl_verify: Optional[VerifyTypes] = None) -> Union[bool, str, ssl.SSLContext]:
def get_ssl_configuration(
ssl_verify: Optional[VerifyTypes] = None,
) -> Union[bool, str, ssl.SSLContext]:
"""
Unified SSL configuration function that handles ssl_context and ssl_verify logic.
@@ -59,7 +61,7 @@ def get_ssl_configuration(ssl_verify: Optional[VerifyTypes] = None) -> Union[boo
- False: Disable SSL verification
- True: Enable SSL verification
- str: Path to CA bundle file
Returns:
Union[bool, str, ssl.SSLContext]: Appropriate SSL configuration
"""
@@ -72,7 +74,9 @@ def get_ssl_configuration(ssl_verify: Optional[VerifyTypes] = None) -> Union[boo
# Get ssl_verify from environment or litellm settings if not provided
if ssl_verify is None:
ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify)
ssl_verify_bool = str_to_bool(ssl_verify) if isinstance(ssl_verify, str) else ssl_verify
ssl_verify_bool = (
str_to_bool(ssl_verify) if isinstance(ssl_verify, str) else ssl_verify
)
if ssl_verify_bool is not None:
ssl_verify = ssl_verify_bool
@@ -89,14 +93,9 @@ def get_ssl_configuration(ssl_verify: Optional[VerifyTypes] = None) -> Union[boo
cafile = certifi.where()
if ssl_verify is not False:
custom_ssl_context = ssl.create_default_context(
cafile=cafile
)
custom_ssl_context = ssl.create_default_context(cafile=cafile)
# If security level is set, apply it to the SSL context
if (
ssl_security_level
and isinstance(ssl_security_level, str)
):
if ssl_security_level and isinstance(ssl_security_level, str):
# Create a custom SSL context with reduced security level
custom_ssl_context.set_ciphers(ssl_security_level)
@@ -260,6 +259,7 @@ class AsyncHTTPHandler:
files: Optional[RequestFiles] = None,
content: Any = None,
):
start_time = time.time()
try:
if timeout is None:
@@ -586,7 +586,7 @@ class AsyncHTTPHandler:
) -> Dict[str, Any]:
"""
Helper method to get SSL connector initialization arguments for aiohttp TCPConnector.
SSL Configuration Priority:
1. If ssl_context is provided -> use the custom SSL context
2. If ssl_verify is False -> disable SSL verification (ssl=False)
@@ -597,14 +597,14 @@ class AsyncHTTPHandler:
connector_kwargs: Dict[str, Any] = {
"local_addr": ("0.0.0.0", 0) if litellm.force_ipv4 else None,
}
if ssl_context is not None:
# Priority 1: Use the provided custom SSL context
connector_kwargs["ssl"] = ssl_context
elif ssl_verify is False:
# Priority 2: Explicitly disable SSL verification
connector_kwargs["verify_ssl"] = False
return connector_kwargs
@staticmethod
litellm/llms/custom_httpx/llm_http_handler.py
+34 -24
View File
@@ -111,6 +111,7 @@ class BaseLLMHTTPHandler:
response: Optional[httpx.Response] = None
for i in range(max(max_retry_on_unprocessable_entity_error, 1)):
try:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
@@ -2712,7 +2713,8 @@ class BaseLLMHTTPHandler:
headers = image_generation_provider_config.validate_environment(
api_key=litellm_params.get("api_key", None),
headers=image_generation_optional_request_params.get("extra_headers", {}) or {},
headers=image_generation_optional_request_params.get("extra_headers", {})
or {},
model=model,
messages=[],
optional_params=image_generation_optional_request_params,
@@ -2763,15 +2765,17 @@ class BaseLLMHTTPHandler:
provider_config=image_generation_provider_config,
)
model_response: ImageResponse = image_generation_provider_config.transform_image_generation_response(
model=model,
raw_response=response,
model_response=litellm.ImageResponse(),
logging_obj=logging_obj,
request_data=data,
optional_params=image_generation_optional_request_params,
litellm_params=dict(litellm_params),
encoding=None,
model_response: ImageResponse = (
image_generation_provider_config.transform_image_generation_response(
model=model,
raw_response=response,
model_response=litellm.ImageResponse(),
logging_obj=logging_obj,
request_data=data,
optional_params=image_generation_optional_request_params,
litellm_params=dict(litellm_params),
encoding=None,
)
)
return model_response
@@ -2804,10 +2808,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers = image_generation_provider_config.validate_environment(
api_key=litellm_params.get("api_key", None),
headers=image_generation_optional_request_params.get("extra_headers", {}) or {},
headers=image_generation_optional_request_params.get("extra_headers", {})
or {},
model=model,
messages=[],
optional_params=image_generation_optional_request_params,
@@ -2858,17 +2862,19 @@ class BaseLLMHTTPHandler:
provider_config=image_generation_provider_config,
)
model_response: ImageResponse = image_generation_provider_config.transform_image_generation_response(
model=model,
raw_response=response,
model_response=litellm.ImageResponse(),
logging_obj=logging_obj,
request_data=data,
optional_params=image_generation_optional_request_params,
litellm_params=dict(litellm_params),
encoding=None,
model_response: ImageResponse = (
image_generation_provider_config.transform_image_generation_response(
model=model,
raw_response=response,
model_response=litellm.ImageResponse(),
logging_obj=logging_obj,
request_data=data,
optional_params=image_generation_optional_request_params,
litellm_params=dict(litellm_params),
encoding=None,
)
)
return model_response
###### VECTOR STORE HANDLER ######
@@ -2936,7 +2942,9 @@ class BaseLLMHTTPHandler:
},
)
request_data = json.dumps(request_body) if signed_json_body is None else signed_json_body
request_data = (
json.dumps(request_body) if signed_json_body is None else signed_json_body
)
try:
response = await async_httpx_client.post(
@@ -3035,7 +3043,9 @@ class BaseLLMHTTPHandler:
},
)
request_data = json.dumps(request_body) if signed_json_body is None else signed_json_body
request_data = (
json.dumps(request_body) if signed_json_body is None else signed_json_body
)
try:
response = sync_httpx_client.post(
litellm/llms/datarobot/chat/transformation.py
+20 -11
View File
@@ -6,8 +6,11 @@ Calls done in OpenAI/openai.py as DataRobot is openai-compatible.
from typing import Optional, Tuple
from litellm.secret_managers.main import get_secret_str
from urllib.parse import urlparse, urlunparse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
LLMGW_PATH = "/genai/llmgw/chat/completions"
class DataRobotConfig(OpenAILikeChatConfig):
@staticmethod
@@ -32,22 +35,28 @@ class DataRobotConfig(OpenAILikeChatConfig):
if api_base is None:
api_base = "https://app.datarobot.com"
# If the api_base is a deployment URL, we do not append the chat completions path
if "api/v2/deployments" not in api_base:
# If the api_base is not a deployment URL, we need to append the chat completions path
if "api/v2/genai/llmgw/chat/completions" not in api_base:
api_base += "/api/v2/genai/llmgw/chat/completions"
parsed = urlparse(api_base)
path = parsed.path
if not path or path == "/": # Add full path to LLMGW
path += f"/api/v2/{LLMGW_PATH}"
elif "api/v2/deployments" in path: # Dedicated deployment, leave it
pass
elif (
"api/v2" in path and LLMGW_PATH not in path
): # Standard ENDPOINT path, add LLMGW
path += LLMGW_PATH
# Ensure the url ends with a trailing slash
if not api_base.endswith("/"):
api_base += "/"
if not path.endswith("/"):
path += "/"
path = path.replace("//", "/")
updated_parsed = parsed._replace(path=path)
return api_base # type: ignore
return urlunparse(updated_parsed)
def _get_openai_compatible_provider_info(
self,
api_base: Optional[str],
api_key: Optional[str]
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
"""Attempts to ensure that the API base and key are set, preferring user-provided values,
before falling back to secret manager values (``DATAROBOT_ENDPOINT`` and ``DATAROBOT_API_TOKEN``
litellm/llms/deepinfra/rerank/transformation.py
+239
View File
@@ -0,0 +1,239 @@
"""
Translate between Cohere's `/rerank` format and Deepinfra's `/rerank` format.
"""
import uuid
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.rerank.transformation import (
BaseLLMException,
BaseRerankConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.rerank import (
OptionalRerankParams,
RerankBilledUnits,
RerankResponse,
RerankResponseMeta,
RerankResponseResult,
RerankTokens,
)
class DeepinfraRerankConfig(BaseRerankConfig):
"""
Deepinfra Rerank - Follows the same Spec as Cohere Rerank
"""
def get_complete_url(self, api_base: Optional[str], model: str) -> str:
"""
Constructs the complete DeepInfra inference endpoint URL for rerank.
Args:
api_base (Optional[str]): The base URL for the DeepInfra API.
model (str): The model identifier.
Returns:
str: The complete URL for the DeepInfra rerank inference endpoint.
Raises:
ValueError: If api_base is None.
"""
if not api_base:
raise ValueError(
"Deepinfra API Base is required. api_base=None. Set in call or via `DEEPINFRA_API_BASE` env var."
)
# Remove 'openai' from the base if present
api_base_clean = (
api_base.replace("openai", "") if "openai" in api_base else api_base
)
# Remove any trailing slashes for consistency, then add one
api_base_clean = api_base_clean.rstrip("/") + "/"
# Compose the full endpoint
return f"{api_base_clean}inference/{model}"
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DEEPINFRA_API_KEY")
if api_key is None:
raise ValueError(
"Deepinfra API key is required. Please set 'DEEPINFRA_API_KEY' environment variable"
)
default_headers = {
"Authorization": f"Bearer {api_key}",
"accept": "application/json",
"content-type": "application/json",
}
# If 'Authorization' is provided in headers, it overrides the default.
if "Authorization" in headers:
default_headers["Authorization"] = headers["Authorization"]
# Merge other headers, overriding any default ones except Authorization
return {**default_headers, **headers}
def map_cohere_rerank_params(
self,
non_default_params: dict,
model: str,
drop_params: bool,
query: str,
documents: List[Union[str, Dict[str, Any]]],
custom_llm_provider: Optional[str] = None,
top_n: Optional[int] = None,
rank_fields: Optional[List[str]] = None,
return_documents: Optional[bool] = True,
max_chunks_per_doc: Optional[int] = None,
max_tokens_per_doc: Optional[int] = None,
) -> OptionalRerankParams:
# Start with the basic parameters
optional_rerank_params = {}
if query:
optional_rerank_params["queries"] = [query] * len(
documents
) # Deepinfra rerank requires queries to be of same length as documents
if non_default_params is not None:
for k, v in non_default_params.items():
if k == "queries" and v is not None:
# This should override the query parameter if it is provided
optional_rerank_params["queries"] = v
elif k == "documents" and v is not None:
optional_rerank_params["documents"] = v
elif k == "service_tier" and v is not None:
optional_rerank_params["service_tier"] = v
elif k == "instruction" and v is not None:
optional_rerank_params["instruction"] = v
elif k == "webhook" and v is not None:
optional_rerank_params["webhook"] = v
return OptionalRerankParams(**optional_rerank_params) # type: ignore
def transform_rerank_request(
self,
model: str,
optional_rerank_params: OptionalRerankParams,
headers: dict,
) -> dict:
# Convert OptionalRerankParams to dict as expected by parent class
if optional_rerank_params is None:
return {}
return dict(optional_rerank_params)
def transform_rerank_response(
self,
model: str,
raw_response: httpx.Response,
model_response: RerankResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
request_data: dict = {},
optional_params: dict = {},
litellm_params: dict = {},
) -> RerankResponse:
try:
response_json = raw_response.json()
logging_obj.post_call(original_response=raw_response.text)
# Extract the scores from the response
scores = response_json.get("scores", [])
input_tokens = response_json.get("input_tokens", 0)
request_id = response_json.get("request_id")
# Create inference status information
inference_status = response_json.get("inference_status", {})
status = inference_status.get("status", "unknown")
runtime_ms = inference_status.get("runtime_ms", 0)
cost = inference_status.get("cost", 0.0)
tokens_generated = inference_status.get("tokens_generated", 0)
tokens_input = inference_status.get("tokens_input", 0)
# Create RerankResponse
results = []
for i, score in enumerate(scores):
results.append(
RerankResponseResult(index=i, relevance_score=float(score))
)
# Create metadata for the response
tokens = RerankTokens(
input_tokens=input_tokens,
output_tokens=0, # DeepInfra doesn't provide output tokens for rerank
)
billed_units = RerankBilledUnits(total_tokens=input_tokens)
meta = RerankResponseMeta(tokens=tokens, billed_units=billed_units)
rerank_response = RerankResponse(
id=request_id or str(uuid.uuid4()), results=results, meta=meta
)
# Store additional information in hidden params
rerank_response._hidden_params = {
"status": status,
"runtime_ms": runtime_ms,
"cost": cost,
"tokens_generated": tokens_generated,
"tokens_input": tokens_input,
"model": model,
}
return rerank_response
except Exception:
# If there's an error parsing the response, fall back to the parent implementation
rerank_response = super().transform_rerank_response(
model=model,
raw_response=raw_response,
model_response=model_response,
logging_obj=logging_obj,
api_key=api_key,
request_data=request_data,
optional_params=optional_params,
litellm_params=litellm_params,
)
rerank_response._hidden_params["model"] = model
return rerank_response
def get_supported_cohere_rerank_params(self, model: str) -> list:
return ["query", "documents"]
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
# Deepinfra errors may come as JSON: {"detail": {"error": "..."}}
import json
# Try to extract a more specific error message if possible
try:
error_data = error_message
if isinstance(error_message, str):
error_data = json.loads(error_message)
if isinstance(error_data, dict):
# Check for {"detail": {"error": "..."}}
detail = error_data.get("detail")
if isinstance(detail, dict) and "error" in detail:
error_message = detail["error"]
elif isinstance(detail, str):
error_message = detail
except Exception:
# If parsing fails, just use the original error_message
pass
raise BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
)
litellm/llms/litellm_proxy/image_edit/transformation.py
+26
View File
@@ -0,0 +1,26 @@
from typing import Optional
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
class LiteLLMProxyImageEditConfig(OpenAIImageEditConfig):
"""Configuration for image edit requests routed through LiteLLM Proxy."""
def validate_environment(
self, headers: dict, model: str, api_key: Optional[str] = None
) -> dict:
api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
headers.update({"Authorization": f"Bearer {api_key}"})
return headers
def get_complete_url(
self, model: str, api_base: Optional[str], litellm_params: dict
) -> str:
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
if api_base is None:
raise ValueError(
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
)
api_base = api_base.rstrip("/")
return f"{api_base}/images/edits"
litellm/llms/litellm_proxy/image_generation/transformation.py
+40
View File
@@ -0,0 +1,40 @@
from typing import Optional
from litellm.llms.openai.image_generation.gpt_transformation import (
GPTImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
class LiteLLMProxyImageGenerationConfig(GPTImageGenerationConfig):
"""Configuration for image generation requests routed through LiteLLM Proxy."""
def validate_environment(
self,
headers: dict,
model: str,
messages,
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
headers.update({"Authorization": f"Bearer {api_key}"})
return headers
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
if api_base is None:
raise ValueError(
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
)
api_base = api_base.rstrip("/")
return f"{api_base}/images/generations"
litellm/llms/mistral/chat/transformation.py
+188 -30
View File
@@ -6,7 +6,18 @@ Why separate file? Make it easy to see how transformation works
Docs - https://docs.mistral.ai/api/
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
from typing import (
Any,
Coroutine,
List,
Literal,
Optional,
Tuple,
Union,
cast,
get_type_hints,
overload,
)
import httpx
@@ -17,7 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralToolCallMessage
from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from litellm.utils import convert_to_model_response_object
@@ -145,7 +156,9 @@ class MistralConfig(OpenAIGPTConfig):
for param, value in non_default_params.items():
if param == "max_tokens":
optional_params["max_tokens"] = value
if param == "max_completion_tokens": # max_completion_tokens should take priority
if (
param == "max_completion_tokens"
): # max_completion_tokens should take priority
optional_params["max_tokens"] = value
if param == "tools":
# Clean tools to remove problematic schema fields for Mistral API
@@ -159,7 +172,9 @@ class MistralConfig(OpenAIGPTConfig):
if param == "stop":
optional_params["stop"] = value
if param == "tool_choice" and isinstance(value, str):
optional_params["tool_choice"] = self._map_tool_choice(tool_choice=value)
optional_params["tool_choice"] = self._map_tool_choice(
tool_choice=value
)
if param == "seed":
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
@@ -185,7 +200,9 @@ class MistralConfig(OpenAIGPTConfig):
) # type: ignore
# if api_base does not end with /v1 we add it
if api_base is not None and not api_base.endswith("/v1"): # Mistral always needs a /v1 at the end
if api_base is not None and not api_base.endswith(
"/v1"
): # Mistral always needs a /v1 at the end
api_base = api_base + "/v1"
dynamic_api_key = (
api_key
@@ -194,10 +211,12 @@ class MistralConfig(OpenAIGPTConfig):
)
return api_base, dynamic_api_key
# fmt: off
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
@@ -206,8 +225,9 @@ class MistralConfig(OpenAIGPTConfig):
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
) -> List[AllMessageValues]:
...
# fmt: on
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
@@ -218,18 +238,20 @@ class MistralConfig(OpenAIGPTConfig):
- if image passed in, then just return as is (user-intended)
- if `name` is passed, then drop it for mistral API: https://github.com/BerriAI/litellm/issues/6696
Motivation: mistral api doesn't support content as a list
Motivation: mistral api doesn't support content as a list.
The above statement is not valid now. Need to plan to remove all the #1,2,3
Mistral API supports content as a list.
"""
## 1. If 'image_url' in content, then return as is
## 1. If 'image_url' or 'file' in content, then transform with base class and mistral-specific handling
for m in messages:
_content_block = m.get("content")
if _content_block and isinstance(_content_block, list):
for c in _content_block:
if c.get("type") == "image_url":
if is_async:
return super()._transform_messages(messages, model, True)
else:
return super()._transform_messages(messages, model, False)
if any(c.get("type") in ["image_url", "file"] for c in _content_block):
if is_async:
return self._transform_messages_async(messages, model)
else:
messages = self._transform_messages_sync(messages, model)
return messages
## 2. If content is list, then convert to string
messages = handle_messages_with_content_list_to_str_conversion(messages)
@@ -239,6 +261,8 @@ class MistralConfig(OpenAIGPTConfig):
for m in messages:
m = MistralConfig._handle_name_in_message(m)
m = MistralConfig._handle_tool_call_message(m)
if MistralConfig._is_empty_assistant_message(m):
continue
m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error
new_messages.append(m)
@@ -247,6 +271,51 @@ class MistralConfig(OpenAIGPTConfig):
else:
return super()._transform_messages(new_messages, model, False)
async def _transform_messages_async(self,
messages: List[AllMessageValues], model: str
) -> List[AllMessageValues]:
"""
Handle modification of messages for Mistral API in an async context.
"""
# Call parent async method to handle basic transformations
# and then apply Mistral-specific handling for files
messages = await super()._transform_messages(messages, model, True)
messages = self._handle_message_with_file(messages)
return messages
def _transform_messages_sync(self,
messages: List[AllMessageValues], model: str
) -> List[AllMessageValues]:
""" Handle modification of messages for Mistral API in a sync context.
"""
# Call parent sync method to handle basic transformations
# and then apply Mistral-specific handling for files
# This is the sync version of the async method above
messages = super()._transform_messages(messages, model, False)
messages = self._handle_message_with_file(messages)
return messages
def _handle_message_with_file(
self,
messages: List[AllMessageValues]) -> List[AllMessageValues]:
"""
Mistral API supports only 'file_id' in message content with type 'file'.
"""
for m in messages:
_content_block = m.get("content")
if _content_block and isinstance(_content_block, list):
if any(c.get("type") == "file" for c in _content_block):
# If file content is present, we get file_id from 'file' attribute of content block
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
file_contents = [c for c in _content_block if c.get("type") == "file"]
for file_content in file_contents:
file_id = file_content.get("file", {}).get("file_id")
if file_id:
# Replace 'file' with 'file_id'
file_content["file_id"] = file_id # type: ignore
file_content.pop("file", None)
return messages
def _add_reasoning_system_prompt_if_needed(
self, messages: List[AllMessageValues], optional_params: dict
) -> List[AllMessageValues]:
@@ -269,20 +338,30 @@ class MistralConfig(OpenAIGPTConfig):
# Handle both string and list content, preserving original format
if isinstance(existing_content, str):
# String content - prepend reasoning prompt
new_content: Union[str, list] = f"{reasoning_prompt}\n\n{existing_content}"
new_content: Union[str, list] = (
f"{reasoning_prompt}\n\n{existing_content}"
)
elif isinstance(existing_content, list):
# List content - prepend reasoning prompt as text block
new_content = [{"type": "text", "text": reasoning_prompt + "\n\n"}] + existing_content
new_content = [
{"type": "text", "text": reasoning_prompt + "\n\n"}
] + existing_content
else:
# Fallback for any other type - convert to string
new_content = f"{reasoning_prompt}\n\n{str(existing_content)}"
messages[i] = cast(AllMessageValues, {**msg, "content": new_content})
messages[i] = cast(
AllMessageValues, {**msg, "content": new_content}
)
break
else:
# Add new system message with reasoning instructions
reasoning_message: AllMessageValues = cast(
AllMessageValues, {"role": "system", "content": self._get_mistral_reasoning_system_prompt()}
AllMessageValues,
{
"role": "system",
"content": self._get_mistral_reasoning_system_prompt(),
},
)
messages = [reasoning_message] + messages
@@ -294,32 +373,34 @@ class MistralConfig(OpenAIGPTConfig):
def _clean_tool_schema_for_mistral(cls, tools: list) -> list:
"""
Clean tool schemas to remove fields that cause issues with Mistral API.
Removes:
- $id and $schema fields (cause grammar validation errors)
- additionalProperties=False (causes OpenAI API schema errors)
- strict field (not supported by Mistral)
Args:
tools: List of tool definitions
max_depth: Maximum recursion depth for schema cleaning (default: 10)
Returns:
Cleaned tools list
"""
if not tools:
return tools
import copy
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.utils import _remove_json_schema_refs
cleaned_tools = copy.deepcopy(tools)
# Apply all cleaning functions with max_depth protection
cleaned_tools = _remove_json_schema_refs(cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH)
cleaned_tools = _remove_json_schema_refs(
cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH
)
return cleaned_tools
@classmethod
@@ -360,6 +441,25 @@ class MistralConfig(OpenAIGPTConfig):
message["tool_calls"] = mistral_tool_calls # type: ignore
return message
@classmethod
def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool:
"""
Mistral API does not support empty string in assistant content.
"""
from litellm.types.llms.openai import ChatCompletionAssistantMessage
set_keys = get_type_hints(ChatCompletionAssistantMessage).keys()
all_expected_values_are_empty = True
for key in set_keys:
if key != "role" and message.get(key) is not None:
if key == "content" and message.get(key) == "":
continue
else:
all_expected_values_are_empty = False
break
return all_expected_values_are_empty
@staticmethod
def _handle_empty_content_response(response_data: dict) -> dict:
"""
@@ -380,6 +480,58 @@ class MistralConfig(OpenAIGPTConfig):
choice["message"]["content"] = None
return response_data
@staticmethod
def _convert_thinking_block_to_reasoning_content(
thinking_blocks: MistralThinkingBlock,
) -> str:
"""
Convert Mistral thinking blocks to reasoning content.
"""
return "\n".join(
[block.get("text", "") for block in thinking_blocks["thinking"]]
)
@staticmethod
def _handle_content_list_to_str_conversion(response_data: dict) -> dict:
"""
Handle Mistral's content list format and extract thinking content.
Map mistral's content list to string and extract thinking blocks:
- Thinking block -> reasoning_content field
- Text block -> content field
"""
if response_data.get("choices") and len(response_data["choices"]) > 0:
for choice in response_data["choices"]:
if choice.get("message") and choice["message"].get("content"):
content = choice["message"]["content"]
# Only process if content is a list
if isinstance(content, list):
thinking_content = ""
text_content = ""
# Process each content block
for block in content:
if block.get("type") == "thinking":
thinking_blocks = block.get("thinking", [])
thinking_texts = []
for thinking_block in thinking_blocks:
if thinking_block.get("type") == "text":
thinking_texts.append(
thinking_block.get("text", "")
)
thinking_content = "\n".join(thinking_texts)
elif block.get("type") == "text":
text_content = block.get("text", "")
# Set the extracted content
choice["message"]["content"] = text_content
if thinking_content:
choice["message"]["reasoning_content"] = thinking_content
return response_data
def transform_request(
self,
model: str,
@@ -396,8 +548,12 @@ class MistralConfig(OpenAIGPTConfig):
dict: The transformed request. Sent as the body of the API call.
"""
# Add reasoning system prompt if needed (for magistral models)
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
if "magistral" in model.lower() and optional_params.get(
"_add_reasoning_prompt", False
):
messages = self._add_reasoning_system_prompt_if_needed(
messages, optional_params
)
# Call parent transform_request which handles _transform_messages
return super().transform_request(
@@ -424,14 +580,16 @@ class MistralConfig(OpenAIGPTConfig):
) -> ModelResponse:
"""
Transform the raw response from Mistral API.
Handles Mistral-specific behavior like converting empty string content to None.
Handles Mistral-specific behavior like converting empty string content to None
and extracting thinking content from content lists.
"""
logging_obj.post_call(original_response=raw_response.text)
logging_obj.model_call_details["response_headers"] = raw_response.headers
# Handle Mistral-specific empty string content conversion to None
# Handle Mistral-specific response transformations
response_data = raw_response.json()
response_data = self._handle_empty_content_response(response_data)
response_data = self._handle_content_list_to_str_conversion(response_data)
final_response_obj = cast(
ModelResponse,
litellm/llms/ollama/completion/transformation.py
+44 -30
View File
@@ -262,38 +262,52 @@ class OllamaConfig(BaseConfig):
## RESPONSE OBJECT
model_response.choices[0].finish_reason = "stop"
if request_data.get("format", "") == "json":
response_content = json.loads(response_json["response"])
# Check if this is a function call format with name/arguments structure
if (
isinstance(response_content, dict)
and "name" in response_content
and "arguments" in response_content
):
# Handle as function call (original behavior)
function_call = response_content
message = litellm.Message(
content=None,
tool_calls=[
{
"id": f"call_{str(uuid.uuid4())}",
"function": {
"name": function_call["name"],
"arguments": json.dumps(function_call["arguments"]),
},
"type": "function",
}
],
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "tool_calls"
else:
# Handle as regular JSON (new behavior)
message = litellm.Message(
content=json.dumps(response_content),
)
# Check if response field exists and is not empty before parsing JSON
response_text = response_json.get("response", "")
if not response_text or not response_text.strip():
# Handle empty response gracefully - set empty content
message = litellm.Message(content="")
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
else:
try:
response_content = json.loads(response_text)
# Check if this is a function call format with name/arguments structure
if (
isinstance(response_content, dict)
and "name" in response_content
and "arguments" in response_content
):
# Handle as function call (original behavior)
function_call = response_content
message = litellm.Message(
content=None,
tool_calls=[
{
"id": f"call_{str(uuid.uuid4())}",
"function": {
"name": function_call["name"],
"arguments": json.dumps(function_call["arguments"]),
},
"type": "function",
}
],
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "tool_calls"
else:
# Handle as regular JSON (new behavior)
message = litellm.Message(
content=json.dumps(response_content),
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
except json.JSONDecodeError:
# If JSON parsing fails, treat as regular text response
message = litellm.Message(content=response_text)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
else:
model_response.choices[0].message.content = response_json["response"] # type: ignore
model_response.created = int(time.time())
litellm/llms/openai/chat/gpt_5_transformation.py
+6 -2
View File
@@ -15,14 +15,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
- Mapping ``max_tokens`` -> ``max_completion_tokens``.
- Dropping unsupported ``temperature`` values when requested.
"""
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:
return "gpt-5" in model
def get_supported_openai_params(self, model: str) -> list:
from litellm.utils import supports_tool_choice
base_gpt_series_params = super().get_supported_openai_params(model=model)
gpt_5_only_params = ["reasoning_effort"]
base_gpt_series_params.extend(gpt_5_only_params)
if not supports_tool_choice(model=model):
base_gpt_series_params.remove("tool_choice")
return base_gpt_series_params
def map_openai_params(
@@ -61,4 +66,3 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
model=model,
drop_params=drop_params,
)
litellm/llms/openai/chat/gpt_transformation.py
+3
View File
@@ -348,6 +348,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
for message in messages:
message_content = message.get("content")
message_role = message.get("role")
if (
message_role == "user"
and message_content
@@ -428,6 +429,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
optional_params.pop("max_retries", None)
return {
"model": model,
"messages": messages,
litellm/llms/openai/responses/transformation.py
+27 -32
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hints
import httpx
from pydantic import BaseModel
@@ -13,6 +13,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from ..common_utils import OpenAIError
@@ -25,38 +26,28 @@ else:
class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.OPENAI
def get_supported_openai_params(self, model: str) -> list:
"""
All OpenAI Responses API params are supported
"""
return [
"input",
"model",
"include",
"instructions",
"max_output_tokens",
"metadata",
"parallel_tool_calls",
"previous_response_id",
"reasoning",
"store",
"background",
"stream",
"prompt",
"temperature",
"text",
"tool_choice",
"tools",
"top_p",
"truncation",
"user",
"service_tier",
"safety_identifier",
"extra_headers",
"extra_query",
"extra_body",
"timeout",
]
supported_params = get_type_hints(ResponsesAPIRequestParams).keys()
return list(
set(
[
"input",
"model",
"extra_headers",
"extra_query",
"extra_body",
"timeout",
]
+ list(supported_params)
)
)
def map_openai_params(
self,
@@ -85,8 +76,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
)
return final_request_params
def _validate_input_param(self, input: Union[str, ResponseInputParam]) -> Union[str, ResponseInputParam]:
def _validate_input_param(
self, input: Union[str, ResponseInputParam]
) -> Union[str, ResponseInputParam]:
"""
Ensure all input fields if pydantic are converted to dict
@@ -114,7 +107,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
"""No transform applied since outputs are in OpenAI spec already"""
try:
raw_response_json = raw_response.json()
raw_response_json["created_at"] = _safe_convert_created_field(raw_response_json["created_at"])
raw_response_json["created_at"] = _safe_convert_created_field(
raw_response_json["created_at"]
)
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code
litellm/llms/openai_like/chat/handler.py
+16 -2
View File
@@ -49,8 +49,15 @@ async def make_call(
model_response = ModelResponse(**response.json())
completion_stream = MockResponseIterator(model_response=model_response)
else:
# Use aiter_text with explicit UTF-8 encoding to avoid ASCII encoding errors
async def utf8_aiter_lines():
async for line in response.aiter_text(encoding='utf-8'):
for line_part in line.splitlines(keepends=True):
if line_part.strip():
yield line_part.rstrip('\r\n')
completion_stream = ModelResponseIterator(
streaming_response=response.aiter_lines(), sync_stream=False
streaming_response=utf8_aiter_lines(), sync_stream=False
)
# LOGGING
logging_obj.post_call(
@@ -93,8 +100,15 @@ def make_sync_call(
model_response = ModelResponse(**response.json())
completion_stream = MockResponseIterator(model_response=model_response)
else:
# Use iter_text with explicit UTF-8 encoding to avoid ASCII encoding errors
def utf8_iter_lines():
for line in response.iter_text(encoding='utf-8'):
for line_part in line.splitlines(keepends=True):
if line_part.strip():
yield line_part.rstrip('\r\n')
completion_stream = ModelResponseIterator(
streaming_response=response.iter_lines(), sync_stream=True
streaming_response=utf8_iter_lines(), sync_stream=True
)
# LOGGING
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+33 -22
View File
@@ -305,9 +305,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return None
for tool in value:
openai_function_object: Optional[
ChatCompletionToolParamFunctionChunk
] = None
openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = (
None
)
if "function" in tool: # tools list
_openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore
**tool["function"]
@@ -597,14 +597,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif param == "seed":
optional_params["seed"] = value
elif param == "reasoning_effort" and isinstance(value, str):
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
)
elif param == "thinking":
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_thinking_param(
cast(AnthropicThinkingParam, value)
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_thinking_param(
cast(AnthropicThinkingParam, value)
)
)
elif param == "modalities" and isinstance(value, list):
response_modalities = self.map_response_modalities(value)
@@ -1000,6 +1000,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
GenerateContentResponseBody, BidiGenerateContentServerMessage
],
) -> Usage:
if (
completion_response is not None
and "usageMetadata" not in completion_response
@@ -1038,6 +1039,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
text_tokens = detail.get("tokenCount", 0)
if "thoughtsTokenCount" in usage_metadata:
reasoning_tokens = usage_metadata["thoughtsTokenCount"]
## adjust 'text_tokens' to subtract cached tokens
if (
(audio_tokens is None or audio_tokens == 0)
and text_tokens is not None
and text_tokens > 0
and cached_tokens is not None
):
text_tokens = text_tokens - cached_tokens
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
audio_tokens=audio_tokens,
@@ -1344,28 +1355,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
## ADD METADATA TO RESPONSE ##
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
model_response._hidden_params[
"vertex_ai_grounding_metadata"
] = grounding_metadata
model_response._hidden_params["vertex_ai_grounding_metadata"] = (
grounding_metadata
)
setattr(
model_response, "vertex_ai_url_context_metadata", url_context_metadata
)
model_response._hidden_params[
"vertex_ai_url_context_metadata"
] = url_context_metadata
model_response._hidden_params["vertex_ai_url_context_metadata"] = (
url_context_metadata
)
setattr(model_response, "vertex_ai_safety_results", safety_ratings)
model_response._hidden_params[
"vertex_ai_safety_results"
] = safety_ratings # older approach - maintaining to prevent regressions
model_response._hidden_params["vertex_ai_safety_results"] = (
safety_ratings # older approach - maintaining to prevent regressions
)
## ADD CITATION METADATA ##
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
model_response._hidden_params[
"vertex_ai_citation_metadata"
] = citation_metadata # older approach - maintaining to prevent regressions
model_response._hidden_params["vertex_ai_citation_metadata"] = (
citation_metadata # older approach - maintaining to prevent regressions
)
except Exception as e:
raise VertexAIError(
litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py
+4 -4
View File
@@ -113,10 +113,10 @@ class VertexAILlama3Config(OpenAIGPTConfig):
status_code=raw_response.status_code,
headers=response_headers,
)
model_response.model = completion_response["model"]
model_response.id = completion_response["id"]
model_response.created = completion_response["created"]
setattr(model_response, "usage", Usage(**completion_response["usage"]))
model_response.model = completion_response.get("model", model)
model_response.id = completion_response.get("id", "")
model_response.created = completion_response.get("created", 0)
setattr(model_response, "usage", Usage(**completion_response.get("usage", {})))
model_response.choices = self._transform_choices( # type: ignore
choices=completion_response["choices"],
litellm/llms/vertex_ai/vertex_ai_partner_models/main.py
+14 -2
View File
@@ -48,9 +48,21 @@ class VertexAIPartnerModels(VertexBase):
or model.startswith("codestral")
or model.startswith("jamba")
or model.startswith("claude")
or model.startswith("qwen")
):
return True
return False
@staticmethod
def should_use_openai_handler(model: str):
OPENAI_LIKE_VERTEX_PROVIDERS = [
"llama",
"deepseek-ai",
"qwen",
]
if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS):
return True
return False
def completion(
self,
@@ -115,7 +127,7 @@ class VertexAIPartnerModels(VertexBase):
optional_params["stream"] = stream
if "llama" in model or "deepseek-ai" in model:
if self.should_use_openai_handler(model):
partner = VertexPartnerProvider.llama
elif "mistral" in model or "codestral" in model:
partner = VertexPartnerProvider.mistralai
@@ -191,7 +203,7 @@ class VertexAIPartnerModels(VertexBase):
client=client,
custom_llm_provider=LlmProviders.VERTEX_AI.value,
)
elif "llama" in model:
elif self.should_use_openai_handler(model):
return base_llm_http_handler.completion(
model=model,
stream=stream,
litellm/llms/voyage/embedding/transformation_contextual.py
+153
View File
@@ -0,0 +1,153 @@
"""
This module is used to transform the request and response for the Voyage contextualized embeddings API.
This would be used for all the contextualized embeddings models in Voyage.
"""
from typing import List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, Usage
class VoyageError(BaseLLMException):
def __init__(
self,
status_code: int,
message: str,
headers: Union[dict, httpx.Headers] = {},
):
self.status_code = status_code
self.message = message
self.request = httpx.Request(
method="POST", url="https://api.voyageai.com/v1/contextualizedembeddings"
)
self.response = httpx.Response(status_code=status_code, request=self.request)
super().__init__(
status_code=status_code,
message=message,
headers=headers,
)
class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig):
"""
Reference: https://docs.voyageai.com/reference/embeddings-api
"""
def __init__(self) -> None:
pass
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if api_base:
if not api_base.endswith("/contextualizedembeddings"):
api_base = f"{api_base}/contextualizedembeddings"
return api_base
return "https://api.voyageai.com/v1/contextualizedembeddings"
def get_supported_openai_params(self, model: str) -> list:
return ["encoding_format", "dimensions"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI params to Voyage params
Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api
"""
if "encoding_format" in non_default_params:
optional_params["encoding_format"] = non_default_params["encoding_format"]
if "dimensions" in non_default_params:
optional_params["output_dimension"] = non_default_params["dimensions"]
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
if api_key is None:
api_key = (
get_secret_str("VOYAGE_API_KEY")
or get_secret_str("VOYAGE_AI_API_KEY")
or get_secret_str("VOYAGE_AI_TOKEN")
)
return {
"Authorization": f"Bearer {api_key}",
}
def transform_embedding_request(
self,
model: str,
input: Union[AllEmbeddingInputValues, List[List[str]]],
optional_params: dict,
headers: dict,
) -> dict:
return {
"inputs": input,
"model": model,
**optional_params,
}
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
request_data: dict = {},
optional_params: dict = {},
litellm_params: dict = {},
) -> EmbeddingResponse:
try:
raw_response_json = raw_response.json()
except Exception:
raise VoyageError(
message=raw_response.text, status_code=raw_response.status_code
)
# model_response.usage
model_response.model = raw_response_json.get("model")
model_response.data = raw_response_json.get("data")
model_response.object = raw_response_json.get("object")
usage = Usage(
prompt_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0),
total_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0),
)
model_response.usage = usage
return model_response
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
return VoyageError(
message=error_message, status_code=status_code, headers=headers
)
@staticmethod
def is_contextualized_embeddings(model: str) -> bool:
return "context" in model.lower()
litellm/main.py
+5 -36
View File
@@ -130,7 +130,6 @@ from .litellm_core_utils.prompt_templates.factory import (
stringify_json_tool_call_content,
)
from .litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor
from .llms import baseten
from .llms.anthropic.chat import AnthropicChatCompletion
from .llms.azure.audio_transcriptions import AzureAudioTranscription
from .llms.azure.azure import AzureChatCompletion, _check_dynamic_azure_params
@@ -1562,6 +1561,7 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "deepseek":
## COMPLETION CALL
try:
response = base_llm_http_handler.completion(
model=model,
@@ -1593,6 +1593,7 @@ def completion( # type: ignore # noqa: PLR0915
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
api_base = AzureFoundryModelInfo.get_api_base(api_base)
# set API KEY
api_key = AzureFoundryModelInfo.get_api_key(api_key)
@@ -1921,6 +1922,7 @@ def completion( # type: ignore # noqa: PLR0915
or custom_llm_provider == "perplexity"
or custom_llm_provider == "nvidia_nim"
or custom_llm_provider == "cerebras"
or custom_llm_provider == "baseten"
or custom_llm_provider == "sambanova"
or custom_llm_provider == "volcengine"
or custom_llm_provider == "anyscale"
@@ -1976,8 +1978,10 @@ def completion( # type: ignore # noqa: PLR0915
use_base_llm_http_handler = get_secret_bool(
"EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER"
)
try:
if use_base_llm_http_handler:
response = base_llm_http_handler.completion(
model=model,
messages=messages,
@@ -3260,42 +3264,7 @@ def completion( # type: ignore # noqa: PLR0915
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
)
elif (
custom_llm_provider == "baseten"
or litellm.api_base == "https://app.baseten.co"
):
custom_llm_provider = "baseten"
baseten_key = (
api_key
or litellm.baseten_key
or os.environ.get("BASETEN_API_KEY")
or litellm.api_key
)
model_response = baseten.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=encoding,
api_key=baseten_key,
logging_obj=logging,
)
if inspect.isgenerator(model_response) or (
"stream" in optional_params and optional_params["stream"] is True
):
# don't try to access stream object,
response = CustomStreamWrapper(
model_response,
model,
custom_llm_provider="baseten",
logging_obj=logging,
)
return response
response = model_response
elif custom_llm_provider == "petals" or model in litellm.petals_models:
api_base = api_base or litellm.api_base
litellm/model_prices_and_context_window_backup.json
+1238 -166
View File
File diff suppressed because it is too large Load Diff
litellm/passthrough/main.py
+7
View File
@@ -24,6 +24,7 @@ import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.passthrough.utils import CommonUtils
from litellm.utils import client
base_llm_http_handler = BaseLLMHTTPHandler()
@@ -241,6 +242,12 @@ def llm_passthrough_route(
request_query_params=request_query_params,
litellm_params=litellm_params_dict,
)
# need to encode the id of application-inference-profile for bedrock
if custom_llm_provider == "bedrock" and "application-inference-profile" in endpoint:
encoded_url_str = CommonUtils.encode_bedrock_runtime_modelid_arn(str(updated_url))
updated_url = httpx.URL(encoded_url_str)
# Add or update query parameters
provider_api_key = provider_config.get_api_key(api_key)
litellm/passthrough/utils.py
+53
View File
@@ -37,3 +37,56 @@ class BasePassthroughUtils:
# Combine request headers with custom headers
headers = {**request_headers, **headers}
return headers
class CommonUtils:
@staticmethod
def encode_bedrock_runtime_modelid_arn(endpoint: str) -> str:
"""
Encodes any "/" found in the modelId of an AWS Bedrock Runtime Endpoint when arns are passed in.
- modelID value can be an ARN which contains slashes that SHOULD NOT be treated as path separators.
e.g endpoint: /model/<modelId>/invoke
<modelId> containing arns with slashes need to be encoded from
arn:aws:bedrock:ap-southeast-1:123456789012:application-inference-profile/abdefg12334 =>
arn:aws:bedrock:ap-southeast-1:123456789012:application-inference-profile%2Fabdefg12334
so that it is treated as one part of the path.
Otherwise, the encoded endpoint will return 500 error when passed to Bedrock endpoint.
See the apis in https://docs.aws.amazon.com/bedrock/latest/APIReference/API_Operations_Amazon_Bedrock_Runtime.html
for more details on the regex patterns of modelId which we use in the regex logic below.
Args:
endpoint (str): The original endpoint string which may contain ARNs that contain slashes.
Returns:
str: The endpoint with properly encoded ARN slashes
"""
import re
# Early exit: if no ARN detected, return unchanged
if 'arn:aws:' not in endpoint:
return endpoint
# Handle all patterns in one go - more efficient and cleaner
patterns = [
# Custom model with 2 slashes (order matters - do this first)
(r'(custom-model)/([a-z0-9.-]+)/([a-z0-9]+)', r'\1%2F\2%2F\3'),
# All other resource types with 1 slash
(r'(:application-inference-profile)/', r'\1%2F'),
(r'(:inference-profile)/', r'\1%2F'),
(r'(:foundation-model)/', r'\1%2F'),
(r'(:imported-model)/', r'\1%2F'),
(r'(:provisioned-model)/', r'\1%2F'),
(r'(:prompt)/', r'\1%2F'),
(r'(:endpoint)/', r'\1%2F'),
(r'(:prompt-router)/', r'\1%2F'),
(r'(:default-prompt-router)/', r'\1%2F'),
]
for pattern, replacement in patterns:
# Check if pattern exists before applying regex (early exit optimization)
if re.search(pattern, endpoint):
endpoint = re.sub(pattern, replacement, endpoint)
break # Exit after first match since each ARN has only one resource type
return endpoint
litellm/proxy/_experimental/mcp_server/server.py
+14 -11
View File
@@ -40,6 +40,7 @@ except ImportError as e:
# Global variables to track initialization
_SESSION_MANAGERS_INITIALIZED = False
_INITIALIZATION_LOCK = asyncio.Lock()
if MCP_AVAILABLE:
from mcp.server import Server
@@ -113,21 +114,23 @@ if MCP_AVAILABLE:
"""Initialize the session managers. Can be called from main app lifespan."""
global _SESSION_MANAGERS_INITIALIZED, _session_manager_cm, _sse_session_manager_cm
if _SESSION_MANAGERS_INITIALIZED:
return
# Use async lock to prevent concurrent initialization
async with _INITIALIZATION_LOCK:
if _SESSION_MANAGERS_INITIALIZED:
return
verbose_logger.info("Initializing MCP session managers...")
verbose_logger.info("Initializing MCP session managers...")
# Start the session managers with context managers
_session_manager_cm = session_manager.run()
_sse_session_manager_cm = sse_session_manager.run()
# Start the session managers with context managers
_session_manager_cm = session_manager.run()
_sse_session_manager_cm = sse_session_manager.run()
# Enter the context managers
await _session_manager_cm.__aenter__()
await _sse_session_manager_cm.__aenter__()
# Enter the context managers
await _session_manager_cm.__aenter__()
await _sse_session_manager_cm.__aenter__()
_SESSION_MANAGERS_INITIALIZED = True
verbose_logger.info("MCP Server started with StreamableHTTP and SSE session managers!")
_SESSION_MANAGERS_INITIALIZED = True
verbose_logger.info("MCP Server started with StreamableHTTP and SSE session managers!")
async def shutdown_session_managers():
"""Shutdown the session managers."""
litellm/proxy/_experimental/out/_next/static/chunks/154-686bf7931b450e3d.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/154-7bf3bbb913e68f71.js
-1
View File
File diff suppressed because one or more lines are too long
ui/litellm-dashboard/out/_next/static/chunks/162-f2925685093720a4.js → litellm/proxy/_experimental/out/_next/static/chunks/162-8e70dcd28bbbec09.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/172-25e8f67ccf021150.js
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/172-c602ad2ba592907b.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/50-d0da2dd7acce2eb9.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/247-7557228b7131016b.js → litellm/proxy/_experimental/out/_next/static/chunks/682-52e4da3cfda27e50.js
+8 -8
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/85-52b9060394399707.js
-1
View File
File diff suppressed because one or more lines are too long
ui/litellm-dashboard/out/_next/static/chunks/866-9e1803a09e9ae8da.js → litellm/proxy/_experimental/out/_next/static/chunks/866-3523e0e07cf314f6.js
+1 -1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-25a743106e1c9456.js
+1
View File
@@ -0,0 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{6580:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=6580)}),_N_E=n.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-f4acf18888f0aa20.js
-1
View File
@@ -1 +0,0 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-72c8a8dbd0d3984f.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-6daa1df7fb2c7a9d.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[85,487,154,162,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,487,154,162,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-10c235fc18df224f.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-aef41992dbd4614b.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{64563:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[85,487,866,154,162,172,971,117,744],function(){return e(e.s=64563)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{58538:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,487,866,154,162,172,971,117,744],function(){return e(e.s=58538)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-3c5840c907b0a5c8.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-6b5d568180da3ccd.js
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/page-aeac1c2242be716a.js
+1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/app/page-c64580821d04b2c5.js
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_experimental/out/_next/static/chunks/main-app-4f7318ae681a6d94.js → litellm/proxy/_experimental/out/_next/static/chunks/main-app-475d6efe4080647d.js
+1 -1
View File
@@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{10264:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(10264)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{20169:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(20169)}),_N_E=e.O()}]);
litellm/proxy/_experimental/out/_next/static/css/618290a32b9bb91a.css
+3
View File
File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More