tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-02 10:21:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: add test_audio_speech_health_check_with_another_voice

Browse Source
This commit is contained in:
.mobo
2025-05-27 10:20:00 +02:00
committed by dotmobo
parent 32b2ebee9f 54fe927da7
commit 2c32ea375a
487 changed files with 24503 additions and 4182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+39 -29
View File
@@ -95,11 +95,11 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
@@ -218,11 +218,11 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
@@ -325,11 +325,11 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
@@ -454,7 +454,7 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
@@ -581,11 +581,11 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
@@ -657,7 +657,7 @@ jobs:
pip install --upgrade pip wheel setuptools
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
@@ -703,7 +703,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
- run:
name: Show current pydantic version
command: |
@@ -740,7 +740,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -783,7 +783,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "mcp==1.5.0"
# Run pytest and generate JUnit XML report
@@ -828,7 +828,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.34.34"
# Run pytest and generate JUnit XML report
@@ -873,7 +873,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -917,20 +917,28 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.5.0"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
- setup_litellm_enterprise_pip
# Run pytest and generate JUnit XML report
- run:
name: Run tests
name: Run litellm tests
command: |
pwd
ls
python -m pytest -vv tests/litellm tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=10
python -m pytest -vv tests/litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 4
no_output_timeout: 120m
- run:
name: Run enterprise tests
command: |
pwd
ls
python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -962,7 +970,7 @@ jobs:
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
@@ -1008,7 +1016,7 @@ jobs:
python -m pip install --upgrade pip
pip install numpydoc
python -m pip install -r requirements.txt
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
@@ -1058,7 +1066,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -1101,7 +1109,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -1145,10 +1153,11 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install pytest-mock
pip install "respx==0.21.1"
pip install "respx==0.22.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "mlflow==2.17.2"
pip install "anthropic==0.52.0"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -1340,6 +1349,7 @@ jobs:
- run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py
- run: python ./tests/documentation_tests/test_circular_imports.py
- run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
@@ -1472,7 +1482,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.68.2"
pip install "openai==1.81.0"
- run:
name: Install Grype
command: |
@@ -1610,7 +1620,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.68.2"
pip install "openai==1.81.0"
# Run pytest and generate JUnit XML report
- run:
name: Build Docker image
@@ -1733,7 +1743,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.68.2"
pip install "openai==1.81.0"
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -2256,7 +2266,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
pip install "openai==1.68.2"
pip install "openai==1.81.0"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
@@ -2275,7 +2285,7 @@ jobs:
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "google-cloud-aiplatform==1.59.0"
pip install "anthropic==0.49.0"
pip install "anthropic==0.52.0"
pip install "langchain_mcp_adapters==0.0.5"
pip install "langchain_openai==0.2.1"
pip install "langgraph==0.3.18"
@@ -2644,7 +2654,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.68.2"
pip install "openai==1.81.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
.circleci/requirements.txt
+1 -1
View File
@@ -1,5 +1,5 @@
# used by CI/CD testing
openai==1.68.2
openai==1.81.0
python-dotenv
tiktoken
importlib_metadata
.github/ISSUE_TEMPLATE/feature_request.yml
+3 -3
View File
@@ -23,10 +23,10 @@ body:
validations:
required: true
- type: dropdown
id: ml-ops-team
id: hiring-interest
attributes:
label: Are you a ML Ops Team?
description: This helps us prioritize your requests correctly
label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?
description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer
options:
- "No"
- "Yes"
.github/workflows/ghcr_deploy.yml
+2 -1
View File
@@ -115,7 +115,8 @@ jobs:
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }}
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
labels: ${{ steps.meta.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
.github/workflows/test-linting.yml
+2 -2
View File
@@ -22,9 +22,9 @@ jobs:
- name: Install dependencies
run: |
pip install openai==1.68.2
pip install openai==1.81.0
poetry install --with dev
pip install openai==1.68.2
pip install openai==1.81.0
.github/workflows/test-litellm.yml
+2 -1
View File
@@ -28,6 +28,7 @@ jobs:
- name: Install dependencies
run: |
poetry install --with dev,proxy-dev --extras proxy
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
- name: Setup litellm-enterprise as local package
run: |
@@ -36,4 +37,4 @@ jobs:
cd ..
- name: Run tests
run: |
poetry run pytest tests/litellm -x -vv -n 4
poetry run pytest tests/test_litellm -x -vv -n 4
.pre-commit-config.yaml
+2 -2
View File
@@ -24,9 +24,9 @@ repos:
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/litellm/|^tests/litellm/
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/
additional_dependencies: [flake8-print]
files: (litellm/|litellm_proxy_extras/).*\.py
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
Dockerfile
+2 -2
View File
@@ -51,7 +51,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
RUN apk add --no-cache openssl
RUN apk add --no-cache openssl tzdata
WORKDIR /app
# Copy the current directory contents into the container at /app
@@ -74,5 +74,5 @@ EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
CMD ["--port", "4000"]
deploy/charts/litellm-helm/Chart.yaml
+1 -1
View File
@@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.3
version: 0.4.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
deploy/charts/litellm-helm/README.md
+1
View File
@@ -34,6 +34,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` |
deploy/charts/litellm-helm/templates/service.yaml
+3
View File
@@ -10,6 +10,9 @@ metadata:
{{- include "litellm.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
{{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }}
loadBalancerClass: {{ .Values.service.loadBalancerClass }}
{{- end }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
deploy/charts/litellm-helm/tests/service_tests.yaml
+116
View File
@@ -0,0 +1,116 @@
suite: Service Configuration Tests
templates:
- service.yaml
tests:
- it: should create a default ClusterIP service
template: service.yaml
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- equal:
path: spec.ports[0].port
value: 4000
- equal:
path: spec.ports[0].targetPort
value: http
- equal:
path: spec.ports[0].protocol
value: TCP
- equal:
path: spec.ports[0].name
value: http
- isNull:
path: spec.loadBalancerClass
- it: should create a NodePort service when specified
template: service.yaml
set:
service.type: NodePort
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: NodePort
- isNull:
path: spec.loadBalancerClass
- it: should create a LoadBalancer service when specified
template: service.yaml
set:
service.type: LoadBalancer
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: LoadBalancer
- isNull:
path: spec.loadBalancerClass
- it: should add loadBalancerClass when specified with LoadBalancer type
template: service.yaml
set:
service.type: LoadBalancer
service.loadBalancerClass: tailscale
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: LoadBalancer
- equal:
path: spec.loadBalancerClass
value: tailscale
- it: should not add loadBalancerClass when specified with ClusterIP type
template: service.yaml
set:
service.type: ClusterIP
service.loadBalancerClass: tailscale
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- isNull:
path: spec.loadBalancerClass
- it: should use custom port when specified
template: service.yaml
set:
service.port: 8080
asserts:
- equal:
path: spec.ports[0].port
value: 8080
- it: should add service annotations when specified
template: service.yaml
set:
service.annotations:
cloud.google.com/load-balancer-type: "Internal"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
asserts:
- isKind:
of: Service
- equal:
path: metadata.annotations
value:
cloud.google.com/load-balancer-type: "Internal"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
- it: should use the correct selector labels
template: service.yaml
asserts:
- isNotNull:
path: spec.selector
- equal:
path: spec.selector
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
deploy/charts/litellm-helm/values.yaml
+3
View File
@@ -56,6 +56,9 @@ environmentConfigMaps: []
service:
type: ClusterIP
port: 4000
# If service type is `LoadBalancer` you can
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
ingress:
enabled: false
docker-compose.yml
+7 -12
View File
@@ -21,18 +21,13 @@ services:
env_file:
- .env # Load local .env file
depends_on:
- db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
healthcheck: # Defines the health check configuration for the container
test: [
"CMD",
"curl",
"-f",
"http://localhost:4000/health/liveliness || exit 1",
] # Command to execute for health check
interval: 30s # Perform health check every 30 seconds
timeout: 10s # Health check command times out after 10 seconds
retries: 3 # Retry up to 3 times if health check fails
start_period: 40s # Wait 40 seconds after container start before beginning health checks
- db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
healthcheck: # Defines the health check configuration for the container
test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ] # Command to execute for health check
interval: 30s # Perform health check every 30 seconds
timeout: 10s # Health check command times out after 10 seconds
retries: 3 # Retry up to 3 times if health check fails
start_period: 40s # Wait 40 seconds after container start before beginning health checks
db:
image: postgres:16
docs/my-website/docs/aiohttp_benchmarks.md
+38
View File
@@ -0,0 +1,38 @@
# LiteLLM v1.71.1 Benchmarks
## Overview
This document presents performance benchmarks comparing LiteLLM's v1.71.1 to prior litellm versions.
**Related PR:** [#11097](https://github.com/BerriAI/litellm/pull/11097)
## Testing Methodology
The load testing was conducted using the following parameters:
- **Request Rate:** 200 RPS (Requests Per Second)
- **User Ramp Up:** 200 concurrent users
- **Transport Comparison:** httpx (existing) vs aiohttp (new implementation)
- **Number of pods/instance of litellm:** 1
- **Machine Specs:** 2 vCPUs, 4GB RAM
- **LiteLLM Settings:**
- Tested against a [fake openai endpoint](https://exampleopenaiendpoint-production.up.railway.app/)
- Set `USE_AIOHTTP_TRANSPORT="True"` in the environment variables. This feature flag enables the aiohttp transport.
## Benchmark Results
| Metric | httpx (Existing) | aiohttp (LiteLLM v1.71.1) | Improvement | Calculation |
|--------|------------------|-------------------|-------------|-------------|
| **RPS** | 50.2 | 224 | **+346%** ✅ | (224 - 50.2) / 50.2 × 100 = 346% |
| **Median Latency** | 2,500ms | 74ms | **-97%** ✅ | (74 - 2500) / 2500 × 100 = -97% |
| **95th Percentile** | 5,600ms | 250ms | **-96%** ✅ | (250 - 5600) / 5600 × 100 = -96% |
| **99th Percentile** | 6,200ms | 330ms | **-95%** ✅ | (330 - 6200) / 6200 × 100 = -95% |
## Key Improvements
- **4.5x increase** in requests per second (from 50.2 to 224 RPS)
- **97% reduction** in median response time (from 2.5 seconds to 74ms)
- **96% reduction** in 95th percentile latency (from 5.6 seconds to 250ms)
- **95% reduction** in 99th percentile latency (from 6.2 seconds to 330ms)
docs/my-website/docs/benchmarks.md
+1 -3
View File
@@ -7,13 +7,11 @@ Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpo
Use this config for testing:
**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `aiohttp_openai/` provider for load testing.
```yaml
model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: aiohttp_openai/any
model: openai/any
api_base: https://your-fake-openai-endpoint.com/chat/completions
api_key: "test"
```
docs/my-website/docs/completion/input.md
+1 -1
View File
@@ -43,7 +43,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | | ✅ | ✅ | | | ✅ |
|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ |
|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
|xAI| ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
docs/my-website/docs/extras/contributing_code.md
+9 -6
View File
@@ -4,20 +4,23 @@
Here are the core requirements for any PR submitted to LiteLLM
- [ ] Sign the Contributor License Agreement (CLA) - [see details](#contributor-license-agreement-cla)
- [ ] Add testing, **Adding at least 1 test is a hard requirement** - [see details](#2-adding-testing-to-your-pr)
- [ ] Ensure your PR passes the following tests:
- [ ] [Unit Tests](#3-running-unit-tests)
- [ ] [Formatting / Linting Tests](#35-running-linting-tests)
- [ ] [Unit Tests](#3-running-unit-tests)
- [ ] [Formatting / Linting Tests](#35-running-linting-tests)
- [ ] Keep scope as isolated as possible. As a general rule, your changes should address 1 specific problem at a time
## **Contributor License Agreement (CLA)**
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](<(https://cla-assistant.io/BerriAI/litellm)>). This is a legal requirement for all contributions to be merged into the main repository. The CLA helps protect both you and the project by clearly defining the terms under which your contributions are made.
**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process. You can find the CLA [here](https://cla-assistant.io/BerriAI/litellm) and sign it through our CLA management system when you submit your first PR.
## Quick start
## 1. Setup your local dev environment
Here's how to modify the repo locally:
Step 1: Clone the repo
@@ -71,9 +74,9 @@ LiteLLM uses mypy for linting. On ci/cd we also run `black` for formatting.
- push your fork to your GitHub repo
- submit a PR from there
## Advanced
### Building LiteLLM Docker Image
### Building LiteLLM Docker Image
Some people might want to build the LiteLLM docker image themselves. Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
docs/my-website/docs/image_edits.md
+210
View File
@@ -0,0 +1,210 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# /images/edits
LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
| Feature | Supported | Notes |
|---------|-----------|--------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Supported operations | Create image edits | |
| Supported LiteLLM Versions | 1.63.8+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
## Usage
### LiteLLM Python SDK
<Tabs>
<TabItem value="openai" label="OpenAI">
#### Basic Image Edit
```python showLineNumbers title="OpenAI Image Edit"
import litellm
# Edit an image with a prompt
response = litellm.image_edit(
model="gpt-image-1",
image=open("original_image.png", "rb"),
prompt="Add a red hat to the person in the image",
n=1,
size="1024x1024"
)
print(response)
```
#### Image Edit with Mask
```python showLineNumbers title="OpenAI Image Edit with Mask"
import litellm
# Edit an image with a mask to specify the area to edit
response = litellm.image_edit(
model="gpt-image-1",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"), # Transparent areas will be edited
prompt="Replace the background with a beach scene",
n=2,
size="512x512",
response_format="url"
)
print(response)
```
#### Async Image Edit
```python showLineNumbers title="Async OpenAI Image Edit"
import litellm
import asyncio
async def edit_image():
response = await litellm.aimage_edit(
model="gpt-image-1",
image=open("original_image.png", "rb"),
prompt="Make the image look like a painting",
n=1,
size="1024x1024",
response_format="b64_json"
)
return response
# Run the async function
response = asyncio.run(edit_image())
print(response)
```
#### Image Edit with Custom Parameters
```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
import litellm
# Edit image with additional parameters
response = litellm.image_edit(
model="gpt-image-1",
image=open("portrait.png", "rb"),
prompt="Add sunglasses and a smile",
n=3,
size="1024x1024",
response_format="url",
user="user-123",
timeout=60,
extra_headers={"Custom-Header": "value"}
)
print(f"Generated {len(response.data)} image variations")
for i, image_data in enumerate(response.data):
print(f"Image {i+1}: {image_data.url}")
```
</TabItem>
</Tabs>
### LiteLLM Proxy with OpenAI SDK
<Tabs>
<TabItem value="openai" label="OpenAI">
First, add this to your litellm proxy config.yaml:
```yaml showLineNumbers title="OpenAI Proxy Configuration"
model_list:
- model_name: gpt-image-1
litellm_params:
model: gpt-image-1
api_key: os.environ/OPENAI_API_KEY
```
Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### Basic Image Edit via Proxy
```python showLineNumbers title="OpenAI Proxy Image Edit"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
# Edit an image
response = client.images.edit(
model="gpt-image-1",
image=open("original_image.png", "rb"),
prompt="Add a red hat to the person in the image",
n=1,
size="1024x1024"
)
print(response)
```
#### cURL Example
```bash showLineNumbers title="cURL Image Edit Request"
curl -X POST "http://localhost:4000/v1/images/edits" \
-H "Authorization: Bearer your-api-key" \
-F "model=gpt-image-1" \
-F "image=@original_image.png" \
-F "mask=@mask_image.png" \
-F "prompt=Add a beautiful sunset in the background" \
-F "n=1" \
-F "size=1024x1024" \
-F "response_format=url"
```
</TabItem>
</Tabs>
## Supported Image Edit Parameters
| Parameter | Type | Description | Required |
|-----------|------|-------------|----------|
| `image` | `FileTypes` | The image to edit. Must be a valid PNG file, less than 4MB, and square. | ✅ |
| `prompt` | `str` | A text description of the desired image edit. | ✅ |
| `model` | `str` | The model to use for image editing | Optional (defaults to `dall-e-2`) |
| `mask` | `str` | An additional image whose fully transparent areas indicate where the original image should be edited. Must be a valid PNG file, less than 4MB, and have the same dimensions as `image`. | Optional |
| `n` | `int` | The number of images to generate. Must be between 1 and 10. | Optional (defaults to 1) |
| `size` | `str` | The size of the generated images. Must be one of `256x256`, `512x512`, or `1024x1024`. | Optional (defaults to `1024x1024`) |
| `response_format` | `str` | The format in which the generated images are returned. Must be one of `url` or `b64_json`. | Optional (defaults to `url`) |
| `user` | `str` | A unique identifier representing your end-user. | Optional |
## Response Format
The response follows the OpenAI Images API format:
```python showLineNumbers title="Image Edit Response Structure"
{
"created": 1677649800,
"data": [
{
"url": "https://example.com/edited_image_1.png"
},
{
"url": "https://example.com/edited_image_2.png"
}
]
}
```
For `b64_json` format:
```python showLineNumbers title="Base64 Response Structure"
{
"created": 1677649800,
"data": [
{
"b64_json": "iVBORw0KGgoAAAANSUhEUgAA..."
}
]
}
```
docs/my-website/docs/observability/deepeval_integration.md
+55
View File
@@ -0,0 +1,55 @@
import Image from '@theme/IdealImage';
# 🔭 DeepEval - Open-Source Evals with Tracing
### What is DeepEval?
[DeepEval](https://deepeval.com) is an open-source evaluation framework for LLMs ([Github](https://github.com/confident-ai/deepeval)).
### What is Confident AI?
[Confident AI](https://documentation.confident-ai.com) (the ***deepeval*** platfrom) offers an Observatory for teams to trace and monitor LLM applications. Think Datadog for LLM apps. The observatory allows you to:
- Detect and debug issues in your LLM applications in real-time
- Search and analyze historical generation data with powerful filters
- Collect human feedback on model responses
- Run evaluations to measure and improve performance
- Track costs and latency to optimize resource usage
<Image img={require('../../img/deepeval_dashboard.png')} />
### Quickstart
```python
import os
import time
import litellm
os.environ['OPENAI_API_KEY']='<your-openai-api-key>'
os.environ['CONFIDENT_API_KEY']='<your-confident-api-key>'
litellm.success_callback = ["deepeval"]
litellm.failure_callback = ["deepeval"]
try:
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "What's the weather like in San Francisco?"}
],
)
except Exception as e:
print(e)
print(response)
```
:::info
You can obtain your `CONFIDENT_API_KEY` by logging into [Confident AI](https://app.confident-ai.com/project) platform.
:::
## Support & Talk with Deepeval team
- [Confident AI Docs 📝](https://documentation.confident-ai.com)
- [Platform 🚀](https://confident-ai.com)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Support ✉️ support@confident-ai.com
docs/my-website/docs/pass_through/vertex_ai.md
+1 -1
View File
@@ -116,7 +116,7 @@ curl \
<TabItem value="curl" label="curl">
```bash
curl http://localhost:4000/vertex_ai/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \
curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-1234" \
-d '{
docs/my-website/docs/providers/anthropic.md
+10 -5
View File
@@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
# Anthropic
LiteLLM supports all anthropic models.
- `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`)
- `claude-3.7` (`claude-3-7-sonnet-20250219`)
- `claude-3.5` (`claude-3-5-sonnet-20240620`)
- `claude-3` (`claude-3-haiku-20240307`, `claude-3-opus-20240229`, `claude-3-sonnet-20240229`)
- `claude-2`
@@ -64,7 +66,7 @@ from litellm import completion
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(model="claude-3-opus-20240229", messages=messages)
response = completion(model="claude-opus-4-20250514", messages=messages)
print(response)
```
@@ -80,7 +82,7 @@ from litellm import completion
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(model="claude-3-opus-20240229", messages=messages, stream=True)
response = completion(model="claude-opus-4-20250514", messages=messages, stream=True)
for chunk in response:
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
```
@@ -102,9 +104,9 @@ export ANTHROPIC_API_KEY="your-api-key"
```yaml
model_list:
- model_name: claude-3 ### RECEIVED MODEL NAME ###
- model_name: claude-4 ### RECEIVED MODEL NAME ###
litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
model: claude-3-opus-20240229 ### MODEL NAME sent to `litellm.completion()` ###
model: claude-opus-4-20250514 ### MODEL NAME sent to `litellm.completion()` ###
api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("AZURE_API_KEY_EU")
```
@@ -156,7 +158,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
<TabItem value="cli" label="cli">
```bash
$ litellm --model claude-3-opus-20240229
$ litellm --model claude-opus-4-20250514
# Server running on http://0.0.0.0:4000
```
@@ -244,6 +246,9 @@ print(response)
| Model Name | Function Call |
|------------------|--------------------------------------------|
| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-5-sonnet | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-haiku | `completion('claude-3-haiku-20240307', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-opus | `completion('claude-3-opus-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
docs/my-website/docs/providers/github.md
+11 -10
View File
@@ -7,6 +7,7 @@ https://github.com/marketplace/models
:::tip
**We support ALL Github models, just set `model=github/<any-model-on-github>` as a prefix when sending litellm requests**
Ignore company prefix: meta/Llama-3.2-11B-Vision-Instruct becomes model=github/Llama-3.2-11B-Vision-Instruct
:::
@@ -23,7 +24,7 @@ import os
os.environ['GITHUB_API_KEY'] = ""
response = completion(
model="github/llama3-8b-8192",
model="github/Llama-3.2-11B-Vision-Instruct",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@@ -38,7 +39,7 @@ import os
os.environ['GITHUB_API_KEY'] = ""
response = completion(
model="github/llama3-8b-8192",
model="github/Llama-3.2-11B-Vision-Instruct",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@@ -57,9 +58,9 @@ for chunk in response:
```yaml
model_list:
- model_name: github-llama3-8b-8192 # Model Alias to use for requests
- model_name: github-Llama-3.2-11B-Vision-Instruct # Model Alias to use for requests
litellm_params:
model: github/llama3-8b-8192
model: github/Llama-3.2-11B-Vision-Instruct
api_key: "os.environ/GITHUB_API_KEY" # ensure you have `GITHUB_API_KEY` in your .env
```
@@ -80,7 +81,7 @@ Make request to litellm proxy
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
"model": "github-llama3-8b-8192",
"model": "github-Llama-3.2-11B-Vision-Instruct",
"messages": [
{
"role": "user",
@@ -100,7 +101,7 @@ client = openai.OpenAI(
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(model="github-llama3-8b-8192", messages = [
response = client.chat.completions.create(model="github-Llama-3.2-11B-Vision-Instruct", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
@@ -124,7 +125,7 @@ from langchain.schema import HumanMessage, SystemMessage
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy
model = "github-llama3-8b-8192",
model = "github-Llama-3.2-11B-Vision-Instruct",
temperature=0.1
)
@@ -152,7 +153,7 @@ We support ALL Github models, just set `github/` as a prefix when sending comple
|--------------------|---------------------------------------------------------|
| llama-3.1-8b-instant | `completion(model="github/llama-3.1-8b-instant", messages)` |
| llama-3.1-70b-versatile | `completion(model="github/llama-3.1-70b-versatile", messages)` |
| llama3-8b-8192 | `completion(model="github/llama3-8b-8192", messages)` |
| Llama-3.2-11B-Vision-Instruct | `completion(model="github/Llama-3.2-11B-Vision-Instruct", messages)` |
| llama3-70b-8192 | `completion(model="github/llama3-70b-8192", messages)` |
| llama2-70b-4096 | `completion(model="github/llama2-70b-4096", messages)` |
| mixtral-8x7b-32768 | `completion(model="github/mixtral-8x7b-32768", messages)` |
@@ -214,7 +215,7 @@ tools = [
}
]
response = litellm.completion(
model="github/llama3-8b-8192",
model="github/Llama-3.2-11B-Vision-Instruct",
messages=messages,
tools=tools,
tool_choice="auto", # auto is default, but we'll be explicit
@@ -254,7 +255,7 @@ if tool_calls:
) # extend conversation with function response
print(f"messages: {messages}")
second_response = litellm.completion(
model="github/llama3-8b-8192", messages=messages
model="github/Llama-3.2-11B-Vision-Instruct", messages=messages
) # get a new response from the model where it can see the function response
print("second response\n", second_response)
```
docs/my-website/docs/providers/openai/responses_api.md
+130
View File
@@ -318,3 +318,133 @@ print(response)
</TabItem>
</Tabs>
## MCP Tools
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="MCP Tools with LiteLLM SDK"
import litellm
from typing import Optional
# Configure MCP Tools
MCP_TOOLS = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"allowed_tools": ["ask_question"]
}
]
# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval
response = litellm.responses(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input="What transport protocols does the 2025-03-26 version of the MCP spec support?"
)
# Get the MCP approval ID
mcp_approval_id = None
for output in response.output:
if output.type == "mcp_approval_request":
mcp_approval_id = output.id
break
# Step 2: Send followup with approval for the MCP call
response_with_mcp_call = litellm.responses(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input=[
{
"type": "mcp_approval_response",
"approve": True,
"approval_request_id": mcp_approval_id
}
],
previous_response_id=response.id,
)
print(response_with_mcp_call)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
1. Set up config.yaml
```yaml showLineNumbers title="OpenAI Proxy Configuration"
model_list:
- model_name: openai/gpt-4.1
litellm_params:
model: openai/gpt-4.1
api_key: os.environ/OPENAI_API_KEY
```
2. Start LiteLLM Proxy Server
```bash title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```python showLineNumbers title="MCP Tools with OpenAI SDK via LiteLLM Proxy"
from openai import OpenAI
from typing import Optional
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
# Configure MCP Tools
MCP_TOOLS = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"allowed_tools": ["ask_question"]
}
]
# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval
response = client.responses.create(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input="What transport protocols does the 2025-03-26 version of the MCP spec support?"
)
# Get the MCP approval ID
mcp_approval_id = None
for output in response.output:
if output.type == "mcp_approval_request":
mcp_approval_id = output.id
break
# Step 2: Send followup with approval for the MCP call
response_with_mcp_call = client.responses.create(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input=[
{
"type": "mcp_approval_response",
"approve": True,
"approval_request_id": mcp_approval_id
}
],
previous_response_id=response.id,
)
print(response_with_mcp_call)
```
</TabItem>
</Tabs>
docs/my-website/docs/providers/vertex.md
+2 -2
View File
@@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
| Description | Vertex AI is a fully-managed AI development platform for building and using generative AI. |
| Provider Route on LiteLLM | `vertex_ai/` |
| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) |
| Base URL | [https://{vertex_location}-aiplatform.googleapis.com/](https://{vertex_location}-aiplatform.googleapis.com/) |
| Base URL | 1. Regional endpoints<br/>[https://{vertex_location}-aiplatform.googleapis.com/](https://{vertex_location}-aiplatform.googleapis.com/)<br/>2. Global endpoints (limited availability)<br/>[https://aiplatform.googleapis.com/](https://{aiplatform.googleapis.com/)|
| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models) |
@@ -832,7 +832,7 @@ OR
You can set:
- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json
- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.)
- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.). Some models support the global location, please see [Vertex AI documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#supported_models)
- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials
as dynamic params for a `litellm.completion` call.
docs/my-website/docs/providers/voyage.md
+5 -3
View File
@@ -25,6 +25,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
| Model Name | Function Call |
|-------------------------|------------------------------------------------------------|
| voyage-3.5 | `embedding(model="voyage/voyage-3.5", input)` |
| voyage-3.5-lite | `embedding(model="voyage/voyage-3.5-lite", input)` |
| voyage-3-large | `embedding(model="voyage/voyage-3-large", input)` |
| voyage-3 | `embedding(model="voyage/voyage-3", input)` |
| voyage-3-lite | `embedding(model="voyage/voyage-3-lite", input)` |
@@ -35,8 +37,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
| voyage-multilingual-2 | `embedding(model="voyage/voyage-multilingual-2 ", input)` |
| voyage-large-2-instruct | `embedding(model="voyage/voyage-large-2-instruct", input)` |
| voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` |
| voyage-2 | `embedding(model="voyage/voyage-2", input)` |
| voyage-2 | `embedding(model="voyage/voyage-2", input)` |
| voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` |
| voyage-01 | `embedding(model="voyage/voyage-01", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-01 | `embedding(model="voyage/voyage-01", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` |
docs/my-website/docs/proxy/call_hooks.md
+2 -1
View File
@@ -44,7 +44,8 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
self,
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
pass
docs/my-website/docs/proxy/config_settings.md
+9
View File
@@ -293,6 +293,7 @@ router_settings:
| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
### environment variables - Reference
@@ -349,7 +350,9 @@ router_settings:
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
| CONFIG_FILE_PATH | File path for configuration file
| CONFIDENT_API_KEY | API key for DeepEval integration
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
| DATABASE_HOST | Hostname for the database server
| DATABASE_NAME | Name of the database
| DATABASE_PASSWORD | Password for the database user
@@ -384,6 +387,7 @@ router_settings:
| DEFAULT_IMAGE_TOKEN_COUNT | Default token count for images. Default is 250
| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300
| DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5
| DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL | Default time-to-live in seconds for management objects (User, Team, Key, Organization) in memory cache. Default is 60 seconds.
| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16
| DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100
| DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER | Default maximum recursion depth for sensitive data masker. Default is 10
@@ -505,6 +509,7 @@ router_settings:
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOG | Enable detailed logging for LiteLLM
@@ -513,6 +518,7 @@ router_settings:
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
@@ -526,7 +532,9 @@ router_settings:
| MAX_TILE_HEIGHT | Maximum height for image tiles. Default is 512
| MAX_TILE_WIDTH | Maximum width for image tiles. Default is 512
| MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10
| MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API
@@ -633,6 +641,7 @@ router_settings:
| UPSTREAM_LANGFUSE_PUBLIC_KEY | Public key for upstream Langfuse authentication
| UPSTREAM_LANGFUSE_RELEASE | Release version identifier for upstream Langfuse
| UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication
| USE_AIOHTTP_TRANSPORT | Flag to enable aiohttp transport. This is a feature flag for the new aiohttp transport. **Default is False**
| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
| USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
| WEBHOOK_URL | URL for receiving webhooks from external services
docs/my-website/docs/proxy/guardrails/pii_masking_v2.md
+1 -1
View File
@@ -418,7 +418,7 @@ guardrails:
Send ad-hoc recognizers to presidio `/analyze` by passing a json file to the proxy
[**Example** ad-hoc recognizer](../../../../litellm/proxy/hooks/example_presidio_ad_hoc_recognize)
[**Example** ad-hoc recognizer](https://github.com/BerriAI/litellm/blob/b69b7503db5aa039a49b7ca96ae5b34db0d25a3d/litellm/proxy/hooks/example_presidio_ad_hoc_recognizer.json)
#### Define ad-hoc recognizer on your LiteLLM config.yaml
docs/my-website/docs/proxy/litellm_managed_files.md
+154 -14
View File
@@ -4,7 +4,8 @@ import Image from '@theme/IdealImage';
# [BETA] LiteLLM Managed Files
Reuse the same file across different providers.
- Reuse the same file across different providers.
- Prevent users from seeing files they don't have access to on `list` and `retrieve` calls.
:::info
@@ -15,22 +16,18 @@ Available via the `litellm[proxy]` package or any `litellm` docker image.
:::
| Feature | Description | Comments |
| Property | Value | Comments |
| --- | --- | --- |
| Proxy | ✅ | |
| SDK | ❌ | Requires postgres DB for storing file ids |
| SDK | ❌ | Requires postgres DB for storing file ids. |
| Available across all providers | ✅ | |
| Supported endpoints | `/chat/completions`, `/batch`, `/fine_tuning` | |
Limitations of LiteLLM Managed Files:
- Only works for `/chat/completions` and `/batch` requests.
Follow [here](https://github.com/BerriAI/litellm/discussions/9632) for multiple models, batches support.
## Usage
### 1. Setup config.yaml
```
```yaml
model_list:
- model_name: "gemini-2.0-flash"
litellm_params:
@@ -41,6 +38,10 @@ model_list:
litellm_params:
model: gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-1234 # alternatively use the env var - LITELLM_MASTER_KEY
database_url: "postgresql://<user>:<password>@<host>:<port>/<dbname>" # alternatively use the env var - DATABASE_URL
```
### 2. Start proxy
@@ -219,8 +220,120 @@ print(completion.choices[0].message)
```
## File Permissions
### Supported Endpoints
Prevent users from seeing files they don't have access to on `list` and `retrieve` calls.
### 1. Setup config.yaml
```yaml
model_list:
- model_name: "gpt-4o-mini-openai"
litellm_params:
model: gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-1234 # alternatively use the env var - LITELLM_MASTER_KEY
database_url: "postgresql://<user>:<password>@<host>:<port>/<dbname>" # alternatively use the env var - DATABASE_URL
```
### 2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
### 3. Issue a key to the user
Let's create a user with the id `user_123`.
```bash
curl -L -X POST 'http://0.0.0.0:4000/user/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{"models": ["gpt-4o-mini-openai"], "user_id": "user_123"}'
```
Get the key from the response.
```json
{
"key": "sk-..."
}
```
### 4. User creates a file
#### 4a. Create a file
```jsonl
{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]}
{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]}
```
#### 4b. Upload the file
```python
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-...", # 👈 Use the key you generated in step 3
max_retries=0
)
# Upload file
finetuning_input_file = client.files.create(
file=open("./fine_tuning.jsonl", "rb"), # {"model": "azure-gpt-4o"} <-> {"model": "gpt-4o-my-special-deployment"}
purpose="fine-tune",
extra_body={"target_model_names": "gpt-4.1-openai"} # 👈 Tells litellm which regions/projects to write the file in.
)
print(finetuning_input_file) # file.id = "litellm_proxy/..." = {"model_name": {"deployment_id": "deployment_file_id"}}
```
### 5. User retrieves a file
<Tabs>
<TabItem value="has_access" label="User created file">
```python
from openai import OpenAI
... # User created file (3b)
file = client.files.retrieve(
file_id=finetuning_input_file.id
)
print(file) # File retrieved successfully
```
</TabItem>
<TabItem value="no_access" label="User did not create file">
```python
```python
from openai import OpenAI
... # User created file (3b)
try:
file = client.files.retrieve(
file_id="bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCwyYTgzOWIyYS03YzI1LTRiNTUtYTUxYS1lZjdhODljNzZkMzU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00by1iYXRjaA"
)
except Exception as e:
print(e) # User does not have access to this file
```
</TabItem>
</Tabs>
## Supported Endpoints
#### Create a file - `/files`
@@ -264,7 +377,23 @@ client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-1234", max_retries=0
file = client.files.delete(file_id=file.id)
```
### FAQ
#### List files - `/files`
```python
client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-1234", max_retries=0)
files = client.files.list(extra_body={"target_model_names": "gpt-4o-mini-openai"})
print(files) # All files user has created
```
Pre-GA Limitations on List Files:
- No multi-model support: Just 1 model name is supported for now.
- No multi-deployment support: Just 1 deployment of the model is supported for now (e.g. if you have 2 deployments with the `gpt-4o-mini-openai` public model name, it will pick one and return all files on that deployment).
Pre-GA Limitations will be fixed before GA of the Managed Files feature.
## FAQ
**1. Does LiteLLM store the file?**
@@ -278,10 +407,21 @@ LiteLLM stores a mapping of the litellm file id to the model-specific file id in
When a file is deleted, LiteLLM deletes the mapping from the postgres DB, and the files on each provider.
### Architecture
**4. Can a user call a file id that was created by another user?**
No, as of `v1.71.2` users can only view/edit/delete files they have created.
## Architecture
<Image img={require('../../img/managed_files_arch.png')} style={{ width: '800px', height: 'auto' }} />
<Image img={require('../../img/managed_files_arch.png')} style={{ width: '800px', height: 'auto' }} />
## See Also
- [Managed Files w/ Finetuning APIs](../../docs/proxy/managed_finetuning)
- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batch)
docs/my-website/docs/proxy/logging.md
+52
View File
@@ -11,6 +11,7 @@ Log Proxy input, output, and exceptions using:
- GCS, s3, Azure (Blob) Buckets
- Lunary
- MLflow
- Deepeval
- Custom Callbacks - Custom code and API endpoints
- Langsmith
- DataDog
@@ -1182,7 +1183,58 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
'
```
## Deepeval
LiteLLM supports logging on [Confidential AI](https://documentation.confident-ai.com/) (The Deepeval Platform):
### Usage:
1. Add `deepeval` in the LiteLLM `config.yaml`
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: gpt-4o
litellm_settings:
success_callback: ["deepeval"]
failure_callback: ["deepeval"]
```
2. Set your environment variables in `.env` file.
```shell
CONFIDENT_API_KEY=<your-api-key>
```
:::info
You can obtain your `CONFIDENT_API_KEY` by logging into [Confident AI](https://app.confident-ai.com/project) platform.
:::
3. Start your proxy server:
```shell
litellm --config config.yaml --debug
```
4. Make a request:
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "system",
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
},
{
"role": "user",
"content": "how can I solve 8x + 7 = -23"
}
]
}'
```
5. Check trace on platform:
<Image img={require('../../img/deepeval_visible_trace.png')} />
## s3 Buckets
docs/my-website/docs/proxy/managed_finetuning.md
+198
View File
@@ -0,0 +1,198 @@
# ✨ [BETA] LiteLLM Managed Files with Finetuning
:::info
This is a free LiteLLM Enterprise feature.
Available via the `litellm[proxy]` package or any `litellm` docker image.
:::
| Property | Value | Comments |
| --- | --- | --- |
| Proxy | ✅ | |
| SDK | ❌ | Requires postgres DB for storing file ids. |
| Available across all [Batch providers](../batches#supported-providers) | ✅ | |
| Supported endpoints | `/fine_tuning/jobs` | |
## Overview
Use this to:
- Create Finetuning jobs across OpenAI/Azure/Vertex AI in the OpenAI format (no additional `custom_llm_provider` param required).
- Control finetuning model access by key/user/team (same as chat completion models)
## (Proxy Admin) Usage
Here's how to give developers access to your Finetuning models.
### 1. Setup config.yaml
Include `/fine_tuning` in the `supported_endpoints` list. Tells developers this model supports the `/fine_tuning` endpoint.
```yaml showLineNumbers title="litellm_config.yaml"
model_list:
- model_name: "gpt-4.1-openai"
litellm_params:
model: gpt-4.1
api_key: os.environ/OPENAI_API_KEY
model_info:
supported_endpoints: ["/chat/completions", "/fine_tuning"]
```
### 2. Create Virtual Key
```bash showLineNumbers title="create_virtual_key.sh"
curl -L -X POST 'https://{PROXY_BASE_URL}/key/generate' \
-H 'Authorization: Bearer ${PROXY_API_KEY}' \
-H 'Content-Type: application/json' \
-d '{"models": ["gpt-4.1-openai"]}'
```
You can now use the virtual key to access the finetuning models (See Developer flow).
## (Developer) Usage
Here's how to create a LiteLLM managed file and execute Finetuning CRUD operations with the file.
### 1. Create request.jsonl
```json showLineNumbers title="request.jsonl"
{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]}
{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]}
```
### 2. Upload File
Specify `target_model_names: "<model-name>"` to enable LiteLLM managed files and request validation.
model-name should be the same as the model-name in the request.jsonl
```python showLineNumbers title="create_finetuning_job.py"
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-1234",
)
# Upload file
finetuning_input_file = client.files.create(
file=open("./request.jsonl", "rb"),
purpose="fine-tune",
extra_body={"target_model_names": "gpt-4.1-openai"}
)
print(finetuning_input_file)
```
**Where is the file written?**:
All gpt-4.1-openai deployments will be written to. This enables loadbalancing across all gpt-4.1-openai deployments in Step 3, when a job is created. Once the job is created, any retrieve/list/cancel operations will be routed to that deployment.
### 3. Create the Finetuning Job
```python showLineNumbers title="create_finetuning_job.py"
... # Step 2
file_id = finetuning_input_file.id
# Create Finetuning Job
ft_job = client.fine_tuning.jobs.create(
model="gpt-4.1-openai", # litellm public model name you want to finetune
training_file=file_id,
)
```
### 4. Retrieve Finetuning Job
```python showLineNumbers title="create_finetuning_job.py"
... # Step 3
response = client.fine_tuning.jobs.retrieve(ft_job.id)
print(response)
```
### 5. List Finetuning Jobs
```python showLineNumbers title="create_finetuning_job.py"
...
client.fine_tuning.jobs.list(extra_body={"target_model_names": "gpt-4.1-openai"})
```
### 6. Cancel a Finetuning Job
```python showLineNumbers title="create_finetuning_job.py"
...
cancel_ft_job = client.fine_tuning.jobs.cancel(
fine_tuning_job_id=ft_job.id, # fine tuning job id
)
```
## E2E Example
```python showLineNumbers title="create_finetuning_job.py"
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-...",
max_retries=0
)
# Upload file
finetuning_input_file = client.files.create(
file=open("./fine_tuning.jsonl", "rb"), # {"model": "azure-gpt-4o"} <-> {"model": "gpt-4o-my-special-deployment"}
purpose="fine-tune",
extra_body={"target_model_names": "gpt-4.1-openai"} # 👈 Tells litellm which regions/projects to write the file in.
)
print(finetuning_input_file) # file.id = "litellm_proxy/..." = {"model_name": {"deployment_id": "deployment_file_id"}}
file_id = finetuning_input_file.id
# # file_id = "bGl0ZWxs..."
# ## create fine-tuning job
ft_job = client.fine_tuning.jobs.create(
model="gpt-4.1-openai", # litellm model name you want to finetune
training_file=file_id,
)
print(f"ft_job: {ft_job}")
ft_job_id = ft_job.id
## cancel fine-tuning job
cancel_ft_job = client.fine_tuning.jobs.cancel(
fine_tuning_job_id=ft_job_id, # fine tuning job id
)
print("response from cancel ft job={}".format(cancel_ft_job))
# list fine-tuning jobs
list_ft_jobs = client.fine_tuning.jobs.list(
extra_query={"target_model_names": "gpt-4.1-openai"} # tell litellm proxy which provider to use
)
print("list of ft jobs={}".format(list_ft_jobs))
# get fine-tuning job
response = client.fine_tuning.jobs.retrieve(ft_job.id)
print(response)
```
## FAQ
### Where are my files written?
When a `target_model_names` is specified, the file is written to all deployments that match the `target_model_names`.
No additional infrastructure is required.
docs/my-website/docs/proxy/ui_logs_sessions.md
+5 -15
View File
@@ -43,9 +43,7 @@ response1 = client.chat.completions.create(
}
],
extra_body={
"metadata": {
"litellm_session_id": session_id # Pass the session ID
}
"litellm_session_id": session_id # Pass the session ID
}
)
```
@@ -64,9 +62,7 @@ response2 = client.chat.completions.create(
}
],
extra_body={
"metadata": {
"litellm_session_id": session_id # Reuse the same session ID
}
"litellm_session_id": session_id # Reuse the same session ID
}
)
```
@@ -89,9 +85,7 @@ chat = ChatOpenAI(
api_key="<your litellm api key>",
model="gpt-4o",
extra_body={
"metadata": {
"litellm_session_id": session_id # Pass the session ID
}
"litellm_session_id": session_id # Pass the session ID
}
)
@@ -132,9 +126,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"content": "Write a short story about a robot"
}
],
"metadata": {
"litellm_session_id": "'$SESSION_ID'"
}
"litellm_session_id": "'$SESSION_ID'"
}'
```
@@ -154,9 +146,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"content": "Now write a poem about that robot"
}
],
"metadata": {
"litellm_session_id": "'$SESSION_ID'"
}
"litellm_session_id": "'$SESSION_ID'"
}'
```
docs/my-website/img/deepeval_dashboard.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 639 KiB

docs/my-website/img/deepeval_visible_trace.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 598 KiB

docs/my-website/img/files_api_graphic.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 542 KiB

docs/my-website/img/perf_imp.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 190 KiB

docs/my-website/release_notes/v1.70.1-stable/index.md
+1 -1
View File
@@ -115,7 +115,7 @@ This release brings improvements to our Presidio PII Integration. As a Proxy Adm
- [`/messages`](../../docs/anthropic_unified)
- allow using dynamic AWS Params - [PR](https://github.com/BerriAI/litellm/pull/10769)
- **[Nvidia NIM](../../docs/providers/nvidia_nim)**
- [`/chat/completion`](../../docs/providers/nvidia_nim#usage---litellm-proxy-server) [NEED DOCS ON SUPPORTED PARAMS]
- [`/chat/completion`](../../docs/providers/nvidia_nim#usage---litellm-proxy-server)
- Add tools, tool_choice, parallel_tool_calls support - [PR](https://github.com/BerriAI/litellm/pull/10763)
- **[Novita AI](../../docs/providers/novita)**
- New Provider added for `/chat/completion` routes - [PR](https://github.com/BerriAI/litellm/pull/9527)
docs/my-website/release_notes/v1.71.1-stable/index.md
+284
View File
@@ -0,0 +1,284 @@
---
title: v1.71.1-stable - 2x Higher Requests Per Second (RPS)
slug: v1.71.1-stable
date: 2025-05-24T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQGrlsJ3aqpHmQ/profile-displayphoto-shrink_400_400/B4DZSAzgP7HYAg-/0/1737327772964?e=1749686400&v=beta&t=Hkl3U8Ps0VtvNxX0BNNq24b4dtX5wQaPFp6oiKCIHD8
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.71.1-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.71.1
```
</TabItem>
</Tabs>
## Key Highlights
LiteLLM v1.71.1-stable is live now. Here are the key highlights of this release:
- **Performance improvements**: LiteLLM can now scale to 200 RPS per instance with a 74ms median response time.
- **File Permissions**: Control file access across OpenAI, Azure, VertexAI.
- **MCP x OpenAI**: Use MCP servers with OpenAI Responses API.
## Performance Improvements
<Image img={require('../../img/perf_imp.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release brings aiohttp support for all LLM api providers. This means that LiteLLM can now scale to 200 RPS per instance with a 40ms median latency overhead.
This change doubles the RPS LiteLLM can scale to at this latency overhead.
You can opt into this by enabling the flag below. (We expect to make this the default in 1 week.)
### Flag to enable
**On LiteLLM Proxy**
Set the `USE_AIOHTTP_TRANSPORT=True` in the environment variables.
```yaml showLineNumbers title="Environment Variable"
export USE_AIOHTTP_TRANSPORT="True"
```
**On LiteLLM Python SDK**
Set the `use_aiohttp_transport=True` to enable aiohttp transport.
```python showLineNumbers title="Python SDK"
import litellm
litellm.use_aiohttp_transport = True # default is False, enable this to use aiohttp transport
result = litellm.completion(
model="openai/gpt-4o",
messages=[{"role": "user", "content": "Hello, world!"}],
)
print(result)
```
## File Permissions
<Image img={require('../../img/files_api_graphic.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release brings support for [File Permissions](../../docs/proxy/litellm_managed_files#file-permissions) and [Finetuning APIs](../../docs/proxy/managed_finetuning) to [LiteLLM Managed Files](../../docs/proxy/litellm_managed_files). This is great for:
- **Proxy Admins**: as users can only view/edit/delete files they’ve created - even when using shared OpenAI/Azure/Vertex deployments.
- **Developers**: get a standard interface to use Files across Chat/Finetuning/Batch APIs.
## New Models / Updated Models
- **Gemini [VertexAI](https://docs.litellm.ai/docs/providers/vertex), [Google AI Studio](https://docs.litellm.ai/docs/providers/gemini)**
- New gemini models - [PR 1](https://github.com/BerriAI/litellm/pull/10991), [PR 2](https://github.com/BerriAI/litellm/pull/10998)
- `gemini-2.5-flash-preview-tts`
- `gemini-2.0-flash-preview-image-generation`
- `gemini/gemini-2.5-flash-preview-05-20`
- `gemini-2.5-flash-preview-05-20`
- **[Anthropic](../../docs/providers/anthropic)**
- Claude-4 model family support - [PR](https://github.com/BerriAI/litellm/pull/11060)
- **[Bedrock](../../docs/providers/bedrock)**
- Claude-4 model family support - [PR](https://github.com/BerriAI/litellm/pull/11060)
- Support for `reasoning_effort` and `thinking` parameters for Claude-4 - [PR](https://github.com/BerriAI/litellm/pull/11114)
- **[VertexAI](../../docs/providers/vertex)**
- Claude-4 model family support - [PR](https://github.com/BerriAI/litellm/pull/11060)
- Global endpoints support - [PR](https://github.com/BerriAI/litellm/pull/10658)
- authorized_user credentials type support - [PR](https://github.com/BerriAI/litellm/pull/10899)
- **[xAI](../../docs/providers/xai)**
- `xai/grok-3` pricing information - [PR](https://github.com/BerriAI/litellm/pull/11028)
- **[LM Studio](../../docs/providers/lm_studio)**
- Structured JSON schema outputs support - [PR](https://github.com/BerriAI/litellm/pull/10929)
- **[SambaNova](../../docs/providers/sambanova)**
- Updated models and parameters - [PR](https://github.com/BerriAI/litellm/pull/10900)
- **[Databricks](../../docs/providers/databricks)**
- Llama 4 Maverick model cost - [PR](https://github.com/BerriAI/litellm/pull/11008)
- Claude 3.7 Sonnet output token cost correction - [PR](https://github.com/BerriAI/litellm/pull/11007)
- **[Azure](../../docs/providers/azure)**
- Mistral Medium 25.05 support - [PR](https://github.com/BerriAI/litellm/pull/11063)
- Certificate-based authentication support - [PR](https://github.com/BerriAI/litellm/pull/11069)
- **[Mistral](../../docs/providers/mistral)**
- devstral-small-2505 model pricing and context window - [PR](https://github.com/BerriAI/litellm/pull/11103)
- **[Ollama](../../docs/providers/ollama)**
- Wildcard model support - [PR](https://github.com/BerriAI/litellm/pull/10982)
- **[CustomLLM](../../docs/providers/custom_llm_server)**
- Embeddings support added - [PR](https://github.com/BerriAI/litellm/pull/10980)
- **[Featherless AI](../../docs/providers/featherless_ai)**
- Access to 4200+ models - [PR](https://github.com/BerriAI/litellm/pull/10596)
## LLM API Endpoints
- **[Image Edits](../../docs/image_generation)**
- `/v1/images/edits` - Support for /images/edits endpoint - [PR](https://github.com/BerriAI/litellm/pull/11020) [PR](https://github.com/BerriAI/litellm/pull/11123)
- Content policy violation error mapping - [PR](https://github.com/BerriAI/litellm/pull/11113)
- **[Responses API](../../docs/response_api)**
- MCP support for Responses API - [PR](https://github.com/BerriAI/litellm/pull/11029)
- **[Files API](../../docs/fine_tuning)**
- LiteLLM Managed Files support for finetuning - [PR](https://github.com/BerriAI/litellm/pull/11039) [PR](https://github.com/BerriAI/litellm/pull/11040)
- Validation for file operations (retrieve/list/delete) - [PR](https://github.com/BerriAI/litellm/pull/11081)
## Management Endpoints / UI
- **Teams**
- Key and member count display - [PR](https://github.com/BerriAI/litellm/pull/10950)
- Spend rounded to 4 decimal points - [PR](https://github.com/BerriAI/litellm/pull/11013)
- Organization and team create buttons repositioned - [PR](https://github.com/BerriAI/litellm/pull/10948)
- **Keys**
- Key reassignment and 'updated at' column - [PR](https://github.com/BerriAI/litellm/pull/10960)
- Show model access groups during creation - [PR](https://github.com/BerriAI/litellm/pull/10965)
- **Logs**
- Model filter on logs - [PR](https://github.com/BerriAI/litellm/pull/11048)
- Passthrough endpoint error logs support - [PR](https://github.com/BerriAI/litellm/pull/10990)
- **Guardrails**
- Config.yaml guardrails display - [PR](https://github.com/BerriAI/litellm/pull/10959)
- **Organizations/Users**
- Spend rounded to 4 decimal points - [PR](https://github.com/BerriAI/litellm/pull/11023)
- Show clear error when adding a user to a team - [PR](https://github.com/BerriAI/litellm/pull/10978)
- **Audit Logs**
- `/list` and `/info` endpoints for Audit Logs - [PR](https://github.com/BerriAI/litellm/pull/11102)
## Logging / Alerting Integrations
- **[Prometheus](../../docs/proxy/prometheus)**
- Track `route` on proxy_* metrics - [PR](https://github.com/BerriAI/litellm/pull/10992)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Support for `prompt_label` parameter - [PR](https://github.com/BerriAI/litellm/pull/11018)
- Consistent modelParams logging - [PR](https://github.com/BerriAI/litellm/pull/11018)
- **[DeepEval/ConfidentAI](../../docs/proxy/logging#deepeval)**
- Logging enabled for proxy and SDK - [PR](https://github.com/BerriAI/litellm/pull/10649)
- **[Logfire](../../docs/proxy/logging)**
- Fix otel proxy server initialization when using Logfire - [PR](https://github.com/BerriAI/litellm/pull/11091)
## Authentication & Security
- **[JWT Authentication](../../docs/proxy/token_auth)**
- Support for applying default internal user parameters when upserting a user via JWT authentication - [PR](https://github.com/BerriAI/litellm/pull/10995)
- Map a user to a team when upserting a user via JWT authentication - [PR](https://github.com/BerriAI/litellm/pull/11108)
- **Custom Auth**
- Support for switching between custom auth and API key auth - [PR](https://github.com/BerriAI/litellm/pull/11070)
## Performance / Reliability Improvements
- **aiohttp Transport**
- 97% lower median latency (feature flagged) - [PR](https://github.com/BerriAI/litellm/pull/11097) [PR](https://github.com/BerriAI/litellm/pull/11132)
- **Background Health Checks**
- Improved reliability - [PR](https://github.com/BerriAI/litellm/pull/10887)
- **Response Handling**
- Better streaming status code detection - [PR](https://github.com/BerriAI/litellm/pull/10962)
- Response ID propagation improvements - [PR](https://github.com/BerriAI/litellm/pull/11006)
- **Thread Management**
- Removed error-creating threads for reliability - [PR](https://github.com/BerriAI/litellm/pull/11066)
## General Proxy Improvements
- **[Proxy CLI](../../docs/proxy/cli)**
- Skip server startup flag - [PR](https://github.com/BerriAI/litellm/pull/10665)
- Avoid DATABASE_URL override when provided - [PR](https://github.com/BerriAI/litellm/pull/11076)
- **Model Management**
- Clear cache and reload after model updates - [PR](https://github.com/BerriAI/litellm/pull/10853)
- Computer use support tracking - [PR](https://github.com/BerriAI/litellm/pull/10881)
- **Helm Chart**
- LoadBalancer class support - [PR](https://github.com/BerriAI/litellm/pull/11064)
## Bug Fixes
This release includes numerous bug fixes to improve stability and reliability:
- **LLM Provider Fixes**
- VertexAI:
- Fixed quota_project_id parameter issue - [PR](https://github.com/BerriAI/litellm/pull/10915)
- Fixed credential refresh exceptions - [PR](https://github.com/BerriAI/litellm/pull/10969)
- Cohere:
Fixes for adding Cohere models through LiteLLM UI - [PR](https://github.com/BerriAI/litellm/pull/10822)
- Anthropic:
- Fixed streaming dict object handling for /v1/messages - [PR](https://github.com/BerriAI/litellm/pull/11032)
- OpenRouter:
- Fixed stream usage ID issues - [PR](https://github.com/BerriAI/litellm/pull/11004)
- **Authentication & Users**
- Fixed invitation email link generation - [PR](https://github.com/BerriAI/litellm/pull/10958)
- Fixed JWT authentication default role - [PR](https://github.com/BerriAI/litellm/pull/10995)
- Fixed user budget reset functionality - [PR](https://github.com/BerriAI/litellm/pull/10993)
- Fixed SSO user compatibility and email validation - [PR](https://github.com/BerriAI/litellm/pull/11106)
- **Database & Infrastructure**
- Fixed DB connection parameter handling - [PR](https://github.com/BerriAI/litellm/pull/10842)
- Fixed email invitation link - [PR](https://github.com/BerriAI/litellm/pull/11031)
- **UI & Display**
- Fixed MCP tool rendering when no arguments required - [PR](https://github.com/BerriAI/litellm/pull/11012)
- Fixed team model alias deletion - [PR](https://github.com/BerriAI/litellm/pull/11121)
- Fixed team viewer permissions - [PR](https://github.com/BerriAI/litellm/pull/11127)
- **Model & Routing**
- Fixed team model mapping in route requests - [PR](https://github.com/BerriAI/litellm/pull/11111)
- Fixed standard optional parameter passing - [PR](https://github.com/BerriAI/litellm/pull/11124)
## New Contributors
* [@DarinVerheijke](https://github.com/DarinVerheijke) made their first contribution in PR [#10596](https://github.com/BerriAI/litellm/pull/10596)
* [@estsauver](https://github.com/estsauver) made their first contribution in PR [#10929](https://github.com/BerriAI/litellm/pull/10929)
* [@mohittalele](https://github.com/mohittalele) made their first contribution in PR [#10665](https://github.com/BerriAI/litellm/pull/10665)
* [@pselden](https://github.com/pselden) made their first contribution in PR [#10899](https://github.com/BerriAI/litellm/pull/10899)
* [@unrealandychan](https://github.com/unrealandychan) made their first contribution in PR [#10842](https://github.com/BerriAI/litellm/pull/10842)
* [@dastaiger](https://github.com/dastaiger) made their first contribution in PR [#10946](https://github.com/BerriAI/litellm/pull/10946)
* [@slytechnical](https://github.com/slytechnical) made their first contribution in PR [#10881](https://github.com/BerriAI/litellm/pull/10881)
* [@daarko10](https://github.com/daarko10) made their first contribution in PR [#11006](https://github.com/BerriAI/litellm/pull/11006)
* [@sorenmat](https://github.com/sorenmat) made their first contribution in PR [#10658](https://github.com/BerriAI/litellm/pull/10658)
* [@matthid](https://github.com/matthid) made their first contribution in PR [#10982](https://github.com/BerriAI/litellm/pull/10982)
* [@jgowdy-godaddy](https://github.com/jgowdy-godaddy) made their first contribution in PR [#11032](https://github.com/BerriAI/litellm/pull/11032)
* [@bepotp](https://github.com/bepotp) made their first contribution in PR [#11008](https://github.com/BerriAI/litellm/pull/11008)
* [@jmorenoc-o](https://github.com/jmorenoc-o) made their first contribution in PR [#11031](https://github.com/BerriAI/litellm/pull/11031)
* [@martin-liu](https://github.com/martin-liu) made their first contribution in PR [#11076](https://github.com/BerriAI/litellm/pull/11076)
* [@gunjan-solanki](https://github.com/gunjan-solanki) made their first contribution in PR [#11064](https://github.com/BerriAI/litellm/pull/11064)
* [@tokoko](https://github.com/tokoko) made their first contribution in PR [#10980](https://github.com/BerriAI/litellm/pull/10980)
* [@spike-spiegel-21](https://github.com/spike-spiegel-21) made their first contribution in PR [#10649](https://github.com/BerriAI/litellm/pull/10649)
* [@kreatoo](https://github.com/kreatoo) made their first contribution in PR [#10927](https://github.com/BerriAI/litellm/pull/10927)
* [@baejooc](https://github.com/baejooc) made their first contribution in PR [#10887](https://github.com/BerriAI/litellm/pull/10887)
* [@keykbd](https://github.com/keykbd) made their first contribution in PR [#11114](https://github.com/BerriAI/litellm/pull/11114)
* [@dalssoft](https://github.com/dalssoft) made their first contribution in PR [#11088](https://github.com/BerriAI/litellm/pull/11088)
* [@jtong99](https://github.com/jtong99) made their first contribution in PR [#10853](https://github.com/BerriAI/litellm/pull/10853)
## Demo Instance
Here's a Demo Instance to test changes:
- Instance: https://demo.litellm.ai/
- Login Credentials:
- Username: admin
- Password: sk-1234
## [Git Diff](https://github.com/BerriAI/litellm/releases)
docs/my-website/sidebars.js
+10 -1
View File
@@ -217,6 +217,7 @@ const sidebars = {
label: "/images",
items: [
"image_generation",
"image_edits",
"image_variations",
]
},
@@ -266,7 +267,14 @@ const sidebars = {
]
},
"realtime",
"fine_tuning",
{
type: "category",
label: "/fine_tuning",
items: [
"fine_tuning",
"proxy/managed_finetuning",
]
},
"moderation",
"apply_guardrail",
],
@@ -460,6 +468,7 @@ const sidebars = {
"observability/agentops_integration",
"observability/langfuse_integration",
"observability/lunary_integration",
"observability/deepeval_integration",
"observability/mlflow",
"observability/gcs_bucket_integration",
"observability/langsmith_integration",
docs/my-website/static/llms-full.txt
+9164
View File
File diff suppressed because it is too large Load Diff
docs/my-website/static/llms.txt
+52
View File
@@ -0,0 +1,52 @@
# https://docs.litellm.ai/ llms.txt
- [LiteLLM Overview](https://docs.litellm.ai/): Access and manage 100+ LLMs with LiteLLM tools.
- [Completion Function Guide](https://docs.litellm.ai/completion/input): Guide for using completion function with various models.
- [Litellm Completion Function](https://docs.litellm.ai/completion/output): Learn about the litellm completion function and its output.
- [AI Completion Models](https://docs.litellm.ai/completion/supported): Explore various AI completion models and their requirements.
- [Contact Litellm](https://docs.litellm.ai/contact): Get in touch with Litellm for support and inquiries.
- [Contributing to Documentation](https://docs.litellm.ai/contributing): Guide for contributing to Litellm documentation and setup.
- [Supported Embedding Models](https://docs.litellm.ai/embedding/supported_embedding): Overview of supported embedding models and their requirements.
- [Docusaurus Setup Guide](https://docs.litellm.ai/intro): Quickly learn to set up a Docusaurus site.
- [Callbacks for Data Output](https://docs.litellm.ai/observability/callbacks): Learn to use callbacks for data output integration.
- [Helicone Integration Guide](https://docs.litellm.ai/observability/helicone_integration): Integrate Helicone for logging and proxying LLM requests.
- [Supabase Integration Guide](https://docs.litellm.ai/observability/supabase_integration): Learn to integrate Supabase for logging LLM requests.
- [LiteLLM Release Notes](https://docs.litellm.ai/release_notes): Explore the latest features and improvements in LiteLLM releases.
- [LiteLLM Release Notes](https://docs.litellm.ai/release_notes/archive): Comprehensive release notes for LiteLLM updates and features.
- [LiteLLM Release Tags](https://docs.litellm.ai/release_notes/tags): Explore various tags related to LiteLLM release notes.
- [LiteLLM Admin UI Updates](https://docs.litellm.ai/release_notes/tags/admin-ui): Explore LiteLLM's admin UI updates and new features.
- [Alerting Features Updates](https://docs.litellm.ai/release_notes/tags/alerting): Latest updates on alerting features and improvements.
- [LiteLLM Azure Storage Updates](https://docs.litellm.ai/release_notes/tags/azure-storage): Updates on LiteLLM Stable release and Azure Storage support.
- [Batch Processing Updates](https://docs.litellm.ai/release_notes/tags/batch): Updates on models, improvements, and integrations for batch processing.
- [Batches API Features](https://docs.litellm.ai/release_notes/tags/batches): Explore cost tracking, guardrails, and team management features.
- [Budgets and Rate Limits](https://docs.litellm.ai/release_notes/tags/budgets-rate-limits): Manage budgets and rate limits for LiteLLM keys effectively.
- [Claude 3.7 Sonnet Release](https://docs.litellm.ai/release_notes/tags/claude-3-7-sonnet): Release notes for Claude 3.7 Sonnet with updates.
- [Cost Tracking Features](https://docs.litellm.ai/release_notes/tags/cost-tracking): Explore cost tracking features, SCIM integration, and API updates.
- [Credential Management Updates](https://docs.litellm.ai/release_notes/tags/credential-management): Latest updates on credential management and LLM features.
- [Custom Auth Features](https://docs.litellm.ai/release_notes/tags/custom-auth): Explore custom authentication features for team management and cost tracking.
- [LiteLLM v1.65.0 Release](https://docs.litellm.ai/release_notes/tags/custom-prompt-management): New features and improvements in LiteLLM v1.65.0 release.
- [LiteLLM Release Notes](https://docs.litellm.ai/release_notes/tags/db-schema): Explore LiteLLM's latest updates and improvements in models.
- [Deepgram Release Notes](https://docs.litellm.ai/release_notes/tags/deepgram): Deepgram integration with speech, vision, and admin features.
- [Dependency Upgrades](https://docs.litellm.ai/release_notes/tags/dependency-upgrades): Dependency upgrades and new model support for LiteLLM.
- [Docker Image Release Notes](https://docs.litellm.ai/release_notes/tags/docker-image): LiteLLM Docker image updates for security and migration.
- [LiteLLM Release Notes](https://docs.litellm.ai/release_notes/tags/fallbacks): Updates on LiteLLM Stable release and new features.
- [Finetuning Updates and Improvements](https://docs.litellm.ai/release_notes/tags/finetuning): Explore finetuning updates, model improvements, and integrations.
- [Fireworks AI Updates](https://docs.litellm.ai/release_notes/tags/fireworks-ai): New features and updates for Fireworks AI models and tools.
- [Guardrails and Logging Updates](https://docs.litellm.ai/release_notes/tags/guardrails): Explore new guardrail features, logging, and model updates.
- [LLM Features and Updates](https://docs.litellm.ai/release_notes/tags/humanloop): Updates on models, integrations, and improvements in LLM features.
- [Key Management Overview](https://docs.litellm.ai/release_notes/tags/key-management): Manage keys, budgets, logging, and guardrails effectively.
- [LiteLLM Release Notes](https://docs.litellm.ai/release_notes/tags/langfuse): Explore new models, improvements, and integrations in LiteLLM.
- [LLM Translation Updates](https://docs.litellm.ai/release_notes/tags/llm-translation): Latest LLM translation updates and UI improvements released.
- [LiteLLM Logging Updates](https://docs.litellm.ai/release_notes/tags/logging): Explore LiteLLM logging updates, features, and improvements.
- [Management Endpoints Updates](https://docs.litellm.ai/release_notes/tags/management-endpoints): Updates on management endpoints for team model handling.
- [MCP Support Updates](https://docs.litellm.ai/release_notes/tags/mcp): MCP support and usage analytics enhancements in LiteLLM.
- [LiteLLM New Features](https://docs.litellm.ai/release_notes/tags/new-models): Explore new features, models, and updates for LiteLLM.
- [Prometheus Integration Updates](https://docs.litellm.ai/release_notes/tags/prometheus): Explore new features and improvements in Prometheus integration.
- [Prompt Management Updates](https://docs.litellm.ai/release_notes/tags/prompt-management): Explore prompt management updates, model improvements, and integrations.
- [LLM Translation Updates](https://docs.litellm.ai/release_notes/tags/reasoning-content): Release notes detailing LLM translation and UI improvements.
- [Release Notes Overview](https://docs.litellm.ai/release_notes/tags/rerank): Latest release notes on LLM translation and UI improvements.
- [Responses API Release Notes](https://docs.litellm.ai/release_notes/tags/responses-api): Explore the latest updates and features of the Responses API.
- [Secret Management Updates](https://docs.litellm.ai/release_notes/tags/secret-management): Enhancements in secret management, alerting, and model updates.
- [LiteLLM Security Updates](https://docs.litellm.ai/release_notes/tags/security): Security updates and features for LiteLLM deployment and management.
- [Session Management Updates](https://docs.litellm.ai/release_notes/tags/session-management): Enhancements in session management and user handling features.
- [LiteLLM Release Notes](https://docs.litellm.ai/release_notes/tags/snowflake): Latest updates on LiteLLM features and improvements.
enterprise/dist/litellm_enterprise-0.1.5-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
enterprise/dist/litellm_enterprise-0.1.5.tar.gz
Vendored
BIN
View File
Binary file not shown.
enterprise/dist/litellm_enterprise-0.1.6-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
enterprise/dist/litellm_enterprise-0.1.6.tar.gz
Vendored
BIN
View File
Binary file not shown.
enterprise/enterprise_hooks/__init__.py
-8
View File
@@ -1,22 +1,14 @@
import os
from typing import Dict, Literal, Type, Union
from litellm.integrations.custom_logger import CustomLogger
from .managed_files import _PROXY_LiteLLMManagedFiles
from .parallel_request_limiter_v2 import _PROXY_MaxParallelRequestsHandler
ENTERPRISE_PROXY_HOOKS: Dict[str, Type[CustomLogger]] = {
"managed_files": _PROXY_LiteLLMManagedFiles,
}
## FEATURE FLAG HOOKS ##
if os.getenv("EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING", "false").lower() == "true":
ENTERPRISE_PROXY_HOOKS["max_parallel_requests"] = _PROXY_MaxParallelRequestsHandler
def get_enterprise_proxy_hook(
hook_name: Union[
Literal[
enterprise/enterprise_hooks/managed_files.py
+273 -19
View File
@@ -1,29 +1,44 @@
# What is this?
## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id
import asyncio
import base64
import json
import uuid
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast
from fastapi import HTTPException
from litellm import Router, verbose_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth
from litellm.proxy._types import (
CallTypes,
LiteLLM_ManagedFileTable,
LiteLLM_ManagedObjectTable,
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
from litellm.types.llms.openai import (
AllMessageValues,
AsyncCursorPage,
ChatCompletionFileObject,
CreateFileRequest,
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.utils import LiteLLMBatch, LLMResponseTypes, SpecialEnums
from litellm.types.utils import (
LiteLLMBatch,
LiteLLMFineTuningJob,
LLMResponseTypes,
SpecialEnums,
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@@ -54,6 +69,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_object: OpenAIFileObject,
litellm_parent_otel_span: Optional[Span],
model_mappings: Dict[str, str],
user_api_key_dict: UserAPIKeyAuth,
) -> None:
verbose_logger.info(
f"Storing LiteLLM Managed File object with id={file_id} in cache"
@@ -62,6 +78,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
unified_file_id=file_id,
file_object=file_object,
model_mappings=model_mappings,
flat_model_file_ids=list(model_mappings.values()),
created_by=user_api_key_dict.user_id,
updated_by=user_api_key_dict.user_id,
)
await self.internal_usage_cache.async_set_cache(
key=file_id,
@@ -74,6 +93,44 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"unified_file_id": file_id,
"file_object": file_object.model_dump_json(),
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"updated_by": user_api_key_dict.user_id,
}
)
async def store_unified_object_id(
self,
unified_object_id: str,
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob],
litellm_parent_otel_span: Optional[Span],
model_object_id: str,
file_purpose: Literal["batch", "fine-tune"],
user_api_key_dict: UserAPIKeyAuth,
) -> None:
verbose_logger.info(
f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache"
)
litellm_managed_object = LiteLLM_ManagedObjectTable(
unified_object_id=unified_object_id,
model_object_id=model_object_id,
file_purpose=file_purpose,
file_object=file_object,
)
await self.internal_usage_cache.async_set_cache(
key=unified_object_id,
value=litellm_managed_object.model_dump(),
litellm_parent_otel_span=litellm_parent_otel_span,
)
await self.prisma_client.db.litellm_managedobjecttable.create(
data={
"unified_object_id": unified_object_id,
"file_object": file_object.model_dump_json(),
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": user_api_key_dict.user_id,
"updated_by": user_api_key_dict.user_id,
}
)
@@ -121,6 +178,71 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return initial_value.file_object
async def can_user_call_unified_file_id(
self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified file id
user_id = user_api_key_dict.user_id
managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
where={"unified_file_id": unified_file_id}
)
if managed_file:
return managed_file.created_by == user_id
return False
async def can_user_call_unified_object_id(
self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified object id
## check if the user has access to the unified object id
user_id = user_api_key_dict.user_id
managed_object = (
await self.prisma_client.db.litellm_managedobjecttable.find_first(
where={"unified_object_id": unified_object_id}
)
)
if managed_object:
return managed_object.created_by == user_id
return False
async def get_user_created_file_ids(
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
) -> List[OpenAIFileObject]:
"""
Get all file ids created by the user for a list of model object ids
Returns:
- List of OpenAIFileObject's
"""
file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many(
where={
"created_by": user_api_key_dict.user_id,
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids]
async def check_managed_file_id_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth
) -> bool:
retrieve_file_id = cast(Optional[str], data.get("file_id"))
potential_file_id = (
_is_base64_encoded_unified_file_id(retrieve_file_id)
if retrieve_file_id
else False
)
if potential_file_id and retrieve_file_id:
if await self.can_user_call_unified_file_id(
retrieve_file_id, user_api_key_dict
):
return True
else:
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}",
)
return False
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
@@ -137,13 +259,28 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"rerank",
"acreate_batch",
"aretrieve_batch",
"acreate_file",
"afile_list",
"afile_delete",
"afile_content",
"acreate_fine_tuning_job",
"aretrieve_fine_tuning_job",
"alist_fine_tuning_jobs",
"acancel_fine_tuning_job",
],
) -> Union[Exception, str, Dict, None]:
"""
- Detect litellm_proxy/ file_id
- add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}}
"""
### HANDLE FILE ACCESS ### - ensure user has access to the file
if (
call_type == CallTypes.afile_content.value
or call_type == CallTypes.afile_delete.value
):
await self.check_managed_file_id_access(data, user_api_key_dict)
### HANDLE TRANSFORMATIONS ###
if call_type == CallTypes.completion.value:
messages = data.get("messages")
if messages:
@@ -176,25 +313,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aretrieve_batch.value:
retrieve_batch_id = cast(Optional[str], data.get("batch_id"))
potential_batch_id = (
_is_base64_encoded_unified_file_id(retrieve_batch_id)
if retrieve_batch_id
elif (
call_type == CallTypes.aretrieve_batch.value
or call_type == CallTypes.acancel_fine_tuning_job.value
or call_type == CallTypes.aretrieve_fine_tuning_job.value
):
accessor_key: Optional[str] = None
retrieve_object_id: Optional[str] = None
if call_type == CallTypes.aretrieve_batch.value:
accessor_key = "batch_id"
elif (
call_type == CallTypes.acancel_fine_tuning_job.value
or call_type == CallTypes.aretrieve_fine_tuning_job.value
):
accessor_key = "fine_tuning_job_id"
if accessor_key:
retrieve_object_id = cast(Optional[str], data.get(accessor_key))
potential_llm_object_id = (
_is_base64_encoded_unified_file_id(retrieve_object_id)
if retrieve_object_id
else False
)
if potential_batch_id:
if potential_llm_object_id and retrieve_object_id:
## VALIDATE USER HAS ACCESS TO THE OBJECT ##
if not await self.can_user_call_unified_object_id(
retrieve_object_id, user_api_key_dict
):
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to the object {retrieve_object_id}",
)
## for managed batch id - get the model id
potential_model_id = self.get_model_id_from_unified_batch_id(
potential_batch_id
potential_llm_object_id
)
if potential_model_id is None:
raise Exception(
f"LiteLLM Managed Batch ID with id={retrieve_batch_id} is invalid - does not contain encoded model_id."
f"LiteLLM Managed {accessor_key} with id={retrieve_object_id} is invalid - does not contain encoded model_id."
)
data["model"] = potential_model_id
data["batch_id"] = self.get_batch_id_from_unified_batch_id(
potential_batch_id
data[accessor_key] = self.get_batch_id_from_unified_batch_id(
potential_llm_object_id
)
elif call_type == CallTypes.acreate_fine_tuning_job.value:
input_file_id = cast(Optional[str], data.get("training_file"))
if input_file_id:
model_file_id_mapping = await self.get_model_file_id_mapping(
[input_file_id], user_api_key_dict.parent_otel_span
)
return data
@@ -205,8 +373,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""
Allow modifying the request just before it's sent to the deployment.
"""
accessor_key: Optional[str] = None
if call_type and call_type == CallTypes.acreate_batch:
input_file_id = cast(Optional[str], kwargs.get("input_file_id"))
accessor_key = "input_file_id"
elif call_type and call_type == CallTypes.acreate_fine_tuning_job:
accessor_key = "training_file"
else:
return kwargs
if accessor_key:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
model_file_id_mapping = cast(
Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")
)
@@ -217,7 +393,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_id, None
)
if mapped_file_id:
kwargs["input_file_id"] = mapped_file_id
kwargs[accessor_key] = mapped_file_id
return kwargs
def get_file_ids_from_messages(self, messages: List[AllMessageValues]) -> List[str]:
@@ -305,6 +482,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
llm_router=llm_router,
@@ -337,6 +515,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_object=response,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
return response
@@ -383,6 +562,20 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return response
def get_unified_generic_response_id(
self, model_id: str, generic_response_id: str
) -> str:
unified_generic_response_id = (
SpecialEnums.LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR.value.format(
model_id, generic_response_id
)
)
return (
base64.urlsafe_b64encode(unified_generic_response_id.encode())
.decode()
.rstrip("=")
)
def get_unified_batch_id(self, batch_id: str, model_id: str) -> str:
unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(
model_id, batch_id
@@ -427,7 +620,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
def get_batch_id_from_unified_batch_id(self, file_id: str) -> str:
## use regex to get the batch_id from the file_id
return file_id.split("llm_batch_id:")[1].split(",")[0]
if "llm_batch_id" in file_id:
return file_id.split("llm_batch_id:")[1].split(",")[0]
else:
return file_id.split("generic_response_id:")[1].split(",")[0]
async def async_post_call_success_hook(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes
@@ -442,6 +638,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) # managed batch id
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
original_response_id = response.id
if (unified_batch_id or unified_file_id) and model_id:
response.id = self.get_unified_batch_id(
batch_id=response.id, model_id=model_id
@@ -455,10 +652,66 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_id=model_id,
model_name=model_name,
)
return await super().async_post_call_success_hook(
data, user_api_key_dict, response
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
)
)
elif isinstance(response, LiteLLMFineTuningJob):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get(
"unified_file_id"
) # managed file id
unified_finetuning_job_id = response._hidden_params.get(
"unified_finetuning_job_id"
) # managed finetuning job id
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
original_response_id = response.id
if (unified_file_id or unified_finetuning_job_id) and model_id:
response.id = self.get_unified_generic_response_id(
model_id=model_id, generic_response_id=response.id
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="fine-tune",
user_api_key_dict=user_api_key_dict,
)
)
elif isinstance(response, AsyncCursorPage):
"""
For listing files, filter for the ones created by the user
"""
## check if file object
if hasattr(response, "data") and isinstance(response.data, list):
if all(
isinstance(file_object, FileObject) for file_object in response.data
):
## Get all file id's
## Check which file id's were created by the user
## Filter the response to only include the files created by the user
## Return the filtered response
file_ids = [
file_object.id
for file_object in cast(List[FileObject], response.data) # type: ignore
]
user_created_file_ids = await self.get_user_created_file_ids(
user_api_key_dict, file_ids
)
## Filter the response to only include the files created by the user
response.data = user_created_file_ids # type: ignore
return response
return response
return response
async def afile_retrieve(
self, file_id: str, litellm_parent_otel_span: Optional[Span]
@@ -477,6 +730,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
litellm_parent_otel_span: Optional[Span],
**data: Dict,
) -> List[OpenAIFileObject]:
"""Handled in files_endpoints.py"""
return []
async def afile_delete(
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
+66 -6
View File
@@ -7,6 +7,12 @@ import json
import os
from typing import List, Optional
from litellm_enterprise.types.enterprise_callbacks.send_emails import (
EmailEvent,
EmailParams,
SendKeyCreatedEmailEvent,
)
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.email_templates.email_footer import EMAIL_FOOTER
@@ -17,10 +23,6 @@ from litellm.integrations.email_templates.user_invitation_email import (
USER_INVITATION_EMAIL_TEMPLATE,
)
from litellm.proxy._types import WebhookEvent
from litellm.types.enterprise.enterprise_callbacks.send_emails import (
EmailParams,
SendKeyCreatedEmailEvent,
)
from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
@@ -33,7 +35,9 @@ class BaseEmailLogger(CustomLogger):
Send email to user after inviting them to the team
"""
email_params = await self._get_email_params(
user_id=event.user_id, user_email=getattr(event, "user_email", None)
email_event=EmailEvent.new_user_invitation,
user_id=event.user_id,
user_email=getattr(event, "user_email", None),
)
# Implement invitation email logic using email_params
@@ -68,6 +72,7 @@ class BaseEmailLogger(CustomLogger):
email_params = await self._get_email_params(
user_id=send_key_created_email_event.user_id,
user_email=send_key_created_email_event.user_email,
email_event=EmailEvent.virtual_key_created,
)
verbose_proxy_logger.debug(
@@ -93,7 +98,10 @@ class BaseEmailLogger(CustomLogger):
pass
async def _get_email_params(
self, user_id: Optional[str] = None, user_email: Optional[str] = None
self,
email_event: EmailEvent,
user_id: Optional[str] = None,
user_email: Optional[str] = None,
) -> EmailParams:
"""
Get common email parameters used across different email sending methods
@@ -113,6 +121,12 @@ class BaseEmailLogger(CustomLogger):
f"User email not found for user_id: {user_id}. User email is required to send email."
)
# if user invited event then send invitation link
if email_event == EmailEvent.new_user_invitation:
base_url = await self._get_invitation_link(
user_id=user_id, base_url=base_url
)
return EmailParams(
logo_url=logo_url,
support_contact=support_contact,
@@ -148,6 +162,52 @@ class BaseEmailLogger(CustomLogger):
return user_row.user_email
return None
async def _get_invitation_link(self, user_id: Optional[str], base_url: str) -> str:
"""
Get invitation link for the user
"""
import asyncio
from litellm.proxy.proxy_server import prisma_client
################################################################################
########## Sleep for 10 seconds to wait for the invitation link to be created ###
################################################################################
# The UI, calls /invitation/new to generate the invitation link
# We wait 10 seconds to ensure the link is created
################################################################################
await asyncio.sleep(10)
if prisma_client is None:
verbose_proxy_logger.debug(
f"Prisma client not found. Unable to lookup user email for user_id: {user_id}"
)
return base_url
if user_id is None:
return base_url
# get the latest invitation link for the user
invitation_rows = await prisma_client.db.litellm_invitationlink.find_many(
where={"user_id": user_id},
order={"created_at": "desc"},
)
if len(invitation_rows) > 0:
invitation_row = invitation_rows[0]
return self._construct_invitation_link(
invitation_id=invitation_row.id, base_url=base_url
)
return base_url
def _construct_invitation_link(self, invitation_id: str, base_url: str) -> str:
"""
Construct invitation link for the user
# http://localhost:4000/ui?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b
"""
return f"{base_url}/ui?invitation_id={invitation_id}"
async def send_email(
self,
from_email: str,
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/endpoints.py
+5 -5
View File
@@ -6,11 +6,7 @@ import json
from typing import Dict
from fastapi import APIRouter, Depends, HTTPException
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.enterprise.enterprise_callbacks.send_emails import (
from litellm_enterprise.types.enterprise_callbacks.send_emails import (
DefaultEmailSettings,
EmailEvent,
EmailEventSettings,
@@ -18,6 +14,10 @@ from litellm.types.enterprise.enterprise_callbacks.send_emails import (
EmailEventSettingsUpdateRequest,
)
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
router = APIRouter()
enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py
+167
View File
@@ -0,0 +1,167 @@
"""
AUDIT LOGGING
All /audit logging endpoints. Attempting to write these as CRUD endpoints.
GET - /audit/{id} - Get audit log by id
GET - /audit - Get all audit logs
"""
from typing import Any, Dict, Optional
#### AUDIT LOGGING ####
from fastapi import APIRouter, Depends, HTTPException, Query
from litellm_enterprise.types.proxy.audit_logging_endpoints import (
AuditLogResponse,
PaginatedAuditLogResponse,
)
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
router = APIRouter()
@router.get(
"/audit",
tags=["Audit Logging"],
dependencies=[Depends(user_api_key_auth)],
response_model=PaginatedAuditLogResponse,
)
async def get_audit_logs(
page: int = Query(1, ge=1),
page_size: int = Query(10, ge=1, le=100),
# Filter parameters
changed_by: Optional[str] = Query(
None, description="Filter by user or system that performed the action"
),
changed_by_api_key: Optional[str] = Query(
None, description="Filter by API key hash that performed the action"
),
action: Optional[str] = Query(
None, description="Filter by action type (create, update, delete)"
),
table_name: Optional[str] = Query(
None, description="Filter by table name that was modified"
),
object_id: Optional[str] = Query(
None, description="Filter by ID of the object that was modified"
),
start_date: Optional[str] = Query(None, description="Filter logs after this date"),
end_date: Optional[str] = Query(None, description="Filter logs before this date"),
# Sorting parameters
sort_by: Optional[str] = Query(
None,
description="Column to sort by (e.g. 'updated_at', 'action', 'table_name')",
),
sort_order: str = Query("desc", description="Sort order ('asc' or 'desc')"),
):
"""
Get all audit logs with filtering and pagination.
Returns a paginated response of audit logs matching the specified filters.
"""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"message": CommonProxyErrors.db_not_connected_error.value},
)
# Build filter conditions
where_conditions: Dict[str, Any] = {}
if changed_by:
where_conditions["changed_by"] = changed_by
if changed_by_api_key:
where_conditions["changed_by_api_key"] = changed_by_api_key
if action:
where_conditions["action"] = action
if table_name:
where_conditions["table_name"] = table_name
if object_id:
where_conditions["object_id"] = object_id
if start_date or end_date:
date_filter = {}
if start_date:
date_filter["gte"] = start_date
if end_date:
date_filter["lte"] = end_date
where_conditions["updated_at"] = date_filter
# Build sort conditions
order_by = {}
if sort_by and isinstance(sort_by, str):
order_by[sort_by] = sort_order
elif sort_order and isinstance(sort_order, str):
order_by["updated_at"] = sort_order # Default sort by updated_at
# Get paginated results
audit_logs = await prisma_client.db.litellm_auditlog.find_many(
where=where_conditions,
order=order_by,
skip=(page - 1) * page_size,
take=page_size,
)
# Get total count for pagination
total_count = await prisma_client.db.litellm_auditlog.count(where=where_conditions)
total_pages = -(-total_count // page_size) # Ceiling division
# Return paginated response
return PaginatedAuditLogResponse(
audit_logs=[
AuditLogResponse(**audit_log.model_dump()) for audit_log in audit_logs
]
if audit_logs
else [],
total=total_count,
page=page,
page_size=page_size,
total_pages=total_pages,
)
@router.get(
"/audit/{id}",
tags=["Audit Logging"],
dependencies=[Depends(user_api_key_auth)],
response_model=AuditLogResponse,
responses={
404: {"description": "Audit log not found"},
500: {"description": "Database connection error"},
},
)
async def get_audit_log_by_id(
id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)
):
"""
Get detailed information about a specific audit log entry by its ID.
Args:
id (str): The unique identifier of the audit log entry
Returns:
AuditLogResponse: Detailed information about the audit log entry
Raises:
HTTPException: If the audit log is not found or if there's a database connection error
"""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"message": CommonProxyErrors.db_not_connected_error.value},
)
# Get the audit log by ID
audit_log = await prisma_client.db.litellm_auditlog.find_unique(where={"id": id})
if audit_log is None:
raise HTTPException(
status_code=404, detail={"message": f"Audit log with ID {id} not found"}
)
# Convert to response model
return AuditLogResponse(**audit_log.model_dump())
enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py
+33
View File
@@ -0,0 +1,33 @@
from typing import Any, Optional
from fastapi import Request
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
async def enterprise_custom_auth(
request: Request, api_key: str, user_custom_auth: Optional[Any]
) -> Optional[UserAPIKeyAuth]:
from litellm_enterprise.proxy.proxy_server import custom_auth_settings
if user_custom_auth is None:
return None
if custom_auth_settings is None:
return await user_custom_auth(request, api_key)
if custom_auth_settings["mode"] == "on":
return await user_custom_auth(request, api_key)
elif custom_auth_settings["mode"] == "off":
return None
elif custom_auth_settings["mode"] == "auto":
try:
return await user_custom_auth(request, api_key)
except Exception as e:
verbose_proxy_logger.debug(
f"Error in custom auth, checking litellm auth: {e}"
)
return None
else:
raise ValueError(f"Invalid mode: {custom_auth_settings['mode']}")
enterprise/litellm_enterprise/proxy/enterprise_routes.py
+2
View File
@@ -4,6 +4,7 @@ from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import (
router as email_events_router,
)
from .audit_logging_endpoints import router as audit_logging_router
from .guardrails.endpoints import router as guardrails_router
from .utils import _should_block_robots
from .vector_stores.endpoints import router as vector_stores_router
@@ -12,6 +13,7 @@ router = APIRouter()
router.include_router(vector_stores_router)
router.include_router(guardrails_router)
router.include_router(email_events_router)
router.include_router(audit_logging_router)
@router.get("/robots.txt")
enterprise/litellm_enterprise/proxy/proxy_server.py
+22
View File
@@ -0,0 +1,22 @@
from typing import Optional
from litellm_enterprise.types.proxy.proxy_server import CustomAuthSettings
custom_auth_settings: Optional[CustomAuthSettings] = None
class EnterpriseProxyConfig:
async def load_custom_auth_settings(
self, general_settings: dict
) -> CustomAuthSettings:
custom_auth_settings = general_settings.get("custom_auth_settings", None)
if custom_auth_settings is not None:
custom_auth_settings = CustomAuthSettings(
mode=custom_auth_settings.get("mode"),
)
return custom_auth_settings
async def load_enterprise_config(self, general_settings: dict) -> None:
global custom_auth_settings
custom_auth_settings = await self.load_custom_auth_settings(general_settings)
return None
enterprise/litellm_enterprise/proxy/readme.md
+5
View File
@@ -4,3 +4,8 @@
This directory contains enterprise features used on the LiteLLM proxy.
## Format
Create a file for every group of endpoints (e.g. `key_management_endpoints.py`, `user_management_endpoints.py`, etc.)
If there is a broader semantic group of endpoints, create a folder for that group (e.g. `management_endpoints`, `auth_endpoints`, etc.)
litellm/types/enterprise/enterprise_callbacks/send_emails.py → enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py
View File
enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py
+30
View File
@@ -0,0 +1,30 @@
from datetime import datetime
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
class AuditLogResponse(BaseModel):
"""Response model for a single audit log entry"""
id: str
updated_at: datetime
changed_by: str
changed_by_api_key: str
action: str
table_name: str
object_id: str
before_value: Optional[Dict[str, Any]] = None
updated_values: Optional[Dict[str, Any]] = None
class PaginatedAuditLogResponse(BaseModel):
"""Response model for paginated audit logs"""
audit_logs: List[AuditLogResponse]
total: int = Field(
..., description="Total number of audit logs matching the filters"
)
page: int = Field(..., description="Current page number")
page_size: int = Field(..., description="Number of items per page")
total_pages: int = Field(..., description="Total number of pages")
enterprise/litellm_enterprise/types/proxy/proxy_server.py
+5
View File
@@ -0,0 +1,5 @@
from typing import Literal, TypedDict
class CustomAuthSettings(TypedDict):
mode: Literal["on", "off", "auto"]
enterprise/poetry.lock
Generated
+2 -2
View File
@@ -1,7 +1,7 @@
# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
package = []
[metadata]
lock-version = "2.1"
lock-version = "2.0"
python-versions = ">=3.8.1,<4.0, !=3.9.7"
content-hash = "2cf39473e67ff0615f0a61c9d2ac9f02b38cc08cbb1bdb893d89bee002646623"
enterprise/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.4"
version = "0.1.6"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.4"
version = "0.1.6"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",
litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql
+32
View File
@@ -0,0 +1,32 @@
-- AlterTable
ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN "created_by" TEXT,
ADD COLUMN "flat_model_file_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
ADD COLUMN "updated_by" TEXT;
-- CreateTable
CREATE TABLE "LiteLLM_ManagedObjectTable" (
"id" TEXT NOT NULL,
"unified_object_id" TEXT NOT NULL,
"model_object_id" TEXT NOT NULL,
"file_object" JSONB NOT NULL,
"file_purpose" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_ManagedObjectTable_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_key" ON "LiteLLM_ManagedObjectTable"("unified_object_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_model_object_id_key" ON "LiteLLM_ManagedObjectTable"("model_object_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_idx" ON "LiteLLM_ManagedObjectTable"("unified_object_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedObjectTable_model_object_id_idx" ON "LiteLLM_ManagedObjectTable"("model_object_id");
litellm-proxy-extras/litellm_proxy_extras/migrations/20250526154401_allow_null_entity_id/migration.sql
+9
View File
@@ -0,0 +1,9 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ALTER COLUMN "tag" DROP NOT NULL;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ALTER COLUMN "team_id" DROP NOT NULL;
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ALTER COLUMN "user_id" DROP NOT NULL;
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+21 -4
View File
@@ -356,7 +356,7 @@ model LiteLLM_AuditLog {
// Track daily user spend metrics per model and key
model LiteLLM_DailyUserSpend {
id String @id @default(uuid())
user_id String
user_id String?
date String
api_key String
model String
@@ -383,7 +383,7 @@ model LiteLLM_DailyUserSpend {
// Track daily team spend metrics per model and key
model LiteLLM_DailyTeamSpend {
id String @id @default(uuid())
team_id String
team_id String?
date String
api_key String
model String
@@ -410,7 +410,7 @@ model LiteLLM_DailyTeamSpend {
// Track daily team spend metrics per model and key
model LiteLLM_DailyTagSpend {
id String @id @default(uuid())
tag String
tag String?
date String
api_key String
model String
@@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable {
id String @id @default(uuid())
unified_file_id String @unique // The base64 encoded unified file ID
file_object Json // Stores the OpenAIFileObject
model_mappings Json // Stores the mapping of model_id -> provider_file_id
model_mappings Json
flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified file ID
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
}
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.1.21"
version = "0.2.0"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.21"
version = "0.2.0"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/__init__.py
+56 -57
View File
@@ -118,6 +118,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"generic_api",
"resend_email",
"smtp_email",
"deepeval",
]
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
_known_custom_logger_compatible_callbacks: List = list(
@@ -126,28 +127,29 @@ _known_custom_logger_compatible_callbacks: List = list(
callbacks: List[
Union[Callable, _custom_logger_compatible_callbacks_literal, CustomLogger]
] = []
initialized_langfuse_clients: int = 0
langfuse_default_tags: Optional[List[str]] = None
langsmith_batch_size: Optional[int] = None
prometheus_initialize_budget_metrics: Optional[bool] = False
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload
gcs_pub_sub_use_v1: Optional[bool] = (
False # if you want to use v1 gcs pubsub logged payload
)
generic_api_use_v1: Optional[bool] = (
False # if you want to use v1 generic api logged payload
)
gcs_pub_sub_use_v1: Optional[
bool
] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[
bool
] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
_async_input_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_success_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_input_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_success_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
@@ -155,18 +157,18 @@ log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
)
add_user_information_to_llm_headers: Optional[
bool
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
store_audit_logs = False # Enterprise feature, allow users to see audit logs
### end of callbacks #############
email: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
token: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
email: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
token: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
@@ -247,24 +249,20 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
enable_caching_on_provider_specific_optional_params: bool = (
False # feature-flag for caching on optional params - e.g. 'top_k'
)
caching: bool = (
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
cache: Optional[Cache] = (
None # cache object <- use this - https://docs.litellm.ai/docs/caching
)
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
cache: Optional[
Cache
] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_alias_map: Dict[str, str] = {}
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[str] = (
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
)
budget_duration: Optional[
str
] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
default_soft_budget: float = (
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
)
@@ -273,15 +271,11 @@ forward_traceparent_to_llm_provider: bool = False
_current_cost = 0.0 # private variable, used if max budget is set
error_logs: Dict = {}
add_function_to_prompt: bool = (
False # if function calling not supported by api, append function call details to system prompt
)
add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
client_session: Optional[httpx.Client] = None
aclient_session: Optional[httpx.AsyncClient] = None
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
model_cost_map_url: str = (
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
)
model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
@@ -304,9 +298,10 @@ disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
custom_prometheus_metadata_labels: List[str] = []
#### REQUEST PRIORITIZATION ####
priority_reservation: Optional[Dict[str, float]] = None
force_ipv4: bool = (
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
)
######## Networking Settings ########
use_aiohttp_transport: bool = True
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
module_level_aclient = AsyncHTTPHandler(
timeout=request_timeout, client_alias="module level aclient"
)
@@ -320,13 +315,13 @@ fallbacks: Optional[List] = None
context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
num_retries_per_request: Optional[int] = (
None # for the request overall (incl. fallbacks + model retries)
)
num_retries_per_request: Optional[
int
] = None # for the request overall (incl. fallbacks + model retries)
####### SECRET MANAGERS #####################
secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
)
secret_manager_client: Optional[
Any
] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional[KeyManagementSystem] = None
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
@@ -364,6 +359,8 @@ project = None
config_path = None
vertex_ai_safety_settings: Optional[dict] = None
BEDROCK_CONVERSE_MODELS = [
"anthropic.claude-opus-4-20250514-v1:0",
"anthropic.claude-sonnet-4-20250514-v1:0",
"anthropic.claude-3-7-sonnet-20250219-v1:0",
"anthropic.claude-3-5-haiku-20241022-v1:0",
"anthropic.claude-3-5-sonnet-20241022-v2:0",
@@ -448,6 +445,7 @@ snowflake_models: List = []
llama_models: List = []
nscale_models: List = []
def is_bedrock_pricing_only_model(key: str) -> bool:
"""
Excludes keys with the pattern 'bedrock/<region>/<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
@@ -1091,6 +1089,7 @@ from .proxy.proxy_cli import run_server
from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .batch_completion.main import * # type: ignore
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
@@ -1117,10 +1116,10 @@ from .types.llms.custom_llm import CustomLLMItem
from .types.utils import GenericStreamingChunk
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[str] = (
[]
) # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
None # disable huggingface tokenizer download. Defaults to openai clk100
)
_custom_providers: List[
str
] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[
bool
] = None # disable huggingface tokenizer download. Defaults to openai clk100
global_disable_no_log_param: bool = False
litellm/_service_logger.py
+1
View File
@@ -276,6 +276,7 @@ class ServiceLogging(CustomLogger):
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
"""
Hook to track failed litellm-service calls
litellm/caching/in_memory_cache.py
+18 -4
View File
@@ -106,6 +106,18 @@ class InMemoryCache(BaseCache):
# One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
# This can occur when an object is referenced by another object, but the reference is never removed.
def allow_ttl_override(self, key: str) -> bool:
"""
Check if ttl is set for a key
"""
ttl_time = self.ttl_dict.get(key)
if ttl_time is None: # if ttl is not set, allow override
return True
elif float(ttl_time) < time.time(): # if ttl is expired, allow override
return True
else:
return False
def set_cache(self, key, value, **kwargs):
if len(self.cache_dict) >= self.max_size_in_memory:
# only evict when cache is full
@@ -114,10 +126,11 @@ class InMemoryCache(BaseCache):
return
self.cache_dict[key] = value
if "ttl" in kwargs and kwargs["ttl"] is not None:
self.ttl_dict[key] = time.time() + kwargs["ttl"]
else:
self.ttl_dict[key] = time.time() + self.default_ttl
if self.allow_ttl_override(key): # if ttl is not set, set it to default ttl
if "ttl" in kwargs and kwargs["ttl"] is not None:
self.ttl_dict[key] = time.time() + kwargs["ttl"]
else:
self.ttl_dict[key] = time.time() + self.default_ttl
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
@@ -145,6 +158,7 @@ class InMemoryCache(BaseCache):
if key in self.ttl_dict:
if time.time() > self.ttl_dict[key]:
self.cache_dict.pop(key, None)
self.ttl_dict.pop(key, None)
return None
original_cached_response = self.cache_dict[key]
try:
litellm/constants.py
+53 -1
View File
@@ -141,6 +141,7 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
STREAM_SSE_DONE_STRING: str = "[DONE]"
STREAM_SSE_DATA_PREFIX: str = "data: "
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = float(
os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@@ -152,13 +153,18 @@ FIREWORKS_AI_16_B = int(os.getenv("FIREWORKS_AI_16_B", 16))
FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80))
#### Logging callback constants ####
REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM"
MAX_LANGFUSE_INITIALIZED_CLIENTS = int(
os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 20)
)
############### LLM Provider Constants ###############
### ANTHROPIC CONSTANTS ###
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = {
"low": 1,
"medium": 5,
"high": 10,
}
DEFAULT_IMAGE_ENDPOINT_MODEL = "dall-e-2"
LITELLM_CHAT_PROVIDERS = [
"openai",
@@ -230,7 +236,7 @@ LITELLM_CHAT_PROVIDERS = [
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
"openai",
"azure",
"hosted_vllm"
"hosted_vllm",
]
@@ -276,6 +282,46 @@ OPENAI_CHAT_COMPLETION_PARAMS = [
"web_search_options",
]
DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
"functions": None,
"function_call": None,
"temperature": None,
"top_p": None,
"n": None,
"stream": None,
"stream_options": None,
"stop": None,
"max_tokens": None,
"max_completion_tokens": None,
"modalities": None,
"prediction": None,
"audio": None,
"presence_penalty": None,
"frequency_penalty": None,
"logit_bias": None,
"user": None,
"model": None,
"custom_llm_provider": "",
"response_format": None,
"seed": None,
"tools": None,
"tool_choice": None,
"max_retries": None,
"logprobs": None,
"top_logprobs": None,
"extra_headers": None,
"api_version": None,
"parallel_tool_calls": None,
"drop_params": None,
"allowed_openai_params": None,
"additional_drop_params": None,
"messages": None,
"reasoning_effort": None,
"thinking": None,
"web_search_options": None,
}
openai_compatible_endpoints: List = [
"api.perplexity.ai",
"api.endpoints.anyscale.com/v1",
@@ -593,6 +639,7 @@ PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = int(
os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5)
)
MCP_TOOL_NAME_PREFIX = "mcp_tool"
MAXIMUM_TRACEBACK_LINES_TO_LOG = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100))
########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################
@@ -665,3 +712,8 @@ LENGTH_OF_LITELLM_GENERATED_KEY = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY
SECRET_MANAGER_REFRESH_INTERVAL = int(
os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400)
)
LITELLM_SETTINGS_SAFE_DB_OVERRIDES = ["default_internal_user_params"]
SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
)
litellm/fine_tuning/main.py
+12 -11
View File
@@ -22,12 +22,9 @@ from litellm.llms.azure.fine_tuning.handler import AzureOpenAIFineTuningAPI
from litellm.llms.openai.fine_tuning.handler import OpenAIFineTuningAPI
from litellm.llms.vertex_ai.fine_tuning.handler import VertexFineTuningAPI
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
FineTuningJob,
FineTuningJobCreate,
Hyperparameters,
)
from litellm.types.llms.openai import FineTuningJobCreate, Hyperparameters
from litellm.types.router import *
from litellm.types.utils import LiteLLMFineTuningJob
from litellm.utils import client, supports_httpx_timeout
####### ENVIRONMENT VARIABLES ###################
@@ -50,7 +47,7 @@ async def acreate_fine_tuning_job(
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
"""
Async: Creates and executes a batch from an uploaded file of request
@@ -104,7 +101,7 @@ def create_fine_tuning_job(
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
"""
Creates a fine-tuning job which begins the process of creating a new model from a given dataset.
@@ -288,13 +285,14 @@ def create_fine_tuning_job(
raise e
@client
async def acancel_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
"""
Async: Immediately cancel a fine-tune job.
"""
@@ -325,13 +323,14 @@ async def acancel_fine_tuning_job(
raise e
@client
def cancel_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
"""
Immediately cancel a fine-tune job.
@@ -609,13 +608,14 @@ def list_fine_tuning_jobs(
raise e
@client
async def aretrieve_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
"""
Async: Get info about a fine-tuning job.
"""
@@ -646,13 +646,14 @@ async def aretrieve_fine_tuning_job(
raise e
@client
def retrieve_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
"""
Get info about a fine-tuning job.
"""
litellm/images/__init__.py
View File
litellm/images/main.py
+736
View File
@@ -0,0 +1,736 @@
import asyncio
import contextvars
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
import litellm
from litellm import Logging, client, exception_type, get_litellm_params
from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_llm import CustomLLM
#################### Initialize provider clients ####################
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
base_llm_http_handler,
bedrock_image_generation,
openai_chat_completions,
openai_image_variations,
vertex_image_generation,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import ImageGenerationRequestQuality
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
FileTypes,
LlmProviders,
all_litellm_params,
)
from litellm.utils import (
ImageResponse,
ProviderConfigManager,
get_llm_provider,
get_optional_params_image_gen,
)
from .utils import ImageEditRequestUtils
##### Image Generation #######################
@client
async def aimage_generation(*args, **kwargs) -> ImageResponse:
"""
Asynchronously calls the `image_generation` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_generation` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_generation` function.
Returns:
- `response` (Any): The response returned by the `image_generation` function.
"""
loop = asyncio.get_event_loop()
model = args[0] if len(args) > 0 else kwargs["model"]
### PASS ARGS TO Image Generation ###
kwargs["aimg_generation"] = True
custom_llm_provider = None
try:
# Use a partial function to pass your keyword arguments
func = partial(image_generation, *args, **kwargs)
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, api_base=kwargs.get("api_base", None)
)
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ImageResponse
): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=args,
extra_kwargs=kwargs,
)
@client
def image_generation( # noqa: PLR0915
prompt: str,
model: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
style: Optional[str] = None,
user: Optional[str] = None,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider=None,
**kwargs,
) -> ImageResponse:
"""
Maps the https://api.openai.com/v1/images/generations endpoint.
Currently supports just Azure + OpenAI.
"""
try:
args = locals()
aimg_generation = kwargs.get("aimg_generation", False)
litellm_call_id = kwargs.get("litellm_call_id", None)
logger_fn = kwargs.get("logger_fn", None)
mock_response: Optional[str] = kwargs.get("mock_response", None) # type: ignore
proxy_server_request = kwargs.get("proxy_server_request", None)
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
model_info = kwargs.get("model_info", None)
metadata = kwargs.get("metadata", {})
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
client = kwargs.get("client", None)
extra_headers = kwargs.get("extra_headers", None)
headers: dict = kwargs.get("headers", None) or {}
base_model = kwargs.get("base_model", None)
if extra_headers is not None:
headers.update(extra_headers)
model_response: ImageResponse = litellm.utils.ImageResponse()
dynamic_api_key: Optional[str] = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, # type: ignore
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
else:
model = "dall-e-2"
custom_llm_provider = "openai" # default to dall-e-2 on openai
model_response._hidden_params["model"] = model
openai_params = [
"user",
"request_timeout",
"api_base",
"api_version",
"api_key",
"deployment_id",
"organization",
"base_url",
"default_headers",
"timeout",
"max_retries",
"n",
"quality",
"size",
"style",
]
litellm_params = all_litellm_params
default_params = openai_params + litellm_params
non_default_params = {
k: v for k, v in kwargs.items() if k not in default_params
} # model-specific params - pass them straight to the model/provider
image_generation_config: Optional[BaseImageGenerationConfig] = None
if (
custom_llm_provider is not None
and custom_llm_provider in LlmProviders._member_map_.values()
):
image_generation_config = (
ProviderConfigManager.get_provider_image_generation_config(
model=base_model or model,
provider=LlmProviders(custom_llm_provider),
)
)
optional_params = get_optional_params_image_gen(
model=base_model or model,
n=n,
quality=quality,
response_format=response_format,
size=size,
style=style,
user=user,
custom_llm_provider=custom_llm_provider,
provider_config=image_generation_config,
**non_default_params,
)
litellm_params_dict = get_litellm_params(**kwargs)
logging: Logging = litellm_logging_obj
logging.update_environment_variables(
model=model,
user=user,
optional_params=optional_params,
litellm_params={
"timeout": timeout,
"azure": False,
"litellm_call_id": litellm_call_id,
"logger_fn": logger_fn,
"proxy_server_request": proxy_server_request,
"model_info": model_info,
"metadata": metadata,
"preset_cache_key": None,
"stream_response": {},
},
custom_llm_provider=custom_llm_provider,
)
if "custom_llm_provider" not in logging.model_call_details:
logging.model_call_details["custom_llm_provider"] = custom_llm_provider
if mock_response is not None:
return mock_image_generation(model=model, mock_response=mock_response)
if custom_llm_provider == "azure":
# azure configs
api_type = get_secret_str("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = (
api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
)
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.pop(
"azure_ad_token", None
) or get_secret_str("AZURE_AD_TOKEN")
default_headers = {
"Content-Type": "application/json;",
"api-key": api_key,
}
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key,
api_base=api_base,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key or dynamic_api_key,
api_base=api_base,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
aimg_generation=aimg_generation,
client=client,
)
elif custom_llm_provider == "bedrock":
if model is None:
raise Exception("Model needs to be set for bedrock")
model_response = bedrock_image_generation.image_generation( # type: ignore
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
aimg_generation=aimg_generation,
client=client,
)
elif custom_llm_provider == "vertex_ai":
vertex_ai_project = (
optional_params.pop("vertex_project", None)
or optional_params.pop("vertex_ai_project", None)
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.pop("vertex_location", None)
or optional_params.pop("vertex_ai_location", None)
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = (
optional_params.pop("vertex_credentials", None)
or optional_params.pop("vertex_ai_credentials", None)
or get_secret_str("VERTEXAI_CREDENTIALS")
)
api_base = (
api_base
or litellm.api_base
or get_secret_str("VERTEXAI_API_BASE")
or get_secret_str("VERTEX_API_BASE")
)
model_response = vertex_image_generation.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
vertex_credentials=vertex_credentials,
aimg_generation=aimg_generation,
api_base=api_base,
client=client,
)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider
# Get the Custom Handler
custom_handler: Optional[CustomLLM] = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
if custom_handler is None:
raise LiteLLMUnknownProvider(
model=model, custom_llm_provider=custom_llm_provider
)
## ROUTE LLM CALL ##
if aimg_generation is True:
async_custom_client: Optional[AsyncHTTPHandler] = None
if client is not None and isinstance(client, AsyncHTTPHandler):
async_custom_client = client
## CALL FUNCTION
model_response = custom_handler.aimage_generation( # type: ignore
model=model,
prompt=prompt,
api_key=api_key,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=async_custom_client,
)
else:
custom_client: Optional[HTTPHandler] = None
if client is not None and isinstance(client, HTTPHandler):
custom_client = client
## CALL FUNCTION
model_response = custom_handler.image_generation(
model=model,
prompt=prompt,
api_key=api_key,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=custom_client,
)
return model_response
except Exception as e:
## Map to OpenAI Exception
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=locals(),
extra_kwargs=kwargs,
)
@client
async def aimage_variation(*args, **kwargs) -> ImageResponse:
"""
Asynchronously calls the `image_variation` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_variation` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_variation` function.
Returns:
- `response` (Any): The response returned by the `image_variation` function.
"""
loop = asyncio.get_event_loop()
model = kwargs.get("model", None)
custom_llm_provider = kwargs.get("custom_llm_provider", None)
### PASS ARGS TO Image Generation ###
kwargs["async_call"] = True
try:
# Use a partial function to pass your keyword arguments
func = partial(image_variation, *args, **kwargs)
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
if custom_llm_provider is None and model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, api_base=kwargs.get("api_base", None)
)
# Await normally
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ImageResponse
): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
return response
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=args,
extra_kwargs=kwargs,
)
@client
def image_variation(
image: FileTypes,
model: str = "dall-e-2", # set to dall-e-2 by default - like OpenAI.
n: int = 1,
response_format: Literal["url", "b64_json"] = "url",
size: Optional[str] = None,
user: Optional[str] = None,
**kwargs,
) -> ImageResponse:
# get non-default params
client = kwargs.get("client", None)
# get logging object
litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj"))
# get the litellm params
litellm_params = get_litellm_params(**kwargs)
# get the custom llm provider
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=litellm_params.get("custom_llm_provider", None),
api_base=litellm_params.get("api_base", None),
api_key=litellm_params.get("api_key", None),
)
# route to the correct provider w/ the params
try:
llm_provider = LlmProviders(custom_llm_provider)
image_variation_provider = LITELLM_IMAGE_VARIATION_PROVIDERS(llm_provider)
except ValueError:
raise ValueError(
f"Invalid image variation provider: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
model_response = ImageResponse()
response: Optional[ImageResponse] = None
provider_config = ProviderConfigManager.get_provider_model_info(
model=model or "", # openai defaults to dall-e-2
provider=llm_provider,
)
if provider_config is None:
raise ValueError(
f"image variation provider has no known model info config - required for getting api keys, etc.: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
api_key = provider_config.get_api_key(litellm_params.get("api_key", None))
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
image=image,
timeout=litellm_params.get("timeout", None),
custom_llm_provider=custom_llm_provider,
logging_obj=litellm_logging_obj,
optional_params={},
litellm_params=litellm_params,
)
elif image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.TOPAZ:
if api_key is None:
raise ValueError("API key is required for Topaz image variations")
if api_base is None:
raise ValueError("API base is required for Topaz image variations")
response = base_llm_aiohttp_handler.image_variations(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
image=image,
timeout=litellm_params.get("timeout", None) or DEFAULT_REQUEST_TIMEOUT,
custom_llm_provider=custom_llm_provider,
logging_obj=litellm_logging_obj,
optional_params={},
litellm_params=litellm_params,
client=client,
)
# return the response
if response is None:
raise ValueError(
f"Invalid image variation provider: {custom_llm_provider}. Supported providers are: {LITELLM_IMAGE_VARIATION_PROVIDERS}"
)
return response
@client
def image_edit(
image: FileTypes,
prompt: str,
model: Optional[str] = None,
mask: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
user: Optional[str] = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
# LiteLLM specific params,
custom_llm_provider: Optional[str] = None,
**kwargs,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse]]:
"""
Maps the image edit functionality, similar to OpenAI's images/edits endpoint.
"""
local_vars = locals()
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
_is_async = kwargs.pop("async_call", False) is True
# get llm provider logic
litellm_params = GenericLiteLLMParams(**kwargs)
model, custom_llm_provider, _, _ = get_llm_provider(
model=model or DEFAULT_IMAGE_ENDPOINT_MODEL,
custom_llm_provider=custom_llm_provider,
)
# get provider config
image_edit_provider_config: Optional[
BaseImageEditConfig
] = ProviderConfigManager.get_provider_image_edit_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
)
if image_edit_provider_config is None:
raise ValueError(f"image edit is not supported for {custom_llm_provider}")
local_vars.update(kwargs)
# Get ImageEditOptionalRequestParams with only valid parameters
image_edit_optional_params: ImageEditOptionalRequestParams = (
ImageEditRequestUtils.get_requested_image_edit_optional_param(local_vars)
)
# Get optional parameters for the responses API
image_edit_request_params: Dict = (
ImageEditRequestUtils.get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
)
)
# Pre Call logging
litellm_logging_obj.update_environment_variables(
model=model,
user=user,
optional_params=dict(image_edit_request_params),
litellm_params={
"litellm_call_id": litellm_call_id,
**image_edit_request_params,
},
custom_llm_provider=custom_llm_provider,
)
# Call the handler with _is_async flag instead of directly calling the async handler
return base_llm_http_handler.image_edit_handler(
model=model,
image=image,
prompt=prompt,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_request_params=image_edit_request_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
extra_headers=extra_headers,
extra_body=extra_body,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
client=kwargs.get("client"),
)
except Exception as e:
raise litellm.exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
@client
async def aimage_edit(
image: FileTypes,
model: str,
prompt: str,
mask: Optional[str] = None,
n: Optional[int] = None,
quality: Optional[Union[str, ImageGenerationRequestQuality]] = None,
response_format: Optional[str] = None,
size: Optional[str] = None,
user: Optional[str] = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
# LiteLLM specific params,
custom_llm_provider: Optional[str] = None,
**kwargs,
) -> ImageResponse:
"""
Asynchronously calls the `image_edit` function with the given arguments and keyword arguments.
Parameters:
- `args` (tuple): Positional arguments to be passed to the `image_edit` function.
- `kwargs` (dict): Keyword arguments to be passed to the `image_edit` function.
Returns:
- `response` (Any): The response returned by the `image_edit` function.
"""
local_vars = locals()
try:
loop = asyncio.get_event_loop()
kwargs["async_call"] = True
# get custom llm provider so we can use this for mapping exceptions
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model, api_base=local_vars.get("base_url", None)
)
func = partial(
image_edit,
image=image,
prompt=prompt,
mask=mask,
model=model,
n=n,
quality=quality,
response_format=response_format,
size=size,
user=user,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
init_response = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
except Exception as e:
raise litellm.exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
litellm/images/utils.py
+142
View File
@@ -0,0 +1,142 @@
from io import BufferedReader, BytesIO
from typing import Any, Dict, cast, get_type_hints
import litellm
from litellm.litellm_core_utils.token_counter import get_image_type
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.files import FILE_MIME_TYPES, FileType
from litellm.types.images.main import ImageEditOptionalRequestParams
class ImageEditRequestUtils:
@staticmethod
def get_optional_params_image_edit(
model: str,
image_edit_provider_config: BaseImageEditConfig,
image_edit_optional_params: ImageEditOptionalRequestParams,
) -> Dict:
"""
Get optional parameters for the image edit API.
Args:
params: Dictionary of all parameters
model: The model name
image_edit_provider_config: The provider configuration for image edit API
Returns:
A dictionary of supported parameters for the image edit API
"""
# Remove None values and internal parameters
# Get supported parameters for the model
supported_params = image_edit_provider_config.get_supported_openai_params(model)
# Check for unsupported parameters
unsupported_params = [
param
for param in image_edit_optional_params
if param not in supported_params
]
if unsupported_params:
raise litellm.UnsupportedParamsError(
model=model,
message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}",
)
# Map parameters to provider-specific format
mapped_params = image_edit_provider_config.map_openai_params(
image_edit_optional_params=image_edit_optional_params,
model=model,
drop_params=litellm.drop_params,
)
return mapped_params
@staticmethod
def get_requested_image_edit_optional_param(
params: Dict[str, Any],
) -> ImageEditOptionalRequestParams:
"""
Filter parameters to only include those defined in ImageEditOptionalRequestParams.
Args:
params: Dictionary of parameters to filter
Returns:
ImageEditOptionalRequestParams instance with only the valid parameters
"""
valid_keys = get_type_hints(ImageEditOptionalRequestParams).keys()
filtered_params = {
k: v for k, v in params.items() if k in valid_keys and v is not None
}
return cast(ImageEditOptionalRequestParams, filtered_params)
@staticmethod
def get_image_content_type(image_data: Any) -> str:
"""
Detect the content type of image data using existing LiteLLM utils.
Args:
image_data: Can be BytesIO, bytes, BufferedReader, or other file-like objects
Returns:
The MIME type string (e.g., "image/png", "image/jpeg")
"""
try:
# Extract bytes for content type detection
if isinstance(image_data, BytesIO):
# Save current position
current_pos = image_data.tell()
image_data.seek(0)
bytes_data = image_data.read(
100
) # First 100 bytes are enough for detection
# Restore position
image_data.seek(current_pos)
elif isinstance(image_data, BufferedReader):
# Save current position
current_pos = image_data.tell()
image_data.seek(0)
bytes_data = image_data.read(100)
# Restore position
image_data.seek(current_pos)
elif isinstance(image_data, bytes):
bytes_data = image_data[:100]
else:
# For other types, try to read if possible
if hasattr(image_data, "read"):
current_pos = getattr(image_data, "tell", lambda: 0)()
if hasattr(image_data, "seek"):
image_data.seek(0)
bytes_data = image_data.read(100)
if hasattr(image_data, "seek"):
image_data.seek(current_pos)
else:
return FILE_MIME_TYPES[FileType.PNG] # Default fallback
# Use the existing get_image_type function to detect image type
image_type_str = get_image_type(bytes_data)
if image_type_str is None:
return FILE_MIME_TYPES[FileType.PNG] # Default if detection fails
# Map detected type string to FileType enum and get MIME type
type_mapping = {
"png": FileType.PNG,
"jpeg": FileType.JPEG,
"gif": FileType.GIF,
"webp": FileType.WEBP,
"heic": FileType.HEIC,
}
file_type = type_mapping.get(image_type_str)
if file_type is None:
return FILE_MIME_TYPES[FileType.PNG] # Default to PNG if unknown
return FILE_MIME_TYPES[file_type]
except Exception:
# If anything goes wrong, default to PNG
return FILE_MIME_TYPES[FileType.PNG]
litellm/integrations/SlackAlerting/utils.py
+12 -5
View File
@@ -5,6 +5,7 @@ Utils used for slack alerting
import asyncio
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import litellm
from litellm.proxy._types import AlertType
from litellm.secret_managers.main import get_secret
@@ -69,7 +70,12 @@ async def _add_langfuse_trace_id_to_alert(
-> trace_id
-> litellm_call_id
"""
# do nothing for now
if "langfuse" not in litellm.logging_callback_manager._get_all_callbacks():
return None
#########################################################
# Only run if langfuse is added as a callback
#########################################################
if (
request_data is not None
and request_data.get("litellm_logging_obj", None) is not None
@@ -82,11 +88,12 @@ async def _add_langfuse_trace_id_to_alert(
if trace_id is not None:
break
await asyncio.sleep(3) # wait 3s before retrying for trace id
_langfuse_object = litellm_logging_obj._get_callback_object(
#########################################################
langfuse_object = litellm_logging_obj._get_callback_object(
service_name="langfuse"
)
if _langfuse_object is not None:
base_url = _langfuse_object.Langfuse.base_url
if langfuse_object is not None:
base_url = langfuse_object.Langfuse.base_url
return f"{base_url}/trace/{trace_id}"
return None
litellm/integrations/anthropic_cache_control_hook.py
+5 -4
View File
@@ -28,6 +28,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Apply cache control directives based on specified injection points.
@@ -79,10 +80,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# Case 1: Target by specific index
if targetted_index is not None:
if 0 <= targetted_index < len(messages):
messages[targetted_index] = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
messages[
targetted_index
] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
# Case 2: Target by role
elif targetted_role is not None:
litellm/integrations/custom_logger.py
+3
View File
@@ -87,6 +87,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@@ -104,6 +105,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@@ -234,6 +236,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
pass
litellm/integrations/custom_prompt_management.py
+2
View File
@@ -18,6 +18,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@@ -43,6 +44,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase):
prompt_id: str,
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
raise NotImplementedError(
"Custom prompt management does not support compile prompt helper"
litellm/integrations/deepeval/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from .deepeval import DeepEvalLogger
__all__ = ["DeepEvalLogger"]
litellm/integrations/deepeval/api.py
+120
View File
@@ -0,0 +1,120 @@
# duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/confident/api.py
import logging
import httpx
from enum import Enum
from litellm._logging import verbose_logger
DEEPEVAL_BASE_URL = "https://deepeval.confident-ai.com"
DEEPEVAL_BASE_URL_EU = "https://eu.deepeval.confident-ai.com"
API_BASE_URL = "https://api.confident-ai.com"
API_BASE_URL_EU = "https://eu.api.confident-ai.com"
retryable_exceptions = httpx.HTTPError
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
def log_retry_error(details):
exception = details.get("exception")
tries = details.get("tries")
if exception:
logging.error(f"Confident AI Error: {exception}. Retrying: {tries} time(s)...")
else:
logging.error(f"Retrying: {tries} time(s)...")
class HttpMethods(Enum):
GET = "GET"
POST = "POST"
DELETE = "DELETE"
PUT = "PUT"
class Endpoints(Enum):
DATASET_ENDPOINT = "/v1/dataset"
TEST_RUN_ENDPOINT = "/v1/test-run"
TRACING_ENDPOINT = "/v1/tracing"
EVENT_ENDPOINT = "/v1/event"
FEEDBACK_ENDPOINT = "/v1/feedback"
PROMPT_ENDPOINT = "/v1/prompt"
RECOMMEND_ENDPOINT = "/v1/recommend-metrics"
EVALUATE_ENDPOINT = "/evaluate"
GUARD_ENDPOINT = "/guard"
GUARDRAILS_ENDPOINT = "/guardrails"
BASELINE_ATTACKS_ENDPOINT = "/generate-baseline-attacks"
class Api:
def __init__(self, api_key: str, base_url=None):
self.api_key = api_key
self._headers = {
"Content-Type": "application/json",
# "User-Agent": "Python/Requests",
"CONFIDENT_API_KEY": api_key,
}
# using the global non-eu variable for base url
self.base_api_url = base_url or API_BASE_URL
self.sync_http_handler = HTTPHandler()
self.async_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
def _http_request(
self, method: str, url: str, headers=None, json=None, params=None
):
if method != "POST":
raise Exception("Only POST requests are supported")
try:
self.sync_http_handler.post(
url=url,
headers=headers,
json=json,
params=params,
)
except httpx.HTTPStatusError as e:
raise Exception(f"DeepEval logging error: {e.response.text}")
except Exception as e:
raise e
def send_request(
self, method: HttpMethods, endpoint: Endpoints, body=None, params=None
):
url = f"{self.base_api_url}{endpoint.value}"
res = self._http_request(
method=method.value,
url=url,
headers=self._headers,
json=body,
params=params,
)
if res.status_code == 200:
try:
return res.json()
except ValueError:
return res.text
else:
verbose_logger.debug(res.json())
raise Exception(res.json().get("error", res.text))
async def a_send_request(
self, method: HttpMethods, endpoint: Endpoints, body=None, params=None
):
if method != HttpMethods.POST:
raise Exception("Only POST requests are supported")
url = f"{self.base_api_url}{endpoint.value}"
try:
await self.async_http_handler.post(
url=url,
headers=self._headers,
json=body,
params=params,
)
except httpx.HTTPStatusError as e:
raise Exception(f"DeepEval logging error: {e.response.text}")
except Exception as e:
raise e
litellm/integrations/deepeval/deepeval.py
+175
View File
@@ -0,0 +1,175 @@
import os
import uuid
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.api import Api, Endpoints, HttpMethods
from litellm.integrations.deepeval.types import (
BaseApiSpan,
SpanApiType,
TraceApi,
TraceSpanApiStatus,
)
from litellm.integrations.deepeval.utils import (
to_zod_compatible_iso,
validate_environment,
)
from litellm._logging import verbose_logger
# This file includes the custom callbacks for LiteLLM Proxy
# Once defined, these can be passed in proxy_config.yaml
class DeepEvalLogger(CustomLogger):
"""Logs litellm traces to DeepEval's platform."""
def __init__(self, *args, **kwargs):
api_key = os.getenv("CONFIDENT_API_KEY")
self.litellm_environment = os.getenv("LITELM_ENVIRONMENT", "development")
validate_environment(self.litellm_environment)
if not api_key:
raise ValueError(
"Please set 'CONFIDENT_API_KEY=<>' in your environment variables."
)
self.api = Api(api_key=api_key)
super().__init__(*args, **kwargs)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
"""Logs a success event to DeepEval's platform."""
self._sync_event_handler(
kwargs, response_obj, start_time, end_time, is_success=True
)
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""Logs a failure event to DeepEval's platform."""
self._sync_event_handler(
kwargs, response_obj, start_time, end_time, is_success=False
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""Logs a failure event to DeepEval's platform."""
await self._async_event_handler(
kwargs, response_obj, start_time, end_time, is_success=False
)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""Logs a success event to DeepEval's platform."""
await self._async_event_handler(
kwargs, response_obj, start_time, end_time, is_success=True
)
def _prepare_trace_api(
self, kwargs, response_obj, start_time, end_time, is_success
):
_start_time = to_zod_compatible_iso(start_time)
_end_time = to_zod_compatible_iso(end_time)
_standard_logging_object = kwargs.get("standard_logging_object", {})
base_api_span = self._create_base_api_span(
kwargs,
standard_logging_object=_standard_logging_object,
start_time=_start_time,
end_time=_end_time,
is_success=is_success,
)
trace_api = self._create_trace_api(
base_api_span,
standard_logging_object=_standard_logging_object,
start_time=_start_time,
end_time=_end_time,
litellm_environment=self.litellm_environment,
)
body = {}
try:
body = trace_api.model_dump(by_alias=True, exclude_none=True)
except AttributeError:
# Pydantic version below 2.0
body = trace_api.dict(by_alias=True, exclude_none=True)
return body
def _sync_event_handler(
self, kwargs, response_obj, start_time, end_time, is_success
):
body = self._prepare_trace_api(
kwargs, response_obj, start_time, end_time, is_success
)
try:
response = self.api.send_request(
method=HttpMethods.POST,
endpoint=Endpoints.TRACING_ENDPOINT,
body=body,
)
except Exception as e:
raise e
verbose_logger.debug(
"DeepEvalLogger: sync_log_failure_event: Api response", response
)
async def _async_event_handler(
self, kwargs, response_obj, start_time, end_time, is_success
):
body = self._prepare_trace_api(
kwargs, response_obj, start_time, end_time, is_success
)
response = await self.api.a_send_request(
method=HttpMethods.POST,
endpoint=Endpoints.TRACING_ENDPOINT,
body=body,
)
verbose_logger.debug(
"DeepEvalLogger: async_event_handler: Api response", response
)
def _create_base_api_span(
self, kwargs, standard_logging_object, start_time, end_time, is_success
):
# extract usage
usage = standard_logging_object.get("response", {}).get("usage", {})
if is_success:
output = (
standard_logging_object.get("response", {})
.get("choices", [{}])[0]
.get("message", {})
.get("content", "NO_OUTPUT")
)
else:
output = str(standard_logging_object.get("error_string", ""))
return BaseApiSpan(
uuid=standard_logging_object.get("id", uuid.uuid4()),
name=(
"litellm_success_callback" if is_success else "litellm_failure_callback"
),
status=(
TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED
),
type=SpanApiType.LLM,
traceUuid=standard_logging_object.get("trace_id", uuid.uuid4()),
startTime=str(start_time),
endTime=str(end_time),
input=kwargs.get("input", "NO_INPUT"),
output=output,
model=standard_logging_object.get("model", None),
inputTokenCount=usage.get("prompt_tokens", None) if is_success else None,
outputTokenCount=(
usage.get("completion_tokens", None) if is_success else None
),
)
def _create_trace_api(
self,
base_api_span,
standard_logging_object,
start_time,
end_time,
litellm_environment,
):
return TraceApi(
uuid=standard_logging_object.get("trace_id", uuid.uuid4()),
baseSpans=[],
agentSpans=[],
llmSpans=[base_api_span],
retrieverSpans=[],
toolSpans=[],
startTime=str(start_time),
endTime=str(end_time),
environment=litellm_environment,
)
litellm/integrations/deepeval/types.py
+64
View File
@@ -0,0 +1,64 @@
# Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py
from enum import Enum
from typing import Any, Dict, List, Optional, Union, Literal
from pydantic import BaseModel, Field
class SpanApiType(Enum):
BASE = "base"
AGENT = "agent"
LLM = "llm"
RETRIEVER = "retriever"
TOOL = "tool"
span_api_type_literals = Literal["base", "agent", "llm", "retriever", "tool"]
class TraceSpanApiStatus(Enum):
SUCCESS = "SUCCESS"
ERRORED = "ERRORED"
class BaseApiSpan(BaseModel):
uuid: str
name: Optional[str] = None
status: TraceSpanApiStatus
type: SpanApiType
trace_uuid: str = Field(alias="traceUuid")
parent_uuid: Optional[str] = Field(None, alias="parentUuid")
start_time: str = Field(alias="startTime")
end_time: str = Field(alias="endTime")
input: Optional[Union[Dict, list, str]] = None
output: Optional[Union[Dict, list, str]] = None
error: Optional[str] = None
# llm
model: Optional[str] = None
input_token_count: Optional[int] = Field(None, alias="inputTokenCount")
output_token_count: Optional[int] = Field(None, alias="outputTokenCount")
cost_per_input_token: Optional[float] = Field(None, alias="costPerInputToken")
cost_per_output_token: Optional[float] = Field(None, alias="costPerOutputToken")
class Config:
use_enum_values = True
class TraceApi(BaseModel):
uuid: str
base_spans: List[BaseApiSpan] = Field(alias="baseSpans")
agent_spans: List[BaseApiSpan] = Field(alias="agentSpans")
llm_spans: List[BaseApiSpan] = Field(alias="llmSpans")
retriever_spans: List[BaseApiSpan] = Field(alias="retrieverSpans")
tool_spans: List[BaseApiSpan] = Field(alias="toolSpans")
start_time: str = Field(alias="startTime")
end_time: str = Field(alias="endTime")
metadata: Optional[Dict[str, Any]] = Field(None)
tags: Optional[List[str]] = Field(None)
environment: Optional[str] = Field(None)
class Environment(Enum):
PRODUCTION = "production"
DEVELOPMENT = "development"
STAGING = "staging"
litellm/integrations/deepeval/utils.py
+18
View File
@@ -0,0 +1,18 @@
from datetime import datetime, timezone
from litellm.integrations.deepeval.types import Environment
def to_zod_compatible_iso(dt: datetime) -> str:
return (
dt.astimezone(timezone.utc)
.isoformat(timespec="milliseconds")
.replace("+00:00", "Z")
)
def validate_environment(environment: str):
if environment not in [env.value for env in Environment]:
valid_values = ", ".join(f'"{env.value}"' for env in Environment)
raise ValueError(
f"Invalid environment: {environment}. Please use one of the following instead: {valid_values}"
)
litellm/integrations/humanloop.py
+2 -5
View File
@@ -155,11 +155,8 @@ class HumanloopLogger(CustomLogger):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict,]:
humanloop_api_key = dynamic_callback_params.get(
"humanloop_api_key"
) or get_secret_str("HUMANLOOP_API_KEY")
litellm/integrations/langfuse/langfuse.py
+22 -3
View File
@@ -10,6 +10,7 @@ from packaging.version import Version
import litellm
from litellm._logging import verbose_logger
from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.secret_managers.main import str_to_bool
@@ -27,12 +28,13 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from langfuse.client import StatefulTraceClient
from langfuse.client import Langfuse, StatefulTraceClient
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
else:
DynamicLoggingCache = Any
StatefulTraceClient = Any
Langfuse = Any
class LangFuseLogger:
@@ -84,8 +86,7 @@ class LangFuseLogger:
if Version(self.langfuse_sdk_version) >= Version("2.6.0"):
parameters["sdk_integration"] = "litellm"
self.Langfuse = Langfuse(**parameters)
self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters)
# set the current langfuse project id in the environ
# this is used by Alerting to link to the correct project
@@ -124,6 +125,24 @@ class LangFuseLogger:
else:
self.upstream_langfuse = None
def safe_init_langfuse_client(self, parameters: dict) -> Langfuse:
"""
Safely init a langfuse client if the number of initialized clients is less than the max
Note:
- Langfuse initializes 1 thread everytime a client is initialized.
- We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
"""
from langfuse import Langfuse
if litellm.initialized_langfuse_clients >= MAX_LANGFUSE_INITIALIZED_CLIENTS:
raise Exception(
f"Max langfuse clients reached: {litellm.initialized_langfuse_clients} is greater than {MAX_LANGFUSE_INITIALIZED_CLIENTS}"
)
langfuse_client = Langfuse(**parameters)
litellm.initialized_langfuse_clients += 1
return langfuse_client
@staticmethod
def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict:
"""
litellm/integrations/langfuse/langfuse_prompt_management.py
+12 -8
View File
@@ -130,9 +130,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
return "langfuse"
def _get_prompt_from_id(
self, langfuse_prompt_id: str, langfuse_client: LangfuseClass
self,
langfuse_prompt_id: str,
langfuse_client: LangfuseClass,
prompt_label: Optional[str] = None,
) -> PROMPT_CLIENT:
return langfuse_client.get_prompt(langfuse_prompt_id)
return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label)
def _compile_prompt(
self,
@@ -176,11 +179,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict,]:
return self.get_chat_completion_prompt(
model,
messages,
@@ -188,6 +188,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_id,
prompt_variables,
dynamic_callback_params,
prompt_label=prompt_label,
)
def should_run_prompt_management(
@@ -211,6 +212,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_id: str,
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
langfuse_client = langfuse_client_init(
langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"),
@@ -219,7 +221,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_host=dynamic_callback_params.get("langfuse_host"),
)
langfuse_prompt_client = self._get_prompt_from_id(
langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client
langfuse_prompt_id=prompt_id,
langfuse_client=langfuse_client,
prompt_label=prompt_label,
)
## SET PROMPT
litellm/integrations/opentelemetry.py
+8 -1
View File
@@ -135,7 +135,13 @@ class OpenTelemetry(CustomLogger):
- Adds Otel as a service callback
- Sets `proxy_server.open_telemetry_logger` to self
"""
from litellm.proxy import proxy_server
try:
from litellm.proxy import proxy_server
except ImportError:
verbose_logger.warning(
"Proxy Server is not installed. Skipping OpenTelemetry initialization."
)
return
# Add Otel as a service callback
if "otel" not in litellm.service_callback:
@@ -282,6 +288,7 @@ class OpenTelemetry(CustomLogger):
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
litellm/integrations/prometheus.py
+3
View File
@@ -802,6 +802,7 @@ class PrometheusLogger(CustomLogger):
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
"""
Track client side failures
@@ -832,6 +833,7 @@ class PrometheusLogger(CustomLogger):
exception_status=str(getattr(original_exception, "status_code", None)),
exception_class=self._get_exception_class_name(original_exception),
tags=_tags,
route=user_api_key_dict.request_route,
)
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.get_labels(
@@ -872,6 +874,7 @@ class PrometheusLogger(CustomLogger):
user=user_api_key_dict.user_id,
user_email=user_api_key_dict.user_email,
status_code="200",
route=user_api_key_dict.request_route,
)
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.get_labels(
litellm/integrations/prompt_management_base.py
+5
View File
@@ -33,6 +33,7 @@ class PromptManagementBase(ABC):
prompt_id: str,
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
pass
@@ -49,11 +50,13 @@ class PromptManagementBase(ABC):
prompt_variables: Optional[dict],
client_messages: List[AllMessageValues],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
compiled_prompt_client = self._compile_prompt_helper(
prompt_id=prompt_id,
prompt_variables=prompt_variables,
dynamic_callback_params=dynamic_callback_params,
prompt_label=prompt_label,
)
try:
@@ -82,6 +85,7 @@ class PromptManagementBase(ABC):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
if prompt_id is None:
raise ValueError("prompt_id is required for Prompt Management Base class")
@@ -95,6 +99,7 @@ class PromptManagementBase(ABC):
prompt_variables=prompt_variables,
client_messages=messages,
dynamic_callback_params=dynamic_callback_params,
prompt_label=prompt_label,
)
completed_messages = prompt_template["completed_messages"] or messages
litellm/integrations/vector_stores/bedrock_vector_store.py
+12 -10
View File
@@ -75,6 +75,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Retrieves the context from the Bedrock Knowledge Base and appends it to the messages.
@@ -99,10 +100,11 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
f"Bedrock Knowledge Base Response: {bedrock_kb_response}"
)
context_message, context_string = (
self.get_chat_completion_message_from_bedrock_kb_response(
bedrock_kb_response
)
(
context_message,
context_string,
) = self.get_chat_completion_message_from_bedrock_kb_response(
bedrock_kb_response
)
if context_message is not None:
messages.append(context_message)
@@ -126,9 +128,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
)
)
litellm_logging_obj.model_call_details["vector_store_request_metadata"] = (
vector_store_request_metadata
)
litellm_logging_obj.model_call_details[
"vector_store_request_metadata"
] = vector_store_request_metadata
return model, messages, non_default_params
@@ -140,9 +142,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
"""
Transform a BedrockKBResponse to a VectorStoreSearchResponse
"""
retrieval_results: Optional[List[BedrockKBRetrievalResult]] = (
bedrock_kb_response.get("retrievalResults", None)
)
retrieval_results: Optional[
List[BedrockKBRetrievalResult]
] = bedrock_kb_response.get("retrievalResults", None)
vector_store_search_response: VectorStoreSearchResponse = (
VectorStoreSearchResponse(search_query=query, data=[])
)
litellm/litellm_core_utils/exception_mapping_utils.py
+12 -5
View File
@@ -317,11 +317,18 @@ def exception_type( # type: ignore # noqa: PLR0915
litellm_debug_info=extra_information,
)
elif (
"invalid_request_error" in error_str
and "content_policy_violation" in error_str
) or (
"Invalid prompt" in error_str
and "violating our usage policy" in error_str
(
"invalid_request_error" in error_str
and "content_policy_violation" in error_str
)
or (
"Invalid prompt" in error_str
and "violating our usage policy" in error_str
)
or (
"request was rejected as a result of the safety system"
in error_str.lower()
)
):
exception_mapping_worked = True
raise ContentPolicyViolationError(
litellm/litellm_core_utils/litellm_logging.py
+27 -6
View File
@@ -52,6 +52,7 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont
from litellm.integrations.arize.arize import ArizeLogger
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.vector_stores.bedrock_vector_store import BedrockVectorStore
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
@@ -198,6 +199,7 @@ s3Logger = None
greenscaleLogger = None
lunaryLogger = None
supabaseClient = None
deepevalLogger = None
callback_list: Optional[List[str]] = []
user_logger_fn = None
additional_details: Optional[Dict[str, str]] = {}
@@ -539,6 +541,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
prompt_management_logger: Optional[CustomLogger] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
custom_logger = (
prompt_management_logger
@@ -559,6 +562,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_id=prompt_id,
prompt_variables=prompt_variables,
dynamic_callback_params=self.standard_callback_dynamic_params,
prompt_label=prompt_label,
)
self.messages = messages
return model, messages, non_default_params
@@ -572,6 +576,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_variables: Optional[dict],
prompt_management_logger: Optional[CustomLogger] = None,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
custom_logger = (
prompt_management_logger
@@ -594,6 +599,7 @@ class Logging(LiteLLMLoggingBaseClass):
dynamic_callback_params=self.standard_callback_dynamic_params,
litellm_logging_obj=self,
tools=tools,
prompt_label=prompt_label,
)
self.messages = messages
return model, messages, non_default_params
@@ -1719,7 +1725,6 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
)
if (
isinstance(callback, CustomLogger)
and self.model_call_details.get("litellm_params", {}).get(
@@ -2667,7 +2672,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
"""
Globally sets the callback client
"""
global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, supabaseClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger
global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, supabaseClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger, deepevalLogger
try:
for callback in callback_list:
@@ -2951,6 +2956,15 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
galileo_logger = GalileoObserve()
_in_memory_loggers.append(galileo_logger)
return galileo_logger # type: ignore
elif logging_integration == "deepeval":
for callback in _in_memory_loggers:
if isinstance(callback, DeepEvalLogger):
return callback # type: ignore
deepeval_logger = DeepEvalLogger()
_in_memory_loggers.append(deepeval_logger)
return deepeval_logger # type: ignore
elif logging_integration == "logfire":
if "LOGFIRE_TOKEN" not in os.environ:
raise ValueError("LOGFIRE_TOKEN not found in environment variables")
@@ -3120,6 +3134,10 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, GalileoObserve):
return callback
elif logging_integration == "deepeval":
for callback in _in_memory_loggers:
if isinstance(callback, DeepEvalLogger):
return callback
elif logging_integration == "langsmith":
for callback in _in_memory_loggers:
if isinstance(callback, LangsmithLogger):
@@ -3594,7 +3612,10 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_error_information(
original_exception: Optional[Exception],
traceback_str: Optional[str] = None,
) -> StandardLoggingPayloadErrorInformation:
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
error_status: str = str(getattr(original_exception, "status_code", ""))
error_class: str = (
str(original_exception.__class__.__name__) if original_exception else ""
@@ -3602,14 +3623,14 @@ class StandardLoggingPayloadSetup:
_llm_provider_in_exception = getattr(original_exception, "llm_provider", "")
# Get traceback information (first 100 lines)
traceback_info = ""
traceback_info = traceback_str or ""
if original_exception:
tb = getattr(original_exception, "__traceback__", None)
if tb:
import traceback
tb_lines = traceback.format_tb(tb)
traceback_info = "".join(tb_lines[:100]) # Limit to first 100 lines
traceback_info += "".join(
tb_lines[:MAXIMUM_TRACEBACK_LINES_TO_LOG]
) # Limit to first 100 lines
# Get additional error details
error_message = str(original_exception)
litellm/litellm_core_utils/logging_callback_manager.py
+9
View File
@@ -266,3 +266,12 @@ class LoggingCallbackManager:
if isinstance(callback, callback_type) and callback not in all_callbacks:
all_callbacks.append(callback)
return all_callbacks
def callback_is_active(self, callback_type: Type[CustomLogger]) -> bool:
"""
Returns True if any of the active callbacks are of the given type
"""
return any(
isinstance(callback, callback_type)
for callback in self._get_all_callbacks()
)
litellm/litellm_core_utils/prompt_templates/common_utils.py
+9
View File
@@ -573,3 +573,12 @@ def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]:
if tool_call_name:
tool_call_names.append(tool_call_name)
return tool_call_names
def is_function_call(optional_params: dict) -> bool:
"""
Checks if the optional params contain the function call
"""
if "functions" in optional_params and optional_params.get("functions"):
return True
return False
litellm/litellm_core_utils/prompt_templates/factory.py
+24 -2
View File
@@ -2267,6 +2267,7 @@ from litellm.types.llms.bedrock import (
)
from litellm.types.llms.bedrock import ToolSpecBlock as BedrockToolSpecBlock
from litellm.types.llms.bedrock import ToolUseBlock as BedrockToolUseBlock
from litellm.types.llms.bedrock import VideoBlock as BedrockVideoBlock
def _parse_content_type(content_type: str) -> str:
@@ -2356,9 +2357,15 @@ class BedrockImageProcessor:
supported_doc_formats = (
litellm.AmazonConverseConfig().get_supported_document_types()
)
supported_video_formats = (
litellm.AmazonConverseConfig().get_supported_video_types()
)
document_types = ["application", "text"]
is_document = any(mime_type.startswith(doc_type) for doc_type in document_types)
supported_image_and_video_formats: List[str] = (
supported_video_formats + supported_image_formats
)
if is_document:
potential_extensions = mimetypes.guess_all_extensions(mime_type)
@@ -2376,9 +2383,12 @@ class BedrockImageProcessor:
# Use first valid extension instead of provided image_format
return valid_extensions[0]
else:
if image_format not in supported_image_formats:
#########################################################
# Check if image_format is an image or video
#########################################################
if image_format not in supported_image_and_video_formats:
raise ValueError(
f"Unsupported image format: {image_format}. Supported formats: {supported_image_formats}"
f"Unsupported image format: {image_format}. Supported formats: {supported_image_and_video_formats}"
)
return image_format
@@ -2392,6 +2402,14 @@ class BedrockImageProcessor:
document_types = ["application", "text"]
is_document = any(mime_type.startswith(doc_type) for doc_type in document_types)
supported_video_formats = (
litellm.AmazonConverseConfig().get_supported_video_types()
)
is_video = any(
image_format.startswith(video_type)
for video_type in supported_video_formats
)
if is_document:
return BedrockContentBlock(
document=BedrockDocumentBlock(
@@ -2400,6 +2418,10 @@ class BedrockImageProcessor:
name=f"DocumentPDFmessages_{str(uuid.uuid4())}",
)
)
elif is_video:
return BedrockContentBlock(
video=BedrockVideoBlock(source=_blob, format=image_format)
)
else:
return BedrockContentBlock(
image=BedrockImageBlock(source=_blob, format=image_format)
litellm/litellm_core_utils/streaming_handler.py
+11 -7
View File
@@ -149,14 +149,14 @@ class CustomStreamWrapper:
)
def check_is_function_call(self, logging_obj) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
is_function_call,
)
if hasattr(logging_obj, "optional_params") and isinstance(
logging_obj.optional_params, dict
):
if (
"litellm_param_is_function_call" in logging_obj.optional_params
and logging_obj.optional_params["litellm_param_is_function_call"]
is True
):
if is_function_call(logging_obj.optional_params):
return True
return False
@@ -695,10 +695,14 @@ class CustomStreamWrapper:
Ensure model id is always the same across all chunks.
If first chunk sent + id set, use that id for all chunks.
If a valid ID is received in any chunk, use it for the response.
"""
if self.response_id is None:
if self.response_id is None and id and isinstance(id, str) and id.strip():
self.response_id = id
if id and isinstance(id, str) and id.strip():
model_response._hidden_params["received_model_id"] = id
if self.response_id is not None and isinstance(self.response_id, str):
model_response.id = self.response_id
return model_response
litellm/llms/anthropic/chat/transformation.py
+6 -2
View File
@@ -49,6 +49,7 @@ from litellm.utils import (
Usage,
add_dummy_tool,
has_tool_call_blocks,
supports_reasoning,
token_counter,
)
@@ -62,7 +63,7 @@ else:
LoggingClass = Any
ANTHROPIC_HOSTED_TOOLS = ["web_search", "bash", "text_editor"]
ANTHROPIC_HOSTED_TOOLS = ["web_search", "bash", "text_editor", "code_execution"]
class AnthropicConfig(AnthropicModelInfo, BaseConfig):
@@ -121,7 +122,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"web_search_options",
]
if "claude-3-7-sonnet" in model:
if "claude-3-7-sonnet" in model or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
):
params.append("thinking")
return params
litellm/llms/azure/image_edit/transformation.py
+83
View File
@@ -0,0 +1,83 @@
from typing import Optional, cast
import httpx
import litellm
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.utils import _add_path_to_api_base
class AzureImageEditConfig(OpenAIImageEditConfig):
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
)
return headers
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Constructs a complete URL for the API request.
Args:
- api_base: Base URL, e.g.,
"https://litellm8397336933.openai.azure.com"
OR
"https://litellm8397336933.openai.azure.com/openai/deployments/<deployment_name>/images/edits?api-version=2024-05-01-preview"
- model: Model name (deployment name).
- litellm_params: Additional query parameters, including "api_version".
Returns:
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/deployments/<deployment_name>/images/edits?api-version=2024-05-01-preview"
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
)
original_url = httpx.URL(api_base)
# Extract api_version or use default
api_version = cast(Optional[str], litellm_params.get("api_version"))
# Create a new dictionary with existing params
query_params = dict(original_url.params)
# Add api_version if needed
if "api-version" not in query_params and api_version:
query_params["api-version"] = api_version
# Add the path to the base URL using the model as deployment name
if "/openai/deployments/" not in api_base:
new_url = _add_path_to_api_base(
api_base=api_base,
ending_path=f"/openai/deployments/{model}/images/edits",
)
else:
new_url = api_base
# Use the new query_params dictionary
final_url = httpx.URL(new_url).copy_with(params=query_params)
return str(final_url)
litellm/llms/base_llm/__init__.py
+2
View File
@@ -2,6 +2,7 @@ from .anthropic_messages.transformation import BaseAnthropicMessagesConfig
from .audio_transcription.transformation import BaseAudioTranscriptionConfig
from .chat.transformation import BaseConfig
from .embedding.transformation import BaseEmbeddingConfig
from .image_edit.transformation import BaseImageEditConfig
from .image_generation.transformation import BaseImageGenerationConfig
__all__ = [
@@ -10,4 +11,5 @@ __all__ = [
"BaseAudioTranscriptionConfig",
"BaseAnthropicMessagesConfig",
"BaseEmbeddingConfig",
"BaseImageEditConfig",
]
litellm/llms/base_llm/files/transformation.py
+2
View File
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import httpx
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
@@ -115,6 +116,7 @@ class BaseFileEndpoints(ABC):
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
pass
litellm/llms/base_llm/image_edit/transformation.py
+121
View File
@@ -0,0 +1,121 @@
import types
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
import httpx
from httpx._types import RequestFiles
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.utils import ImageResponse as _ImageResponse
from ..chat.transformation import BaseLLMException as _BaseLLMException
LiteLLMLoggingObj = _LiteLLMLoggingObj
BaseLLMException = _BaseLLMException
ImageResponse = _ImageResponse
else:
LiteLLMLoggingObj = Any
BaseLLMException = Any
ImageResponse = Any
class BaseImageEditConfig(ABC):
def __init__(self):
pass
@classmethod
def get_config(cls):
return {
k: v
for k, v in cls.__dict__.items()
if not k.startswith("__")
and not k.startswith("_abc")
and not isinstance(
v,
(
types.FunctionType,
types.BuiltinFunctionType,
classmethod,
staticmethod,
),
)
and v is not None
}
@abstractmethod
def get_supported_openai_params(self, model: str) -> list:
pass
@abstractmethod
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
pass
@abstractmethod
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
return {}
@abstractmethod
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
OPTIONAL
Get the complete url for the request
Some providers need `model` in `api_base`
"""
if api_base is None:
raise ValueError("api_base is required")
return api_base
@abstractmethod
def transform_image_edit_request(
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
pass
@abstractmethod
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ImageResponse:
pass
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
from ..chat.transformation import BaseLLMException
raise BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
)
litellm/llms/bedrock/chat/converse_transformation.py
+9 -5
View File
@@ -45,7 +45,7 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
Usage,
)
from litellm.utils import add_dummy_tool, has_tool_call_blocks
from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning
from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name
@@ -146,9 +146,10 @@ class AmazonConverseConfig(BaseConfig):
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
supported_params.append("tool_choice")
if (
"claude-3-7" in model
): # [TODO]: move to a 'supports_reasoning_content' param from model cost map
if "claude-3-7" in model or "claude-sonnet-4" in model or "claude-opus-4" in model or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
):
supported_params.append("thinking")
supported_params.append("reasoning_effort")
return supported_params
@@ -190,8 +191,11 @@ class AmazonConverseConfig(BaseConfig):
def get_supported_document_types(self) -> List[str]:
return ["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"]
def get_supported_video_types(self) -> List[str]:
return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
def get_all_supported_content_types(self) -> List[str]:
return self.get_supported_image_types() + self.get_supported_document_types()
return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types()
def _create_json_tool_call_for_response_format(
self,

Some files were not shown because too many files have changed in this diff Show More