mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-11 14:22:48 +00:00
Merge branch 'main' into litellm_dev_03_16_2025_p1
This commit is contained in:
Krish Dholakia
@@ -80,7 +80,6 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
#
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
@@ -112,7 +111,11 @@ jobs:
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
|
||||
tags: |
|
||||
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
||||
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
||||
|
||||
@@ -151,8 +154,12 @@ jobs:
|
||||
context: .
|
||||
file: ./docker/Dockerfile.database
|
||||
push: true
|
||||
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
labels: ${{ steps.meta-database.outputs.labels }}
|
||||
tags: |
|
||||
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
||||
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
|
||||
labels: ${{ steps.meta-database.outputs.labels }}
|
||||
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
||||
|
||||
build-and-push-image-non_root:
|
||||
@@ -190,7 +197,11 @@ jobs:
|
||||
context: .
|
||||
file: ./docker/Dockerfile.non_root
|
||||
push: true
|
||||
tags: ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
tags: |
|
||||
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
||||
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
|
||||
labels: ${{ steps.meta-non_root.outputs.labels }}
|
||||
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
||||
|
||||
@@ -229,7 +240,11 @@ jobs:
|
||||
context: .
|
||||
file: ./litellm-js/spend-logs/Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
tags: |
|
||||
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
||||
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
||||
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
|
||||
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
||||
|
||||
build-and-push-helm-chart:
|
||||
|
||||
@@ -9,6 +9,7 @@ help:
|
||||
@echo " make test - Run all tests"
|
||||
@echo " make test-unit - Run unit tests"
|
||||
@echo " make test-integration - Run integration tests"
|
||||
@echo " make test-unit-helm - Run helm unit tests"
|
||||
|
||||
install-dev:
|
||||
poetry install --with dev
|
||||
@@ -25,4 +26,7 @@ test-unit:
|
||||
poetry run pytest tests/litellm/
|
||||
|
||||
test-integration:
|
||||
poetry run pytest tests/ -k "not litellm"
|
||||
poetry run pytest tests/ -k "not litellm"
|
||||
|
||||
test-unit-helm:
|
||||
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
||||
@@ -18,7 +18,7 @@ type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.4.1
|
||||
version: 0.4.2
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
|
||||
@@ -22,6 +22,8 @@ If `db.useStackgresOperator` is used (not yet implemented):
|
||||
| Name | Description | Value |
|
||||
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
|
||||
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
|
||||
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
|
||||
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
|
||||
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A |
|
||||
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
|
||||
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
|
||||
|
||||
@@ -78,8 +78,8 @@ spec:
|
||||
- name: PROXY_MASTER_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "litellm.fullname" . }}-masterkey
|
||||
key: masterkey
|
||||
name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }}
|
||||
key: {{ .Values.masterkeySecretKey | default "masterkey" }}
|
||||
{{- if .Values.redis.enabled }}
|
||||
- name: REDIS_HOST
|
||||
value: {{ include "litellm.redis.serviceName" . }}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
{{- if not .Values.masterkeySecretName }}
|
||||
{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
@@ -5,4 +6,5 @@ metadata:
|
||||
name: {{ include "litellm.fullname" . }}-masterkey
|
||||
data:
|
||||
masterkey: {{ $masterkey | b64enc }}
|
||||
type: Opaque
|
||||
type: Opaque
|
||||
{{- end }}
|
||||
|
||||
@@ -52,3 +52,31 @@ tests:
|
||||
- equal:
|
||||
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0]
|
||||
value: antarctica-east1
|
||||
- it: should work without masterkeySecretName or masterkeySecretKey
|
||||
template: deployment.yaml
|
||||
set:
|
||||
masterkeySecretName: ""
|
||||
masterkeySecretKey: ""
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: PROXY_MASTER_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: RELEASE-NAME-litellm-masterkey
|
||||
key: masterkey
|
||||
- it: should work with masterkeySecretName and masterkeySecretKey
|
||||
template: deployment.yaml
|
||||
set:
|
||||
masterkeySecretName: my-secret
|
||||
masterkeySecretKey: my-key
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: PROXY_MASTER_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: my-secret
|
||||
key: my-key
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
suite: test masterkey secret
|
||||
templates:
|
||||
- secret-masterkey.yaml
|
||||
tests:
|
||||
- it: should create a secret if masterkeySecretName is not set
|
||||
template: secret-masterkey.yaml
|
||||
set:
|
||||
masterkeySecretName: ""
|
||||
asserts:
|
||||
- isKind:
|
||||
of: Secret
|
||||
- it: should not create a secret if masterkeySecretName is set
|
||||
template: secret-masterkey.yaml
|
||||
set:
|
||||
masterkeySecretName: my-secret
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
@@ -75,6 +75,12 @@ ingress:
|
||||
|
||||
# masterkey: changeit
|
||||
|
||||
# if set, use this secret for the master key; otherwise, autogenerate a new one
|
||||
masterkeySecretName: ""
|
||||
|
||||
# if set, use this secret key for the master key; otherwise, use the default key
|
||||
masterkeySecretKey: ""
|
||||
|
||||
# The elements within proxy_config are rendered as config.yaml for the proxy
|
||||
# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
|
||||
# Reference: https://docs.litellm.ai/docs/proxy/configs
|
||||
|
||||
@@ -3,7 +3,13 @@ import TabItem from '@theme/TabItem';
|
||||
|
||||
# Prompt Caching
|
||||
|
||||
For OpenAI + Anthropic + Deepseek, LiteLLM follows the OpenAI prompt caching usage object format:
|
||||
Supported Providers:
|
||||
- OpenAI (`deepseek/`)
|
||||
- Anthropic API (`anthropic/`)
|
||||
- Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
|
||||
- Deepseek API (`deepseek/`)
|
||||
|
||||
For the supported providers, LiteLLM follows the OpenAI prompt caching usage object format:
|
||||
|
||||
```bash
|
||||
"usage": {
|
||||
|
||||
@@ -79,6 +79,7 @@ aws_session_name: Optional[str],
|
||||
aws_profile_name: Optional[str],
|
||||
aws_role_name: Optional[str],
|
||||
aws_web_identity_token: Optional[str],
|
||||
aws_bedrock_runtime_endpoint: Optional[str],
|
||||
```
|
||||
|
||||
### 2. Start the proxy
|
||||
@@ -1262,308 +1263,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Boto3 - Authentication
|
||||
|
||||
### Passing credentials as parameters - Completion()
|
||||
Pass AWS credentials as parameters to litellm.completion
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_access_key_id="",
|
||||
aws_secret_access_key="",
|
||||
aws_region_name="",
|
||||
)
|
||||
```
|
||||
|
||||
### Passing extra headers + Custom API Endpoints
|
||||
|
||||
This can be used to override existing headers (e.g. `Authorization`) when calling custom api endpoints
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
litellm.set_verbose = True # 👈 SEE RAW REQUEST
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_access_key_id="",
|
||||
aws_secret_access_key="",
|
||||
aws_region_name="",
|
||||
aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com",
|
||||
extra_headers={"key": "value"}
|
||||
)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock-model
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-instant-v1
|
||||
aws_access_key_id: "",
|
||||
aws_secret_access_key: "",
|
||||
aws_region_name: "",
|
||||
aws_bedrock_runtime_endpoint: "https://my-fake-endpoint.com",
|
||||
extra_headers: {"key": "value"}
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml --detailed_debug
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "bedrock-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "how can I solve 8x + 7 = -23"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### SSO Login (AWS Profile)
|
||||
- Set `AWS_PROFILE` environment variable
|
||||
- Make bedrock completion call
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
or pass `aws_profile_name`:
|
||||
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_profile_name="dev-profile",
|
||||
)
|
||||
```
|
||||
|
||||
### STS (Role-based Auth)
|
||||
|
||||
- Set `aws_role_name` and `aws_session_name`
|
||||
|
||||
|
||||
| LiteLLM Parameter | Boto3 Parameter | Description | Boto3 Documentation |
|
||||
|------------------|-----------------|-------------|-------------------|
|
||||
| `aws_access_key_id` | `aws_access_key_id` | AWS access key associated with an IAM user or role | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) |
|
||||
| `aws_secret_access_key` | `aws_secret_access_key` | AWS secret key associated with the access key | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) |
|
||||
| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
|
||||
| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
|
||||
|
||||
|
||||
|
||||
Make the bedrock completion call
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
```
|
||||
|
||||
If you also need to dynamically set the aws user accessing the role, add the additional args in the completion()/embedding() function
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_region_name=aws_region_name,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock/*
|
||||
litellm_params:
|
||||
model: bedrock/*
|
||||
aws_role_name: arn:aws:iam::888602223428:role/iam_local_role # AWS RoleArn
|
||||
aws_session_name: "bedrock-session" # AWS RoleSessionName
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # [OPTIONAL - not required if using role]
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # [OPTIONAL - not required if using role]
|
||||
```
|
||||
|
||||
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
|
||||
### Passing an external BedrockRuntime.Client as a parameter - Completion()
|
||||
|
||||
:::warning
|
||||
|
||||
This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues)
|
||||
|
||||
|
||||
Experimental - 2024-Jun-23:
|
||||
`aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` will be extracted from boto3.client and be passed into the httpx client
|
||||
|
||||
:::
|
||||
|
||||
Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth.
|
||||
|
||||
Create a client from session credentials:
|
||||
```python
|
||||
import boto3
|
||||
from litellm import completion
|
||||
|
||||
bedrock = boto3.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id="",
|
||||
aws_secret_access_key="",
|
||||
aws_session_token="",
|
||||
)
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_bedrock_client=bedrock,
|
||||
)
|
||||
```
|
||||
|
||||
Create a client from AWS profile in `~/.aws/config`:
|
||||
```python
|
||||
import boto3
|
||||
from litellm import completion
|
||||
|
||||
dev_session = boto3.Session(profile_name="dev-profile")
|
||||
bedrock = dev_session.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
)
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_bedrock_client=bedrock,
|
||||
)
|
||||
```
|
||||
## Calling via Internal Proxy
|
||||
|
||||
Use the `bedrock/converse_like/model` endpoint to call bedrock converse model via your internal proxy.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/converse_like/some-model",
|
||||
messages=[{"role": "user", "content": "What's AWS?"}],
|
||||
api_key="sk-1234",
|
||||
api_base="https://some-api-url/models",
|
||||
extra_headers={"test": "hello world"},
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: anthropic-claude
|
||||
litellm_params:
|
||||
model: bedrock/converse_like/some-model
|
||||
api_base: https://some-api-url/models
|
||||
```
|
||||
|
||||
2. Start proxy server
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "anthropic-claude",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
|
||||
},
|
||||
{ "content": "Hello, how are you?", "role": "user" }
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
**Expected Output URL**
|
||||
|
||||
```bash
|
||||
https://some-api-url/models
|
||||
```
|
||||
|
||||
## Bedrock Imported Models (Deepseek, Deepseek R1)
|
||||
|
||||
### Deepseek R1
|
||||
@@ -1958,3 +1657,377 @@ curl http://0.0.0.0:4000/rerank \
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Bedrock Application Inference Profile
|
||||
|
||||
Use Bedrock Application Inference Profile to track costs for projects on AWS.
|
||||
|
||||
You can either pass it in the model name - `model="bedrock/arn:...` or as a separate `model_id="arn:..` param.
|
||||
|
||||
### Set via `model_id`
|
||||
|
||||
<Tabs>
|
||||
<TabItem label="SDK" value="sdk">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = ""
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
|
||||
os.environ["AWS_REGION_NAME"] = ""
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem label="PROXY" value="proxy">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: anthropic-claude-3-5-sonnet
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
|
||||
# You have to set the ARN application inference profile in the model_id parameter
|
||||
model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer $LITELLM_API_KEY' \
|
||||
-d '{
|
||||
"model": "anthropic-claude-3-5-sonnet",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "List 5 important events in the XIX century"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Boto3 - Authentication
|
||||
|
||||
### Passing credentials as parameters - Completion()
|
||||
Pass AWS credentials as parameters to litellm.completion
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_access_key_id="",
|
||||
aws_secret_access_key="",
|
||||
aws_region_name="",
|
||||
)
|
||||
```
|
||||
|
||||
### Passing extra headers + Custom API Endpoints
|
||||
|
||||
This can be used to override existing headers (e.g. `Authorization`) when calling custom api endpoints
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
litellm.set_verbose = True # 👈 SEE RAW REQUEST
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_access_key_id="",
|
||||
aws_secret_access_key="",
|
||||
aws_region_name="",
|
||||
aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com",
|
||||
extra_headers={"key": "value"}
|
||||
)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock-model
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-instant-v1
|
||||
aws_access_key_id: "",
|
||||
aws_secret_access_key: "",
|
||||
aws_region_name: "",
|
||||
aws_bedrock_runtime_endpoint: "https://my-fake-endpoint.com",
|
||||
extra_headers: {"key": "value"}
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml --detailed_debug
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "bedrock-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "how can I solve 8x + 7 = -23"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### SSO Login (AWS Profile)
|
||||
- Set `AWS_PROFILE` environment variable
|
||||
- Make bedrock completion call
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
or pass `aws_profile_name`:
|
||||
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_profile_name="dev-profile",
|
||||
)
|
||||
```
|
||||
|
||||
### STS (Role-based Auth)
|
||||
|
||||
- Set `aws_role_name` and `aws_session_name`
|
||||
|
||||
|
||||
| LiteLLM Parameter | Boto3 Parameter | Description | Boto3 Documentation |
|
||||
|------------------|-----------------|-------------|-------------------|
|
||||
| `aws_access_key_id` | `aws_access_key_id` | AWS access key associated with an IAM user or role | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) |
|
||||
| `aws_secret_access_key` | `aws_secret_access_key` | AWS secret key associated with the access key | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) |
|
||||
| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
|
||||
| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
|
||||
|
||||
|
||||
|
||||
Make the bedrock completion call
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
```
|
||||
|
||||
If you also need to dynamically set the aws user accessing the role, add the additional args in the completion()/embedding() function
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_region_name=aws_region_name,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock/*
|
||||
litellm_params:
|
||||
model: bedrock/*
|
||||
aws_role_name: arn:aws:iam::888602223428:role/iam_local_role # AWS RoleArn
|
||||
aws_session_name: "bedrock-session" # AWS RoleSessionName
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # [OPTIONAL - not required if using role]
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # [OPTIONAL - not required if using role]
|
||||
```
|
||||
|
||||
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
|
||||
### Passing an external BedrockRuntime.Client as a parameter - Completion()
|
||||
|
||||
:::warning
|
||||
|
||||
This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues)
|
||||
|
||||
|
||||
Experimental - 2024-Jun-23:
|
||||
`aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` will be extracted from boto3.client and be passed into the httpx client
|
||||
|
||||
:::
|
||||
|
||||
Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth.
|
||||
|
||||
Create a client from session credentials:
|
||||
```python
|
||||
import boto3
|
||||
from litellm import completion
|
||||
|
||||
bedrock = boto3.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id="",
|
||||
aws_secret_access_key="",
|
||||
aws_session_token="",
|
||||
)
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_bedrock_client=bedrock,
|
||||
)
|
||||
```
|
||||
|
||||
Create a client from AWS profile in `~/.aws/config`:
|
||||
```python
|
||||
import boto3
|
||||
from litellm import completion
|
||||
|
||||
dev_session = boto3.Session(profile_name="dev-profile")
|
||||
bedrock = dev_session.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
)
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
aws_bedrock_client=bedrock,
|
||||
)
|
||||
```
|
||||
## Calling via Internal Proxy (not bedrock url compatible)
|
||||
|
||||
Use the `bedrock/converse_like/model` endpoint to call bedrock converse model via your internal proxy.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="bedrock/converse_like/some-model",
|
||||
messages=[{"role": "user", "content": "What's AWS?"}],
|
||||
api_key="sk-1234",
|
||||
api_base="https://some-api-url/models",
|
||||
extra_headers={"test": "hello world"},
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: anthropic-claude
|
||||
litellm_params:
|
||||
model: bedrock/converse_like/some-model
|
||||
api_base: https://some-api-url/models
|
||||
```
|
||||
|
||||
2. Start proxy server
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "anthropic-claude",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
|
||||
},
|
||||
{ "content": "Hello, how are you?", "role": "user" }
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
**Expected Output URL**
|
||||
|
||||
```bash
|
||||
https://some-api-url/models
|
||||
```
|
||||
|
||||
@@ -10,17 +10,13 @@ Role-based access control (RBAC) is based on Organizations, Teams and Internal U
|
||||
|
||||
## Roles
|
||||
|
||||
**Admin Roles**
|
||||
- `proxy_admin`: admin over the platform
|
||||
- `proxy_admin_viewer`: can login, view all keys, view all spend. **Cannot** create keys/delete keys/add new users
|
||||
|
||||
**Organization Roles**
|
||||
- `org_admin`: admin over the organization. Can create teams and users within their organization
|
||||
|
||||
**Internal User Roles**
|
||||
- `internal_user`: can login, view/create/delete their own keys, view their spend. **Cannot** add new users.
|
||||
- `internal_user_viewer`: can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users.
|
||||
|
||||
| Role Type | Role Name | Permissions |
|
||||
|-----------|-----------|-------------|
|
||||
| **Admin** | `proxy_admin` | Admin over the platform |
|
||||
| | `proxy_admin_viewer` | Can login, view all keys, view all spend. **Cannot** create keys/delete keys/add new users |
|
||||
| **Organization** | `org_admin` | Admin over the organization. Can create teams and users within their organization |
|
||||
| **Internal User** | `internal_user` | Can login, view/create/delete their own keys, view their spend. **Cannot** add new users |
|
||||
| | `internal_user_viewer` | Can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users |
|
||||
|
||||
## Onboarding Organizations
|
||||
|
||||
|
||||
@@ -102,7 +102,19 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Advanced - Set Accepted JWT Scope Names
|
||||
## Advanced
|
||||
|
||||
### Multiple OIDC providers
|
||||
|
||||
Use this if you want LiteLLM to validate your JWT against multiple OIDC providers (e.g. Google Cloud, GitHub Auth)
|
||||
|
||||
Set `JWT_PUBLIC_KEY_URL` in your environment to a comma-separated list of URLs for your OIDC providers.
|
||||
|
||||
```bash
|
||||
export JWT_PUBLIC_KEY_URL="https://demo.duendesoftware.com/.well-known/openid-configuration/jwks,https://accounts.google.com/.well-known/openid-configuration/jwks"
|
||||
```
|
||||
|
||||
### Set Accepted JWT Scope Names
|
||||
|
||||
Change the string in JWT 'scopes', that litellm evaluates to see if a user has admin access.
|
||||
|
||||
@@ -114,7 +126,7 @@ general_settings:
|
||||
admin_jwt_scope: "litellm-proxy-admin"
|
||||
```
|
||||
|
||||
## Tracking End-Users / Internal Users / Team / Org
|
||||
### Tracking End-Users / Internal Users / Team / Org
|
||||
|
||||
Set the field in the jwt token, which corresponds to a litellm user / team / org.
|
||||
|
||||
@@ -156,7 +168,7 @@ scope: ["litellm-proxy-admin",...]
|
||||
scope: "litellm-proxy-admin ..."
|
||||
```
|
||||
|
||||
## Control model access with Teams
|
||||
### Control model access with Teams
|
||||
|
||||
|
||||
1. Specify the JWT field that contains the team ids, that the user belongs to.
|
||||
@@ -207,11 +219,11 @@ OIDC Auth for API: [**See Walkthrough**](https://www.loom.com/share/00fe2deab59a
|
||||
- If all checks pass, allow the request
|
||||
|
||||
|
||||
## Advanced - Custom Validate
|
||||
### Custom JWT Validate
|
||||
|
||||
Validate a JWT Token using custom logic, if you need an extra way to verify if tokens are valid for LiteLLM Proxy.
|
||||
|
||||
### 1. Setup custom validate function
|
||||
#### 1. Setup custom validate function
|
||||
|
||||
```python
|
||||
from typing import Literal
|
||||
@@ -230,7 +242,7 @@ def my_custom_validate(token: str) -> Literal[True]:
|
||||
return True
|
||||
```
|
||||
|
||||
### 2. Setup config.yaml
|
||||
#### 2. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
@@ -243,7 +255,7 @@ general_settings:
|
||||
custom_validate: custom_validate.my_custom_validate # 👈 custom validate function
|
||||
```
|
||||
|
||||
### 3. Test the flow
|
||||
#### 3. Test the flow
|
||||
|
||||
**Expected JWT**
|
||||
|
||||
@@ -265,7 +277,7 @@ general_settings:
|
||||
|
||||
|
||||
|
||||
## Advanced - Allowed Routes
|
||||
### Allowed Routes
|
||||
|
||||
Configure which routes a JWT can access via the config.
|
||||
|
||||
@@ -297,7 +309,7 @@ general_settings:
|
||||
team_allowed_routes: ["/v1/chat/completions"] # 👈 Set accepted routes
|
||||
```
|
||||
|
||||
## Advanced - Caching Public Keys
|
||||
### Caching Public Keys
|
||||
|
||||
Control how long public keys are cached for (in seconds).
|
||||
|
||||
@@ -311,7 +323,7 @@ general_settings:
|
||||
public_key_ttl: 600 # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
## Advanced - Custom JWT Field
|
||||
### Custom JWT Field
|
||||
|
||||
Set a custom field in which the team_id exists. By default, the 'client_id' field is checked.
|
||||
|
||||
@@ -323,14 +335,7 @@ general_settings:
|
||||
team_id_jwt_field: "client_id" # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
## All Params
|
||||
|
||||
[**See Code**](https://github.com/BerriAI/litellm/blob/b204f0c01c703317d812a1553363ab0cb989d5b6/litellm/proxy/_types.py#L95)
|
||||
|
||||
|
||||
|
||||
|
||||
## Advanced - Block Teams
|
||||
### Block Teams
|
||||
|
||||
To block all requests for a certain team id, use `/team/block`
|
||||
|
||||
@@ -357,7 +362,7 @@ curl --location 'http://0.0.0.0:4000/team/unblock' \
|
||||
```
|
||||
|
||||
|
||||
## Advanced - Upsert Users + Allowed Email Domains
|
||||
### Upsert Users + Allowed Email Domains
|
||||
|
||||
Allow users who belong to a specific email domain, automatic access to the proxy.
|
||||
|
||||
@@ -494,4 +499,10 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
```
|
||||
|
||||
## All JWT Params
|
||||
|
||||
[**See Code**](https://github.com/BerriAI/litellm/blob/b204f0c01c703317d812a1553363ab0cb989d5b6/litellm/proxy/_types.py#L95)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Adding LLM Credentials
|
||||
|
||||
You can add LLM provider credentials on the UI. Once you add credentials you can re-use them when adding new models
|
||||
|
||||
## Add a credential + model
|
||||
|
||||
### 1. Navigate to LLM Credentials page
|
||||
|
||||
Go to Models -> LLM Credentials -> Add Credential
|
||||
|
||||
<Image img={require('../../img/ui_cred_add.png')} />
|
||||
|
||||
### 2. Add credentials
|
||||
|
||||
Select your LLM provider, enter your API Key and click "Add Credential"
|
||||
|
||||
**Note: Credentials are based on the provider, if you select Vertex AI then you will see `Vertex Project`, `Vertex Location` and `Vertex Credentials` fields**
|
||||
|
||||
<Image img={require('../../img/ui_add_cred_2.png')} />
|
||||
|
||||
|
||||
### 3. Use credentials when adding a model
|
||||
|
||||
Go to Add Model -> Existing Credentials -> Select your credential in the dropdown
|
||||
|
||||
<Image img={require('../../img/ui_cred_3.png')} />
|
||||
|
||||
|
||||
## Create a Credential from an existing model
|
||||
|
||||
Use this if you have already created a model and want to store the model credentials for future use
|
||||
|
||||
### 1. Select model to create a credential from
|
||||
|
||||
Go to Models -> Select your model -> Credential -> Create Credential
|
||||
|
||||
<Image img={require('../../img/ui_cred_4.png')} />
|
||||
|
||||
### 2. Use new credential when adding a model
|
||||
|
||||
Go to Add Model -> Existing Credentials -> Select your credential in the dropdown
|
||||
|
||||
<Image img={require('../../img/use_model_cred.png')} />
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
|
||||
How are credentials stored?
|
||||
Credentials in the DB are encrypted/decrypted using `LITELLM_SALT_KEY`, if set. If not, then they are encrypted using `LITELLM_MASTER_KEY`. These keys should be kept secret and not shared with others.
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# UI Logs Page
|
||||
|
||||
View Spend, Token Usage, Key, Team Name for Each Request to LiteLLM
|
||||
|
||||
|
||||
<Image img={require('../../img/ui_request_logs.png')}/>
|
||||
|
||||
|
||||
## Overview
|
||||
|
||||
| Log Type | Tracked by Default |
|
||||
|----------|-------------------|
|
||||
| Success Logs | ✅ Yes |
|
||||
| Error Logs | ✅ Yes |
|
||||
| Request/Response Content Stored | ❌ No by Default, **opt in with `store_prompts_in_spend_logs`** |
|
||||
|
||||
|
||||
|
||||
**By default LiteLLM does not track the request and response content.**
|
||||
|
||||
## Tracking - Request / Response Content in Logs Page
|
||||
|
||||
If you want to view request and response content on LiteLLM Logs, you need to opt in with this setting
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
store_prompts_in_spend_logs: true
|
||||
```
|
||||
|
||||
<Image img={require('../../img/ui_request_logs_content.png')}/>
|
||||
|
||||
|
||||
## Stop storing Error Logs in DB
|
||||
|
||||
If you do not want to store error logs in DB, you can opt out with this setting
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
disable_error_logs: True # Only disable writing error logs to DB, regular spend logs will still be written unless `disable_spend_logs: True`
|
||||
```
|
||||
|
||||
## Stop storing Spend Logs in DB
|
||||
|
||||
If you do not want to store spend logs in DB, you can opt out with this setting
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
disable_spend_logs: True # Disable writing spend logs to DB
|
||||
```
|
||||
|
||||
@@ -14,6 +14,9 @@ Supported Providers:
|
||||
- Anthropic API (`anthropic/`)
|
||||
- Bedrock (Anthropic + Deepseek) (`bedrock/`)
|
||||
- Vertex AI (Anthropic) (`vertexai/`)
|
||||
- OpenRouter (`openrouter/`)
|
||||
|
||||
LiteLLM will standardize the `reasoning_content` in the response and `thinking_blocks` in the assistant message.
|
||||
|
||||
```python
|
||||
"message": {
|
||||
|
||||
@@ -96,7 +96,7 @@ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
|
||||
### Using K/V pairs in 1 AWS Secret
|
||||
#### Using K/V pairs in 1 AWS Secret
|
||||
|
||||
You can read multiple keys from a single AWS Secret using the `primary_secret_name` parameter:
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 371 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 67 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 255 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 283 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 255 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 204 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 567 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 344 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 282 KiB |
@@ -0,0 +1,176 @@
|
||||
---
|
||||
title: v1.63.11-stable
|
||||
slug: v1.63.11-stable
|
||||
date: 2025-03-15T10:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D4D03AQGrlsJ3aqpHmQ/profile-displayphoto-shrink_400_400/B4DZSAzgP7HYAg-/0/1737327772964?e=1743638400&v=beta&t=39KOXMUFedvukiWWVPHf3qI45fuQD7lNglICwN31DrI
|
||||
- name: Ishaan Jaffer
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
|
||||
tags: [credential management, thinking content, responses api, snowflake]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
|
||||
These are the changes since `v1.63.2-stable`.
|
||||
|
||||
This release is primarily focused on:
|
||||
- [Beta] Responses API Support
|
||||
- Snowflake Cortex Support, Amazon Nova Image Generation
|
||||
- UI - Credential Management, re-use credentials when adding new models
|
||||
- UI - Test Connection to LLM Provider before adding a model
|
||||
|
||||
:::info
|
||||
|
||||
This release will be live on 03/16/2025
|
||||
|
||||
:::
|
||||
|
||||
<!-- <Image img={require('../../img/release_notes/v16311_release.jpg')} /> -->
|
||||
|
||||
## Docker Run LiteLLM Proxy
|
||||
|
||||
```
|
||||
docker run
|
||||
-e STORE_MODEL_IN_DB=True
|
||||
-p 4000:4000
|
||||
ghcr.io/berriai/litellm:main-v1.63.11-stable
|
||||
```
|
||||
|
||||
## Demo Instance
|
||||
|
||||
Here's a Demo Instance to test changes:
|
||||
- Instance: https://demo.litellm.ai/
|
||||
- Login Credentials:
|
||||
- Username: admin
|
||||
- Password: sk-1234
|
||||
|
||||
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
- Image Generation support for Amazon Nova Canvas [Getting Started](https://docs.litellm.ai/docs/providers/bedrock#image-generation)
|
||||
- Add pricing for Jamba new models [PR](https://github.com/BerriAI/litellm/pull/9032/files)
|
||||
- Add pricing for Amazon EU models [PR](https://github.com/BerriAI/litellm/pull/9056/files)
|
||||
- Add Bedrock Deepseek R1 model pricing [PR](https://github.com/BerriAI/litellm/pull/9108/files)
|
||||
- Update Gemini pricing: Gemma 3, Flash 2 thinking update, LearnLM [PR](https://github.com/BerriAI/litellm/pull/9190/files)
|
||||
- Mark Cohere Embedding 3 models as Multimodal [PR](https://github.com/BerriAI/litellm/pull/9176/commits/c9a576ce4221fc6e50dc47cdf64ab62736c9da41)
|
||||
- Add Azure Data Zone pricing [PR](https://github.com/BerriAI/litellm/pull/9185/files#diff-19ad91c53996e178c1921cbacadf6f3bae20cfe062bd03ee6bfffb72f847ee37)
|
||||
- LiteLLM Tracks cost for `azure/eu` and `azure/us` models
|
||||
|
||||
|
||||
|
||||
## LLM Translation
|
||||
|
||||
<Image img={require('../../img/release_notes/responses_api.png')} />
|
||||
|
||||
1. **New Endpoints**
|
||||
- [Beta] POST `/responses` API. [Getting Started](https://docs.litellm.ai/docs/response_api)
|
||||
|
||||
2. **New LLM Providers**
|
||||
- Snowflake Cortex [Getting Started](https://docs.litellm.ai/docs/providers/snowflake)
|
||||
|
||||
3. **New LLM Features**
|
||||
|
||||
- Support OpenRouter `reasoning_content` on streaming [Getting Started](https://docs.litellm.ai/docs/reasoning_content)
|
||||
|
||||
4. **Bug Fixes**
|
||||
|
||||
- OpenAI: Return `code`, `param` and `type` on bad request error [More information on litellm exceptions](https://docs.litellm.ai/docs/exception_mapping)
|
||||
- Bedrock: Fix converse chunk parsing to only return empty dict on tool use [PR](https://github.com/BerriAI/litellm/pull/9166)
|
||||
- Bedrock: Support extra_headers [PR](https://github.com/BerriAI/litellm/pull/9113)
|
||||
- Azure: Fix Function Calling Bug & Update Default API Version to `2025-02-01-preview` [PR](https://github.com/BerriAI/litellm/pull/9191)
|
||||
- Azure: Fix AI services URL [PR](https://github.com/BerriAI/litellm/pull/9185)
|
||||
- Vertex AI: Handle HTTP 201 status code in response [PR](https://github.com/BerriAI/litellm/pull/9193)
|
||||
- Perplexity: Fix incorrect streaming response [PR](https://github.com/BerriAI/litellm/pull/9081)
|
||||
- Triton: Fix streaming completions bug [PR](https://github.com/BerriAI/litellm/pull/8386)
|
||||
- Deepgram: Support bytes.IO when handling audio files for transcription [PR](https://github.com/BerriAI/litellm/pull/9071)
|
||||
- Ollama: Fix "system" role has become unacceptable [PR](https://github.com/BerriAI/litellm/pull/9261)
|
||||
- All Providers (Streaming): Fix String `data:` stripped from entire content in streamed responses [PR](https://github.com/BerriAI/litellm/pull/9070)
|
||||
|
||||
|
||||
|
||||
## Spend Tracking Improvements
|
||||
|
||||
1. Support Bedrock converse cache token tracking [Getting Started](https://docs.litellm.ai/docs/completion/prompt_caching)
|
||||
2. Cost Tracking for Responses API [Getting Started](https://docs.litellm.ai/docs/response_api)
|
||||
3. Fix Azure Whisper cost tracking [Getting Started](https://docs.litellm.ai/docs/audio_transcription)
|
||||
|
||||
|
||||
## UI
|
||||
|
||||
### Re-Use Credentials on UI
|
||||
|
||||
You can now onboard LLM provider credentials on LiteLLM UI. Once these credentials are added you can re-use them when adding new models [Getting Started](https://docs.litellm.ai/docs/proxy/ui_credentials)
|
||||
|
||||
<Image img={require('../../img/release_notes/credentials.jpg')} />
|
||||
|
||||
|
||||
### Test Connections before adding models
|
||||
|
||||
Before adding a model you can test the connection to the LLM provider to verify you have setup your API Base + API Key correctly
|
||||
|
||||
<Image img={require('../../img/release_notes/litellm_test_connection.gif')} />
|
||||
|
||||
### General UI Improvements
|
||||
1. Add Models Page
|
||||
- Allow adding Cerebras, Sambanova, Perplexity, Fireworks, Openrouter, TogetherAI Models, Text-Completion OpenAI on Admin UI
|
||||
- Allow adding EU OpenAI models
|
||||
- Fix: Instantly show edit + deletes to models
|
||||
2. Keys Page
|
||||
- Fix: Instantly show newly created keys on Admin UI (don't require refresh)
|
||||
- Fix: Allow clicking into Top Keys when showing users Top API Key
|
||||
- Fix: Allow Filter Keys by Team Alias, Key Alias and Org
|
||||
- UI Improvements: Show 100 Keys Per Page, Use full height, increase width of key alias
|
||||
3. Users Page
|
||||
- Fix: Show correct count of internal user keys on Users Page
|
||||
- Fix: Metadata not updating in Team UI
|
||||
4. Logs Page
|
||||
- UI Improvements: Keep expanded log in focus on LiteLLM UI
|
||||
- UI Improvements: Minor improvements to logs page
|
||||
- Fix: Allow internal user to query their own logs
|
||||
- Allow switching off storing Error Logs in DB [Getting Started](https://docs.litellm.ai/docs/proxy/ui_logs)
|
||||
5. Sign In/Sign Out
|
||||
- Fix: Correctly use `PROXY_LOGOUT_URL` when set [Getting Started](https://docs.litellm.ai/docs/proxy/self_serve#setting-custom-logout-urls)
|
||||
|
||||
|
||||
## Security
|
||||
|
||||
1. Support for Rotating Master Keys [Getting Started](https://docs.litellm.ai/docs/proxy/master_key_rotations)
|
||||
2. Fix: Internal User Viewer Permissions, don't allow `internal_user_viewer` role to see `Test Key Page` or `Create Key Button` [More information on role based access controls](https://docs.litellm.ai/docs/proxy/access_control)
|
||||
3. Emit audit logs on All user + model Create/Update/Delete endpoints [Getting Started](https://docs.litellm.ai/docs/proxy/multiple_admins)
|
||||
4. JWT
|
||||
- Support multiple JWT OIDC providers [Getting Started](https://docs.litellm.ai/docs/proxy/token_auth)
|
||||
- Fix JWT access with Groups not working when team is assigned All Proxy Models access
|
||||
5. Using K/V pairs in 1 AWS Secret [Getting Started](https://docs.litellm.ai/docs/secret#using-kv-pairs-in-1-aws-secret)
|
||||
|
||||
|
||||
## Logging Integrations
|
||||
|
||||
1. Prometheus: Track Azure LLM API latency metric [Getting Started](https://docs.litellm.ai/docs/proxy/prometheus#request-latency-metrics)
|
||||
2. Athina: Added tags, user_feedback and model_options to additional_keys which can be sent to Athina [Getting Started](https://docs.litellm.ai/docs/observability/athina_integration)
|
||||
|
||||
|
||||
## Performance / Reliability improvements
|
||||
|
||||
1. Redis + litellm router - Fix Redis cluster mode for litellm router [PR](https://github.com/BerriAI/litellm/pull/9010)
|
||||
|
||||
|
||||
## General Improvements
|
||||
|
||||
1. OpenWebUI Integration - display `thinking` tokens
|
||||
- Guide on getting started with LiteLLM x OpenWebUI. [Getting Started](https://docs.litellm.ai/docs/tutorials/openweb_ui)
|
||||
- Display `thinking` tokens on OpenWebUI (Bedrock, Anthropic, Deepseek) [Getting Started](https://docs.litellm.ai/docs/tutorials/openweb_ui#render-thinking-content-on-openweb-ui)
|
||||
|
||||
<Image img={require('../../img/litellm_thinking_openweb.gif')} />
|
||||
|
||||
|
||||
## Complete Git Diff
|
||||
|
||||
[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.2-stable...v1.63.11-stable)
|
||||
@@ -101,7 +101,9 @@ const sidebars = {
|
||||
"proxy/admin_ui_sso",
|
||||
"proxy/self_serve",
|
||||
"proxy/public_teams",
|
||||
"proxy/custom_sso"
|
||||
"proxy/custom_sso",
|
||||
"proxy/ui_credentials",
|
||||
"proxy/ui_logs"
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -279,16 +279,30 @@ class BaseAWSLLM:
|
||||
return None
|
||||
|
||||
def _get_aws_region_name(
|
||||
self, optional_params: dict, model: Optional[str] = None
|
||||
self,
|
||||
optional_params: dict,
|
||||
model: Optional[str] = None,
|
||||
model_id: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the AWS region name from the environment variables
|
||||
Get the AWS region name from the environment variables.
|
||||
|
||||
Parameters:
|
||||
optional_params (dict): Optional parameters for the model call
|
||||
model (str): The model name
|
||||
model_id (str): The model ID. This is the ARN of the model, if passed in as a separate param.
|
||||
|
||||
Returns:
|
||||
str: The AWS region name
|
||||
"""
|
||||
aws_region_name = optional_params.get("aws_region_name", None)
|
||||
### SET REGION NAME ###
|
||||
if aws_region_name is None:
|
||||
# check model arn #
|
||||
aws_region_name = self._get_aws_region_from_model_arn(model)
|
||||
if model_id is not None:
|
||||
aws_region_name = self._get_aws_region_from_model_arn(model_id)
|
||||
else:
|
||||
aws_region_name = self._get_aws_region_from_model_arn(model)
|
||||
# check env #
|
||||
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper, get_secret
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, Credentials
|
||||
from ..common_utils import BedrockError
|
||||
@@ -268,23 +268,29 @@ class BedrockConverseLLM(BaseAWSLLM):
|
||||
|
||||
## SETUP ##
|
||||
stream = optional_params.pop("stream", None)
|
||||
modelId = optional_params.pop("model_id", None)
|
||||
unencoded_model_id = optional_params.pop("model_id", None)
|
||||
fake_stream = optional_params.pop("fake_stream", False)
|
||||
json_mode = optional_params.get("json_mode", False)
|
||||
if modelId is not None:
|
||||
modelId = self.encode_model_id(model_id=modelId)
|
||||
if unencoded_model_id is not None:
|
||||
modelId = self.encode_model_id(model_id=unencoded_model_id)
|
||||
else:
|
||||
modelId = self.encode_model_id(model_id=model)
|
||||
|
||||
if stream is True and "ai21" in modelId:
|
||||
fake_stream = True
|
||||
|
||||
### SET REGION NAME ###
|
||||
aws_region_name = self._get_aws_region_name(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
model_id=unencoded_model_id,
|
||||
)
|
||||
|
||||
## CREDENTIALS ##
|
||||
# pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
|
||||
aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
|
||||
aws_access_key_id = optional_params.pop("aws_access_key_id", None)
|
||||
aws_session_token = optional_params.pop("aws_session_token", None)
|
||||
aws_region_name = optional_params.pop("aws_region_name", None)
|
||||
aws_role_name = optional_params.pop("aws_role_name", None)
|
||||
aws_session_name = optional_params.pop("aws_session_name", None)
|
||||
aws_profile_name = optional_params.pop("aws_profile_name", None)
|
||||
@@ -293,25 +299,7 @@ class BedrockConverseLLM(BaseAWSLLM):
|
||||
) # https://bedrock-runtime.{region_name}.amazonaws.com
|
||||
aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
|
||||
aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
|
||||
|
||||
### SET REGION NAME ###
|
||||
if aws_region_name is None:
|
||||
# check env #
|
||||
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
||||
|
||||
if litellm_aws_region_name is not None and isinstance(
|
||||
litellm_aws_region_name, str
|
||||
):
|
||||
aws_region_name = litellm_aws_region_name
|
||||
|
||||
standard_aws_region_name = get_secret("AWS_REGION", None)
|
||||
if standard_aws_region_name is not None and isinstance(
|
||||
standard_aws_region_name, str
|
||||
):
|
||||
aws_region_name = standard_aws_region_name
|
||||
|
||||
if aws_region_name is None:
|
||||
aws_region_name = "us-west-2"
|
||||
optional_params.pop("aws_region_name", None)
|
||||
|
||||
litellm_params["aws_region_name"] = (
|
||||
aws_region_name # [DO NOT DELETE] important for async calls
|
||||
|
||||
@@ -2950,15 +2950,39 @@ async def test_bedrock_stream_thinking_content_openwebui():
|
||||
), "There should be non-empty content after thinking tags"
|
||||
|
||||
|
||||
def test_bedrock_streaming():
|
||||
resp = completion(
|
||||
model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello who is this?"}],
|
||||
stream=True,
|
||||
max_tokens=1080,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
def test_bedrock_application_inference_profile():
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler
|
||||
|
||||
for chunk in resp:
|
||||
if hasattr(chunk, "usage"):
|
||||
print(f"received chunk usage: {chunk.usage}")
|
||||
client = HTTPHandler()
|
||||
client2 = HTTPHandler()
|
||||
|
||||
with patch.object(client, "post") as mock_post, patch.object(
|
||||
client2, "post"
|
||||
) as mock_post2:
|
||||
try:
|
||||
resp = completion(
|
||||
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
|
||||
client=client,
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
try:
|
||||
resp = completion(
|
||||
model="bedrock/converse/arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
client=client2,
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
mock_post2.assert_called_once()
|
||||
print(mock_post.call_args.kwargs)
|
||||
json_data = mock_post.call_args.kwargs["data"]
|
||||
assert mock_post.call_args.kwargs["url"].startswith(
|
||||
"https://bedrock-runtime.eu-central-1.amazonaws.com/"
|
||||
)
|
||||
assert mock_post2.call_args.kwargs["url"] == mock_post.call_args.kwargs["url"]
|
||||
|
||||
Reference in New Issue
Block a user