diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index 587abc8ea7..58c8a1e2e1 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -80,7 +80,6 @@ jobs: permissions: contents: read packages: write - # steps: - name: Checkout repository uses: actions/checkout@v4 @@ -112,7 +111,11 @@ jobs: with: context: . push: true - tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' + tags: | + ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, + ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }} labels: ${{ steps.meta.outputs.labels }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 @@ -151,8 +154,12 @@ jobs: context: . file: ./docker/Dockerfile.database push: true - tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} - labels: ${{ steps.meta-database.outputs.labels }} + tags: | + ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, + ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }} + labels: ${{ steps.meta-database.outputs.labels }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 build-and-push-image-non_root: @@ -190,7 +197,11 @@ jobs: context: . file: ./docker/Dockerfile.non_root push: true - tags: ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} + tags: | + ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, + ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }} labels: ${{ steps.meta-non_root.outputs.labels }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 @@ -229,7 +240,11 @@ jobs: context: . file: ./litellm-js/spend-logs/Dockerfile push: true - tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} + tags: | + ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, + ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, + ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 build-and-push-helm-chart: diff --git a/Makefile b/Makefile index 6555326168..6c231d3cc2 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ help: @echo " make test - Run all tests" @echo " make test-unit - Run unit tests" @echo " make test-integration - Run integration tests" + @echo " make test-unit-helm - Run helm unit tests" install-dev: poetry install --with dev @@ -25,4 +26,7 @@ test-unit: poetry run pytest tests/litellm/ test-integration: - poetry run pytest tests/ -k "not litellm" \ No newline at end of file + poetry run pytest tests/ -k "not litellm" + +test-unit-helm: + helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm \ No newline at end of file diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index f1f2fd8d64..4d856fdc0f 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.1 +version: 0.4.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 8b2196f577..a0ba5781df 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -22,6 +22,8 @@ If `db.useStackgresOperator` is used (not yet implemented): | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | | `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | +| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | +| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | | `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A | | `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | | `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 697148abf8..52f761ed15 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -78,8 +78,8 @@ spec: - name: PROXY_MASTER_KEY valueFrom: secretKeyRef: - name: {{ include "litellm.fullname" . }}-masterkey - key: masterkey + name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} + key: {{ .Values.masterkeySecretKey | default "masterkey" }} {{- if .Values.redis.enabled }} - name: REDIS_HOST value: {{ include "litellm.redis.serviceName" . }} diff --git a/deploy/charts/litellm-helm/templates/secret-masterkey.yaml b/deploy/charts/litellm-helm/templates/secret-masterkey.yaml index 57b854cc0f..5632957dc0 100644 --- a/deploy/charts/litellm-helm/templates/secret-masterkey.yaml +++ b/deploy/charts/litellm-helm/templates/secret-masterkey.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.masterkeySecretName }} {{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }} apiVersion: v1 kind: Secret @@ -5,4 +6,5 @@ metadata: name: {{ include "litellm.fullname" . }}-masterkey data: masterkey: {{ $masterkey | b64enc }} -type: Opaque \ No newline at end of file +type: Opaque +{{- end }} diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml index e7ce44b052..0e4b8e0b1f 100644 --- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml +++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml @@ -52,3 +52,31 @@ tests: - equal: path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] value: antarctica-east1 + - it: should work without masterkeySecretName or masterkeySecretKey + template: deployment.yaml + set: + masterkeySecretName: "" + masterkeySecretKey: "" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: RELEASE-NAME-litellm-masterkey + key: masterkey + - it: should work with masterkeySecretName and masterkeySecretKey + template: deployment.yaml + set: + masterkeySecretName: my-secret + masterkeySecretKey: my-key + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: my-secret + key: my-key diff --git a/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml b/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml new file mode 100644 index 0000000000..eb1d3c3967 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml @@ -0,0 +1,18 @@ +suite: test masterkey secret +templates: + - secret-masterkey.yaml +tests: + - it: should create a secret if masterkeySecretName is not set + template: secret-masterkey.yaml + set: + masterkeySecretName: "" + asserts: + - isKind: + of: Secret + - it: should not create a secret if masterkeySecretName is set + template: secret-masterkey.yaml + set: + masterkeySecretName: my-secret + asserts: + - hasDocuments: + count: 0 diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index 9f21fc40ad..70f6c2ef23 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -75,6 +75,12 @@ ingress: # masterkey: changeit +# if set, use this secret for the master key; otherwise, autogenerate a new one +masterkeySecretName: "" + +# if set, use this secret key for the master key; otherwise, use the default key +masterkeySecretKey: "" + # The elements within proxy_config are rendered as config.yaml for the proxy # Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml # Reference: https://docs.litellm.ai/docs/proxy/configs diff --git a/docs/my-website/docs/completion/prompt_caching.md b/docs/my-website/docs/completion/prompt_caching.md index 5c795778ef..6fbf89bd6d 100644 --- a/docs/my-website/docs/completion/prompt_caching.md +++ b/docs/my-website/docs/completion/prompt_caching.md @@ -3,7 +3,13 @@ import TabItem from '@theme/TabItem'; # Prompt Caching -For OpenAI + Anthropic + Deepseek, LiteLLM follows the OpenAI prompt caching usage object format: +Supported Providers: +- OpenAI (`deepseek/`) +- Anthropic API (`anthropic/`) +- Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html)) +- Deepseek API (`deepseek/`) + +For the supported providers, LiteLLM follows the OpenAI prompt caching usage object format: ```bash "usage": { diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 1416006bf1..45ad3f0c61 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -79,6 +79,7 @@ aws_session_name: Optional[str], aws_profile_name: Optional[str], aws_role_name: Optional[str], aws_web_identity_token: Optional[str], +aws_bedrock_runtime_endpoint: Optional[str], ``` ### 2. Start the proxy @@ -1262,308 +1263,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -## Boto3 - Authentication - -### Passing credentials as parameters - Completion() -Pass AWS credentials as parameters to litellm.completion -```python -import os -from litellm import completion - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_access_key_id="", - aws_secret_access_key="", - aws_region_name="", -) -``` - -### Passing extra headers + Custom API Endpoints - -This can be used to override existing headers (e.g. `Authorization`) when calling custom api endpoints - - - - -```python -import os -import litellm -from litellm import completion - -litellm.set_verbose = True # 👈 SEE RAW REQUEST - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_access_key_id="", - aws_secret_access_key="", - aws_region_name="", - aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com", - extra_headers={"key": "value"} -) -``` - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: bedrock-model - litellm_params: - model: bedrock/anthropic.claude-instant-v1 - aws_access_key_id: "", - aws_secret_access_key: "", - aws_region_name: "", - aws_bedrock_runtime_endpoint: "https://my-fake-endpoint.com", - extra_headers: {"key": "value"} -``` - -2. Start proxy - -```bash -litellm --config /path/to/config.yaml --detailed_debug -``` - -3. Test it! - -```bash -curl -X POST 'http://0.0.0.0:4000/chat/completions' \ --H 'Content-Type: application/json' \ --H 'Authorization: Bearer sk-1234' \ --d '{ - "model": "bedrock-model", - "messages": [ - { - "role": "system", - "content": "You are a helpful math tutor. Guide the user through the solution step by step." - }, - { - "role": "user", - "content": "how can I solve 8x + 7 = -23" - } - ] -}' -``` - - - - -### SSO Login (AWS Profile) -- Set `AWS_PROFILE` environment variable -- Make bedrock completion call -```python -import os -from litellm import completion - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - -or pass `aws_profile_name`: - -```python -import os -from litellm import completion - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_profile_name="dev-profile", -) -``` - -### STS (Role-based Auth) - -- Set `aws_role_name` and `aws_session_name` - - -| LiteLLM Parameter | Boto3 Parameter | Description | Boto3 Documentation | -|------------------|-----------------|-------------|-------------------| -| `aws_access_key_id` | `aws_access_key_id` | AWS access key associated with an IAM user or role | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) | -| `aws_secret_access_key` | `aws_secret_access_key` | AWS secret key associated with the access key | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) | -| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) | -| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) | - - - -Make the bedrock completion call - - - - -```python -from litellm import completion - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=messages, - max_tokens=10, - temperature=0.1, - aws_role_name=aws_role_name, - aws_session_name="my-test-session", - ) -``` - -If you also need to dynamically set the aws user accessing the role, add the additional args in the completion()/embedding() function - -```python -from litellm import completion - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=messages, - max_tokens=10, - temperature=0.1, - aws_region_name=aws_region_name, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_role_name=aws_role_name, - aws_session_name="my-test-session", - ) -``` - - - - -```yaml -model_list: - - model_name: bedrock/* - litellm_params: - model: bedrock/* - aws_role_name: arn:aws:iam::888602223428:role/iam_local_role # AWS RoleArn - aws_session_name: "bedrock-session" # AWS RoleSessionName - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # [OPTIONAL - not required if using role] - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # [OPTIONAL - not required if using role] -``` - - - - - - - -### Passing an external BedrockRuntime.Client as a parameter - Completion() - -:::warning - -This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues) - - -Experimental - 2024-Jun-23: - `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` will be extracted from boto3.client and be passed into the httpx client - -::: - -Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth. - -Create a client from session credentials: -```python -import boto3 -from litellm import completion - -bedrock = boto3.client( - service_name="bedrock-runtime", - region_name="us-east-1", - aws_access_key_id="", - aws_secret_access_key="", - aws_session_token="", -) - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_bedrock_client=bedrock, -) -``` - -Create a client from AWS profile in `~/.aws/config`: -```python -import boto3 -from litellm import completion - -dev_session = boto3.Session(profile_name="dev-profile") -bedrock = dev_session.client( - service_name="bedrock-runtime", - region_name="us-east-1", -) - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_bedrock_client=bedrock, -) -``` -## Calling via Internal Proxy - -Use the `bedrock/converse_like/model` endpoint to call bedrock converse model via your internal proxy. - - - - -```python -from litellm import completion - -response = completion( - model="bedrock/converse_like/some-model", - messages=[{"role": "user", "content": "What's AWS?"}], - api_key="sk-1234", - api_base="https://some-api-url/models", - extra_headers={"test": "hello world"}, -) -``` - - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: anthropic-claude - litellm_params: - model: bedrock/converse_like/some-model - api_base: https://some-api-url/models -``` - -2. Start proxy server - -```bash -litellm --config config.yaml - -# RUNNING on http://0.0.0.0:4000 -``` - -3. Test it! - -```bash -curl -X POST 'http://0.0.0.0:4000/chat/completions' \ --H 'Content-Type: application/json' \ --H 'Authorization: Bearer sk-1234' \ --d '{ - "model": "anthropic-claude", - "messages": [ - { - "role": "system", - "content": "You are a helpful math tutor. Guide the user through the solution step by step." - }, - { "content": "Hello, how are you?", "role": "user" } - ] -}' -``` - - - - -**Expected Output URL** - -```bash -https://some-api-url/models -``` - ## Bedrock Imported Models (Deepseek, Deepseek R1) ### Deepseek R1 @@ -1958,3 +1657,377 @@ curl http://0.0.0.0:4000/rerank \ +## Bedrock Application Inference Profile + +Use Bedrock Application Inference Profile to track costs for projects on AWS. + +You can either pass it in the model name - `model="bedrock/arn:...` or as a separate `model_id="arn:..` param. + +### Set via `model_id` + + + + +```python +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +response = completion( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + messages=[{"role": "user", "content": "Hello, how are you?"}], + model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", +) + +print(response) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: anthropic-claude-3-5-sonnet + litellm_params: + model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + # You have to set the ARN application inference profile in the model_id parameter + model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0 +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer $LITELLM_API_KEY' \ +-d '{ + "model": "anthropic-claude-3-5-sonnet", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "List 5 important events in the XIX century" + } + ] + } + ] +}' +``` + + + + +## Boto3 - Authentication + +### Passing credentials as parameters - Completion() +Pass AWS credentials as parameters to litellm.completion +```python +import os +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_access_key_id="", + aws_secret_access_key="", + aws_region_name="", +) +``` + +### Passing extra headers + Custom API Endpoints + +This can be used to override existing headers (e.g. `Authorization`) when calling custom api endpoints + + + + +```python +import os +import litellm +from litellm import completion + +litellm.set_verbose = True # 👈 SEE RAW REQUEST + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_access_key_id="", + aws_secret_access_key="", + aws_region_name="", + aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com", + extra_headers={"key": "value"} +) +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: bedrock-model + litellm_params: + model: bedrock/anthropic.claude-instant-v1 + aws_access_key_id: "", + aws_secret_access_key: "", + aws_region_name: "", + aws_bedrock_runtime_endpoint: "https://my-fake-endpoint.com", + extra_headers: {"key": "value"} +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml --detailed_debug +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "bedrock-model", + "messages": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step." + }, + { + "role": "user", + "content": "how can I solve 8x + 7 = -23" + } + ] +}' +``` + + + + +### SSO Login (AWS Profile) +- Set `AWS_PROFILE` environment variable +- Make bedrock completion call +```python +import os +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + +or pass `aws_profile_name`: + +```python +import os +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_profile_name="dev-profile", +) +``` + +### STS (Role-based Auth) + +- Set `aws_role_name` and `aws_session_name` + + +| LiteLLM Parameter | Boto3 Parameter | Description | Boto3 Documentation | +|------------------|-----------------|-------------|-------------------| +| `aws_access_key_id` | `aws_access_key_id` | AWS access key associated with an IAM user or role | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) | +| `aws_secret_access_key` | `aws_secret_access_key` | AWS secret key associated with the access key | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) | +| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) | +| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) | + + + +Make the bedrock completion call + + + + +```python +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=messages, + max_tokens=10, + temperature=0.1, + aws_role_name=aws_role_name, + aws_session_name="my-test-session", + ) +``` + +If you also need to dynamically set the aws user accessing the role, add the additional args in the completion()/embedding() function + +```python +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=messages, + max_tokens=10, + temperature=0.1, + aws_region_name=aws_region_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_role_name=aws_role_name, + aws_session_name="my-test-session", + ) +``` + + + + +```yaml +model_list: + - model_name: bedrock/* + litellm_params: + model: bedrock/* + aws_role_name: arn:aws:iam::888602223428:role/iam_local_role # AWS RoleArn + aws_session_name: "bedrock-session" # AWS RoleSessionName + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # [OPTIONAL - not required if using role] + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # [OPTIONAL - not required if using role] +``` + + + + + + + +### Passing an external BedrockRuntime.Client as a parameter - Completion() + +:::warning + +This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues) + + +Experimental - 2024-Jun-23: + `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` will be extracted from boto3.client and be passed into the httpx client + +::: + +Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth. + +Create a client from session credentials: +```python +import boto3 +from litellm import completion + +bedrock = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id="", + aws_secret_access_key="", + aws_session_token="", +) + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_bedrock_client=bedrock, +) +``` + +Create a client from AWS profile in `~/.aws/config`: +```python +import boto3 +from litellm import completion + +dev_session = boto3.Session(profile_name="dev-profile") +bedrock = dev_session.client( + service_name="bedrock-runtime", + region_name="us-east-1", +) + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_bedrock_client=bedrock, +) +``` +## Calling via Internal Proxy (not bedrock url compatible) + +Use the `bedrock/converse_like/model` endpoint to call bedrock converse model via your internal proxy. + + + + +```python +from litellm import completion + +response = completion( + model="bedrock/converse_like/some-model", + messages=[{"role": "user", "content": "What's AWS?"}], + api_key="sk-1234", + api_base="https://some-api-url/models", + extra_headers={"test": "hello world"}, +) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: anthropic-claude + litellm_params: + model: bedrock/converse_like/some-model + api_base: https://some-api-url/models +``` + +2. Start proxy server + +```bash +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "anthropic-claude", + "messages": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step." + }, + { "content": "Hello, how are you?", "role": "user" } + ] +}' +``` + + + + +**Expected Output URL** + +```bash +https://some-api-url/models +``` diff --git a/docs/my-website/docs/proxy/access_control.md b/docs/my-website/docs/proxy/access_control.md index 3d335380f4..69b8a3ff6d 100644 --- a/docs/my-website/docs/proxy/access_control.md +++ b/docs/my-website/docs/proxy/access_control.md @@ -10,17 +10,13 @@ Role-based access control (RBAC) is based on Organizations, Teams and Internal U ## Roles -**Admin Roles** - - `proxy_admin`: admin over the platform - - `proxy_admin_viewer`: can login, view all keys, view all spend. **Cannot** create keys/delete keys/add new users - -**Organization Roles** - - `org_admin`: admin over the organization. Can create teams and users within their organization - -**Internal User Roles** - - `internal_user`: can login, view/create/delete their own keys, view their spend. **Cannot** add new users. - - `internal_user_viewer`: can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users. - +| Role Type | Role Name | Permissions | +|-----------|-----------|-------------| +| **Admin** | `proxy_admin` | Admin over the platform | +| | `proxy_admin_viewer` | Can login, view all keys, view all spend. **Cannot** create keys/delete keys/add new users | +| **Organization** | `org_admin` | Admin over the organization. Can create teams and users within their organization | +| **Internal User** | `internal_user` | Can login, view/create/delete their own keys, view their spend. **Cannot** add new users | +| | `internal_user_viewer` | Can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users | ## Onboarding Organizations diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index c6d280cb82..c562c7fb71 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -102,7 +102,19 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \ -## Advanced - Set Accepted JWT Scope Names +## Advanced + +### Multiple OIDC providers + +Use this if you want LiteLLM to validate your JWT against multiple OIDC providers (e.g. Google Cloud, GitHub Auth) + +Set `JWT_PUBLIC_KEY_URL` in your environment to a comma-separated list of URLs for your OIDC providers. + +```bash +export JWT_PUBLIC_KEY_URL="https://demo.duendesoftware.com/.well-known/openid-configuration/jwks,https://accounts.google.com/.well-known/openid-configuration/jwks" +``` + +### Set Accepted JWT Scope Names Change the string in JWT 'scopes', that litellm evaluates to see if a user has admin access. @@ -114,7 +126,7 @@ general_settings: admin_jwt_scope: "litellm-proxy-admin" ``` -## Tracking End-Users / Internal Users / Team / Org +### Tracking End-Users / Internal Users / Team / Org Set the field in the jwt token, which corresponds to a litellm user / team / org. @@ -156,7 +168,7 @@ scope: ["litellm-proxy-admin",...] scope: "litellm-proxy-admin ..." ``` -## Control model access with Teams +### Control model access with Teams 1. Specify the JWT field that contains the team ids, that the user belongs to. @@ -207,11 +219,11 @@ OIDC Auth for API: [**See Walkthrough**](https://www.loom.com/share/00fe2deab59a - If all checks pass, allow the request -## Advanced - Custom Validate +### Custom JWT Validate Validate a JWT Token using custom logic, if you need an extra way to verify if tokens are valid for LiteLLM Proxy. -### 1. Setup custom validate function +#### 1. Setup custom validate function ```python from typing import Literal @@ -230,7 +242,7 @@ def my_custom_validate(token: str) -> Literal[True]: return True ``` -### 2. Setup config.yaml +#### 2. Setup config.yaml ```yaml general_settings: @@ -243,7 +255,7 @@ general_settings: custom_validate: custom_validate.my_custom_validate # 👈 custom validate function ``` -### 3. Test the flow +#### 3. Test the flow **Expected JWT** @@ -265,7 +277,7 @@ general_settings: -## Advanced - Allowed Routes +### Allowed Routes Configure which routes a JWT can access via the config. @@ -297,7 +309,7 @@ general_settings: team_allowed_routes: ["/v1/chat/completions"] # 👈 Set accepted routes ``` -## Advanced - Caching Public Keys +### Caching Public Keys Control how long public keys are cached for (in seconds). @@ -311,7 +323,7 @@ general_settings: public_key_ttl: 600 # 👈 KEY CHANGE ``` -## Advanced - Custom JWT Field +### Custom JWT Field Set a custom field in which the team_id exists. By default, the 'client_id' field is checked. @@ -323,14 +335,7 @@ general_settings: team_id_jwt_field: "client_id" # 👈 KEY CHANGE ``` -## All Params - -[**See Code**](https://github.com/BerriAI/litellm/blob/b204f0c01c703317d812a1553363ab0cb989d5b6/litellm/proxy/_types.py#L95) - - - - -## Advanced - Block Teams +### Block Teams To block all requests for a certain team id, use `/team/block` @@ -357,7 +362,7 @@ curl --location 'http://0.0.0.0:4000/team/unblock' \ ``` -## Advanced - Upsert Users + Allowed Email Domains +### Upsert Users + Allowed Email Domains Allow users who belong to a specific email domain, automatic access to the proxy. @@ -494,4 +499,10 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ } ] }' -``` \ No newline at end of file +``` + +## All JWT Params + +[**See Code**](https://github.com/BerriAI/litellm/blob/b204f0c01c703317d812a1553363ab0cb989d5b6/litellm/proxy/_types.py#L95) + + diff --git a/docs/my-website/docs/proxy/ui_credentials.md b/docs/my-website/docs/proxy/ui_credentials.md new file mode 100644 index 0000000000..ba9d1c4c66 --- /dev/null +++ b/docs/my-website/docs/proxy/ui_credentials.md @@ -0,0 +1,55 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Adding LLM Credentials + +You can add LLM provider credentials on the UI. Once you add credentials you can re-use them when adding new models + +## Add a credential + model + +### 1. Navigate to LLM Credentials page + +Go to Models -> LLM Credentials -> Add Credential + + + +### 2. Add credentials + +Select your LLM provider, enter your API Key and click "Add Credential" + +**Note: Credentials are based on the provider, if you select Vertex AI then you will see `Vertex Project`, `Vertex Location` and `Vertex Credentials` fields** + + + + +### 3. Use credentials when adding a model + +Go to Add Model -> Existing Credentials -> Select your credential in the dropdown + + + + +## Create a Credential from an existing model + +Use this if you have already created a model and want to store the model credentials for future use + +### 1. Select model to create a credential from + +Go to Models -> Select your model -> Credential -> Create Credential + + + +### 2. Use new credential when adding a model + +Go to Add Model -> Existing Credentials -> Select your credential in the dropdown + + + +## Frequently Asked Questions + + +How are credentials stored? +Credentials in the DB are encrypted/decrypted using `LITELLM_SALT_KEY`, if set. If not, then they are encrypted using `LITELLM_MASTER_KEY`. These keys should be kept secret and not shared with others. + + diff --git a/docs/my-website/docs/proxy/ui_logs.md b/docs/my-website/docs/proxy/ui_logs.md new file mode 100644 index 0000000000..a3c5237962 --- /dev/null +++ b/docs/my-website/docs/proxy/ui_logs.md @@ -0,0 +1,55 @@ + +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# UI Logs Page + +View Spend, Token Usage, Key, Team Name for Each Request to LiteLLM + + + + + +## Overview + +| Log Type | Tracked by Default | +|----------|-------------------| +| Success Logs | ✅ Yes | +| Error Logs | ✅ Yes | +| Request/Response Content Stored | ❌ No by Default, **opt in with `store_prompts_in_spend_logs`** | + + + +**By default LiteLLM does not track the request and response content.** + +## Tracking - Request / Response Content in Logs Page + +If you want to view request and response content on LiteLLM Logs, you need to opt in with this setting + +```yaml +general_settings: + store_prompts_in_spend_logs: true +``` + + + + +## Stop storing Error Logs in DB + +If you do not want to store error logs in DB, you can opt out with this setting + +```yaml +general_settings: + disable_error_logs: True # Only disable writing error logs to DB, regular spend logs will still be written unless `disable_spend_logs: True` +``` + +## Stop storing Spend Logs in DB + +If you do not want to store spend logs in DB, you can opt out with this setting + +```yaml +general_settings: + disable_spend_logs: True # Disable writing spend logs to DB +``` + diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index f0bcd30a9b..1cce3f0570 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -14,6 +14,9 @@ Supported Providers: - Anthropic API (`anthropic/`) - Bedrock (Anthropic + Deepseek) (`bedrock/`) - Vertex AI (Anthropic) (`vertexai/`) +- OpenRouter (`openrouter/`) + +LiteLLM will standardize the `reasoning_content` in the response and `thinking_blocks` in the assistant message. ```python "message": { diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index 7676164259..9f0ff7059c 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -96,7 +96,7 @@ litellm --config /path/to/config.yaml ``` -### Using K/V pairs in 1 AWS Secret +#### Using K/V pairs in 1 AWS Secret You can read multiple keys from a single AWS Secret using the `primary_secret_name` parameter: diff --git a/docs/my-website/img/release_notes/credentials.jpg b/docs/my-website/img/release_notes/credentials.jpg new file mode 100644 index 0000000000..1f11c67f05 Binary files /dev/null and b/docs/my-website/img/release_notes/credentials.jpg differ diff --git a/docs/my-website/img/release_notes/litellm_test_connection.gif b/docs/my-website/img/release_notes/litellm_test_connection.gif new file mode 100644 index 0000000000..2c8ea45ab4 Binary files /dev/null and b/docs/my-website/img/release_notes/litellm_test_connection.gif differ diff --git a/docs/my-website/img/release_notes/responses_api.png b/docs/my-website/img/release_notes/responses_api.png new file mode 100644 index 0000000000..045d86825d Binary files /dev/null and b/docs/my-website/img/release_notes/responses_api.png differ diff --git a/docs/my-website/img/ui_add_cred_2.png b/docs/my-website/img/ui_add_cred_2.png new file mode 100644 index 0000000000..199a15e178 Binary files /dev/null and b/docs/my-website/img/ui_add_cred_2.png differ diff --git a/docs/my-website/img/ui_cred_3.png b/docs/my-website/img/ui_cred_3.png new file mode 100644 index 0000000000..67a614d51b Binary files /dev/null and b/docs/my-website/img/ui_cred_3.png differ diff --git a/docs/my-website/img/ui_cred_4.png b/docs/my-website/img/ui_cred_4.png new file mode 100644 index 0000000000..84e70e0347 Binary files /dev/null and b/docs/my-website/img/ui_cred_4.png differ diff --git a/docs/my-website/img/ui_cred_add.png b/docs/my-website/img/ui_cred_add.png new file mode 100644 index 0000000000..7b03270b3c Binary files /dev/null and b/docs/my-website/img/ui_cred_add.png differ diff --git a/docs/my-website/img/ui_request_logs.png b/docs/my-website/img/ui_request_logs.png new file mode 100644 index 0000000000..912123522b Binary files /dev/null and b/docs/my-website/img/ui_request_logs.png differ diff --git a/docs/my-website/img/ui_request_logs_content.png b/docs/my-website/img/ui_request_logs_content.png new file mode 100644 index 0000000000..74673b5553 Binary files /dev/null and b/docs/my-website/img/ui_request_logs_content.png differ diff --git a/docs/my-website/img/use_model_cred.png b/docs/my-website/img/use_model_cred.png new file mode 100644 index 0000000000..35d4248555 Binary files /dev/null and b/docs/my-website/img/use_model_cred.png differ diff --git a/docs/my-website/release_notes/v1.63.11-stable/index.md b/docs/my-website/release_notes/v1.63.11-stable/index.md new file mode 100644 index 0000000000..1a9583c8c3 --- /dev/null +++ b/docs/my-website/release_notes/v1.63.11-stable/index.md @@ -0,0 +1,176 @@ +--- +title: v1.63.11-stable +slug: v1.63.11-stable +date: 2025-03-15T10:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://media.licdn.com/dms/image/v2/D4D03AQGrlsJ3aqpHmQ/profile-displayphoto-shrink_400_400/B4DZSAzgP7HYAg-/0/1737327772964?e=1743638400&v=beta&t=39KOXMUFedvukiWWVPHf3qI45fuQD7lNglICwN31DrI + - name: Ishaan Jaffer + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +tags: [credential management, thinking content, responses api, snowflake] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +These are the changes since `v1.63.2-stable`. + +This release is primarily focused on: +- [Beta] Responses API Support +- Snowflake Cortex Support, Amazon Nova Image Generation +- UI - Credential Management, re-use credentials when adding new models +- UI - Test Connection to LLM Provider before adding a model + +:::info + +This release will be live on 03/16/2025 + +::: + + + +## Docker Run LiteLLM Proxy + +``` +docker run +-e STORE_MODEL_IN_DB=True +-p 4000:4000 +ghcr.io/berriai/litellm:main-v1.63.11-stable +``` + +## Demo Instance + +Here's a Demo Instance to test changes: +- Instance: https://demo.litellm.ai/ +- Login Credentials: + - Username: admin + - Password: sk-1234 + + + +## New Models / Updated Models + +- Image Generation support for Amazon Nova Canvas [Getting Started](https://docs.litellm.ai/docs/providers/bedrock#image-generation) +- Add pricing for Jamba new models [PR](https://github.com/BerriAI/litellm/pull/9032/files) +- Add pricing for Amazon EU models [PR](https://github.com/BerriAI/litellm/pull/9056/files) +- Add Bedrock Deepseek R1 model pricing [PR](https://github.com/BerriAI/litellm/pull/9108/files) +- Update Gemini pricing: Gemma 3, Flash 2 thinking update, LearnLM [PR](https://github.com/BerriAI/litellm/pull/9190/files) +- Mark Cohere Embedding 3 models as Multimodal [PR](https://github.com/BerriAI/litellm/pull/9176/commits/c9a576ce4221fc6e50dc47cdf64ab62736c9da41) +- Add Azure Data Zone pricing [PR](https://github.com/BerriAI/litellm/pull/9185/files#diff-19ad91c53996e178c1921cbacadf6f3bae20cfe062bd03ee6bfffb72f847ee37) + - LiteLLM Tracks cost for `azure/eu` and `azure/us` models + + + +## LLM Translation + + + +1. **New Endpoints** +- [Beta] POST `/responses` API. [Getting Started](https://docs.litellm.ai/docs/response_api) + +2. **New LLM Providers** +- Snowflake Cortex [Getting Started](https://docs.litellm.ai/docs/providers/snowflake) + +3. **New LLM Features** + +- Support OpenRouter `reasoning_content` on streaming [Getting Started](https://docs.litellm.ai/docs/reasoning_content) + +4. **Bug Fixes** + +- OpenAI: Return `code`, `param` and `type` on bad request error [More information on litellm exceptions](https://docs.litellm.ai/docs/exception_mapping) +- Bedrock: Fix converse chunk parsing to only return empty dict on tool use [PR](https://github.com/BerriAI/litellm/pull/9166) +- Bedrock: Support extra_headers [PR](https://github.com/BerriAI/litellm/pull/9113) +- Azure: Fix Function Calling Bug & Update Default API Version to `2025-02-01-preview` [PR](https://github.com/BerriAI/litellm/pull/9191) +- Azure: Fix AI services URL [PR](https://github.com/BerriAI/litellm/pull/9185) +- Vertex AI: Handle HTTP 201 status code in response [PR](https://github.com/BerriAI/litellm/pull/9193) +- Perplexity: Fix incorrect streaming response [PR](https://github.com/BerriAI/litellm/pull/9081) +- Triton: Fix streaming completions bug [PR](https://github.com/BerriAI/litellm/pull/8386) +- Deepgram: Support bytes.IO when handling audio files for transcription [PR](https://github.com/BerriAI/litellm/pull/9071) +- Ollama: Fix "system" role has become unacceptable [PR](https://github.com/BerriAI/litellm/pull/9261) +- All Providers (Streaming): Fix String `data:` stripped from entire content in streamed responses [PR](https://github.com/BerriAI/litellm/pull/9070) + + + +## Spend Tracking Improvements + +1. Support Bedrock converse cache token tracking [Getting Started](https://docs.litellm.ai/docs/completion/prompt_caching) +2. Cost Tracking for Responses API [Getting Started](https://docs.litellm.ai/docs/response_api) +3. Fix Azure Whisper cost tracking [Getting Started](https://docs.litellm.ai/docs/audio_transcription) + + +## UI + +### Re-Use Credentials on UI + +You can now onboard LLM provider credentials on LiteLLM UI. Once these credentials are added you can re-use them when adding new models [Getting Started](https://docs.litellm.ai/docs/proxy/ui_credentials) + + + + +### Test Connections before adding models + +Before adding a model you can test the connection to the LLM provider to verify you have setup your API Base + API Key correctly + + + +### General UI Improvements +1. Add Models Page + - Allow adding Cerebras, Sambanova, Perplexity, Fireworks, Openrouter, TogetherAI Models, Text-Completion OpenAI on Admin UI + - Allow adding EU OpenAI models + - Fix: Instantly show edit + deletes to models +2. Keys Page + - Fix: Instantly show newly created keys on Admin UI (don't require refresh) + - Fix: Allow clicking into Top Keys when showing users Top API Key + - Fix: Allow Filter Keys by Team Alias, Key Alias and Org + - UI Improvements: Show 100 Keys Per Page, Use full height, increase width of key alias +3. Users Page + - Fix: Show correct count of internal user keys on Users Page + - Fix: Metadata not updating in Team UI +4. Logs Page + - UI Improvements: Keep expanded log in focus on LiteLLM UI + - UI Improvements: Minor improvements to logs page + - Fix: Allow internal user to query their own logs + - Allow switching off storing Error Logs in DB [Getting Started](https://docs.litellm.ai/docs/proxy/ui_logs) +5. Sign In/Sign Out + - Fix: Correctly use `PROXY_LOGOUT_URL` when set [Getting Started](https://docs.litellm.ai/docs/proxy/self_serve#setting-custom-logout-urls) + + +## Security + +1. Support for Rotating Master Keys [Getting Started](https://docs.litellm.ai/docs/proxy/master_key_rotations) +2. Fix: Internal User Viewer Permissions, don't allow `internal_user_viewer` role to see `Test Key Page` or `Create Key Button` [More information on role based access controls](https://docs.litellm.ai/docs/proxy/access_control) +3. Emit audit logs on All user + model Create/Update/Delete endpoints [Getting Started](https://docs.litellm.ai/docs/proxy/multiple_admins) +4. JWT + - Support multiple JWT OIDC providers [Getting Started](https://docs.litellm.ai/docs/proxy/token_auth) + - Fix JWT access with Groups not working when team is assigned All Proxy Models access +5. Using K/V pairs in 1 AWS Secret [Getting Started](https://docs.litellm.ai/docs/secret#using-kv-pairs-in-1-aws-secret) + + +## Logging Integrations + +1. Prometheus: Track Azure LLM API latency metric [Getting Started](https://docs.litellm.ai/docs/proxy/prometheus#request-latency-metrics) +2. Athina: Added tags, user_feedback and model_options to additional_keys which can be sent to Athina [Getting Started](https://docs.litellm.ai/docs/observability/athina_integration) + + +## Performance / Reliability improvements + +1. Redis + litellm router - Fix Redis cluster mode for litellm router [PR](https://github.com/BerriAI/litellm/pull/9010) + + +## General Improvements + +1. OpenWebUI Integration - display `thinking` tokens +- Guide on getting started with LiteLLM x OpenWebUI. [Getting Started](https://docs.litellm.ai/docs/tutorials/openweb_ui) +- Display `thinking` tokens on OpenWebUI (Bedrock, Anthropic, Deepseek) [Getting Started](https://docs.litellm.ai/docs/tutorials/openweb_ui#render-thinking-content-on-openweb-ui) + + + + +## Complete Git Diff + +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.2-stable...v1.63.11-stable) \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 385983fd33..47d69e5d3f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -101,7 +101,9 @@ const sidebars = { "proxy/admin_ui_sso", "proxy/self_serve", "proxy/public_teams", - "proxy/custom_sso" + "proxy/custom_sso", + "proxy/ui_credentials", + "proxy/ui_logs" ], }, { diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 86b47675d4..5482d80687 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -279,16 +279,30 @@ class BaseAWSLLM: return None def _get_aws_region_name( - self, optional_params: dict, model: Optional[str] = None + self, + optional_params: dict, + model: Optional[str] = None, + model_id: Optional[str] = None, ) -> str: """ - Get the AWS region name from the environment variables + Get the AWS region name from the environment variables. + + Parameters: + optional_params (dict): Optional parameters for the model call + model (str): The model name + model_id (str): The model ID. This is the ARN of the model, if passed in as a separate param. + + Returns: + str: The AWS region name """ aws_region_name = optional_params.get("aws_region_name", None) ### SET REGION NAME ### if aws_region_name is None: # check model arn # - aws_region_name = self._get_aws_region_from_model_arn(model) + if model_id is not None: + aws_region_name = self._get_aws_region_from_model_arn(model_id) + else: + aws_region_name = self._get_aws_region_from_model_arn(model) # check env # litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index d45ab40c4b..a4230177b5 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -13,7 +13,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.types.utils import ModelResponse -from litellm.utils import CustomStreamWrapper, get_secret +from litellm.utils import CustomStreamWrapper from ..base_aws_llm import BaseAWSLLM, Credentials from ..common_utils import BedrockError @@ -268,23 +268,29 @@ class BedrockConverseLLM(BaseAWSLLM): ## SETUP ## stream = optional_params.pop("stream", None) - modelId = optional_params.pop("model_id", None) + unencoded_model_id = optional_params.pop("model_id", None) fake_stream = optional_params.pop("fake_stream", False) json_mode = optional_params.get("json_mode", False) - if modelId is not None: - modelId = self.encode_model_id(model_id=modelId) + if unencoded_model_id is not None: + modelId = self.encode_model_id(model_id=unencoded_model_id) else: modelId = self.encode_model_id(model_id=model) if stream is True and "ai21" in modelId: fake_stream = True + ### SET REGION NAME ### + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, + model=model, + model_id=unencoded_model_id, + ) + ## CREDENTIALS ## # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) aws_access_key_id = optional_params.pop("aws_access_key_id", None) aws_session_token = optional_params.pop("aws_session_token", None) - aws_region_name = optional_params.pop("aws_region_name", None) aws_role_name = optional_params.pop("aws_role_name", None) aws_session_name = optional_params.pop("aws_session_name", None) aws_profile_name = optional_params.pop("aws_profile_name", None) @@ -293,25 +299,7 @@ class BedrockConverseLLM(BaseAWSLLM): ) # https://bedrock-runtime.{region_name}.amazonaws.com aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None) - - ### SET REGION NAME ### - if aws_region_name is None: - # check env # - litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) - - if litellm_aws_region_name is not None and isinstance( - litellm_aws_region_name, str - ): - aws_region_name = litellm_aws_region_name - - standard_aws_region_name = get_secret("AWS_REGION", None) - if standard_aws_region_name is not None and isinstance( - standard_aws_region_name, str - ): - aws_region_name = standard_aws_region_name - - if aws_region_name is None: - aws_region_name = "us-west-2" + optional_params.pop("aws_region_name", None) litellm_params["aws_region_name"] = ( aws_region_name # [DO NOT DELETE] important for async calls diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 7228d22d03..1694033a09 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2950,15 +2950,39 @@ async def test_bedrock_stream_thinking_content_openwebui(): ), "There should be non-empty content after thinking tags" -def test_bedrock_streaming(): - resp = completion( - model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", - messages=[{"role": "user", "content": "Hello who is this?"}], - stream=True, - max_tokens=1080, - stream_options={"include_usage": True}, - ) +def test_bedrock_application_inference_profile(): + from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler - for chunk in resp: - if hasattr(chunk, "usage"): - print(f"received chunk usage: {chunk.usage}") + client = HTTPHandler() + client2 = HTTPHandler() + + with patch.object(client, "post") as mock_post, patch.object( + client2, "post" + ) as mock_post2: + try: + resp = completion( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + messages=[{"role": "user", "content": "Hello, how are you?"}], + model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", + client=client, + ) + except Exception as e: + print(e) + + try: + resp = completion( + model="bedrock/converse/arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", + messages=[{"role": "user", "content": "Hello, how are you?"}], + client=client2, + ) + except Exception as e: + print(e) + + mock_post.assert_called_once() + mock_post2.assert_called_once() + print(mock_post.call_args.kwargs) + json_data = mock_post.call_args.kwargs["data"] + assert mock_post.call_args.kwargs["url"].startswith( + "https://bedrock-runtime.eu-central-1.amazonaws.com/" + ) + assert mock_post2.call_args.kwargs["url"] == mock_post.call_args.kwargs["url"]