mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-10 21:04:46 +00:00
Merge branch 'main' of https://github.com/BerriAI/litellm into litellm_ftr_bedrock_aws_session_token
This commit is contained in:
Brian Schultheiss
@@ -48,7 +48,8 @@ jobs:
|
||||
pip install opentelemetry-sdk==1.25.0
|
||||
pip install opentelemetry-exporter-otlp==1.25.0
|
||||
pip install openai
|
||||
pip install prisma
|
||||
pip install prisma
|
||||
pip install "detect_secrets==1.5.0"
|
||||
pip install "httpx==0.24.1"
|
||||
pip install fastapi
|
||||
pip install "gunicorn==21.2.0"
|
||||
|
||||
@@ -61,3 +61,4 @@ litellm/proxy/_experimental/out/model_hub/index.html
|
||||
litellm/proxy/_experimental/out/onboarding/index.html
|
||||
litellm/tests/log.txt
|
||||
litellm/tests/langfuse.log
|
||||
litellm/tests/langfuse.log
|
||||
|
||||
@@ -47,7 +47,8 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
|
||||
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
|
||||
|
||||
> [!IMPORTANT]
|
||||
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
|
||||
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
|
||||
> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
|
||||
|
||||
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
|
||||
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
|
||||
|
||||
@@ -502,10 +502,10 @@ response = completion(model="gpt-3.5-turbo-0613", messages=messages, functions=f
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Function calling for Non-OpenAI LLMs
|
||||
## Function calling for Models w/out function-calling support
|
||||
|
||||
### Adding Function to prompt
|
||||
For Non OpenAI LLMs LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True`
|
||||
For Models/providers without function calling support, LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True`
|
||||
|
||||
#### Usage
|
||||
```python
|
||||
|
||||
@@ -31,9 +31,15 @@ response = completion(
|
||||
)
|
||||
```
|
||||
|
||||
## Fallbacks
|
||||
## Fallbacks (SDK)
|
||||
|
||||
### Context Window Fallbacks
|
||||
:::info
|
||||
|
||||
[See how to do on PROXY](../proxy/reliability.md)
|
||||
|
||||
:::
|
||||
|
||||
### Context Window Fallbacks (SDK)
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
@@ -43,7 +49,7 @@ messages = [{"content": "how does a court case get to the Supreme Court?" * 500,
|
||||
completion(model="gpt-3.5-turbo", messages=messages, context_window_fallback_dict=ctx_window_fallback_dict)
|
||||
```
|
||||
|
||||
### Fallbacks - Switch Models/API Keys/API Bases
|
||||
### Fallbacks - Switch Models/API Keys/API Bases (SDK)
|
||||
|
||||
LLM APIs can be unstable, completion() with fallbacks ensures you'll always get a response from your calls
|
||||
|
||||
@@ -69,7 +75,7 @@ response = completion(model="azure/gpt-4", messages=messages, api_key=api_key,
|
||||
|
||||
[Check out this section for implementation details](#fallbacks-1)
|
||||
|
||||
## Implementation Details
|
||||
## Implementation Details (SDK)
|
||||
|
||||
### Fallbacks
|
||||
#### Output from calls
|
||||
|
||||
@@ -12,7 +12,9 @@ This covers:
|
||||
- ✅ [**Secure UI access with Single Sign-On**](../docs/proxy/ui.md#setup-ssoauth-for-ui)
|
||||
- ✅ [**Audit Logs with retention policy**](../docs/proxy/enterprise.md#audit-logs)
|
||||
- ✅ [**JWT-Auth**](../docs/proxy/token_auth.md)
|
||||
- ✅ [**Prompt Injection Detection**](#prompt-injection-detection-lakeraai)
|
||||
- ✅ [**Control available public, private routes**](../docs/proxy/enterprise.md#control-available-public-private-routes)
|
||||
- ✅ [**Guardrails, Content Moderation, PII Masking, Secret/API Key Masking**](../docs/proxy/enterprise.md#prompt-injection-detection---lakeraai)
|
||||
- ✅ [**Prompt Injection Detection**](../docs/proxy/enterprise.md#prompt-injection-detection---lakeraai)
|
||||
- ✅ [**Invite Team Members to access `/spend` Routes**](../docs/proxy/cost_tracking#allowing-non-proxy-admins-to-access-spend-endpoints)
|
||||
- ✅ **Feature Prioritization**
|
||||
- ✅ **Custom Integrations**
|
||||
|
||||
@@ -1,13 +1,8 @@
|
||||
# Telemetry
|
||||
|
||||
LiteLLM contains a telemetry feature that tells us what models are used, and what errors are hit.
|
||||
There is no Telemetry on LiteLLM - no data is stored by us
|
||||
|
||||
## What is logged?
|
||||
|
||||
Only the model name and exception raised is logged.
|
||||
NOTHING - no data is sent to LiteLLM Servers
|
||||
|
||||
## Why?
|
||||
We use this information to help us understand how LiteLLM is used, and improve stability.
|
||||
|
||||
## Opting out
|
||||
If you prefer to opt out of telemetry, you can do this by setting `litellm.telemetry = False`.
|
||||
@@ -0,0 +1,103 @@
|
||||
# Nvidia NIM
|
||||
https://docs.api.nvidia.com/nim/reference/
|
||||
|
||||
:::tip
|
||||
|
||||
**We support ALL Nvidia NIM models, just set `model=nvidia_nim/<any-model-on-nvidia_nim>` as a prefix when sending litellm requests**
|
||||
|
||||
:::
|
||||
|
||||
## API Key
|
||||
```python
|
||||
# env variable
|
||||
os.environ['NVIDIA_NIM_API_KEY']
|
||||
```
|
||||
|
||||
## Sample Usage
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['NVIDIA_NIM_API_KEY'] = ""
|
||||
response = completion(
|
||||
model="nvidia_nim/meta/llama3-70b-instruct",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Boston today in Fahrenheit?",
|
||||
}
|
||||
],
|
||||
temperature=0.2, # optional
|
||||
top_p=0.9, # optional
|
||||
frequency_penalty=0.1, # optional
|
||||
presence_penalty=0.1, # optional
|
||||
max_tokens=10, # optional
|
||||
stop=["\n\n"], # optional
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Sample Usage - Streaming
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['NVIDIA_NIM_API_KEY'] = ""
|
||||
response = completion(
|
||||
model="nvidia_nim/meta/llama3-70b-instruct",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Boston today in Fahrenheit?",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
temperature=0.2, # optional
|
||||
top_p=0.9, # optional
|
||||
frequency_penalty=0.1, # optional
|
||||
presence_penalty=0.1, # optional
|
||||
max_tokens=10, # optional
|
||||
stop=["\n\n"], # optional
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
|
||||
## Supported Models - 💥 ALL Nvidia NIM Models Supported!
|
||||
We support ALL `nvidia_nim` models, just set `nvidia_nim/` as a prefix when sending completion requests
|
||||
|
||||
| Model Name | Function Call |
|
||||
|------------|---------------|
|
||||
| nvidia/nemotron-4-340b-reward | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-reward", messages)` |
|
||||
| 01-ai/yi-large | `completion(model="nvidia_nim/01-ai/yi-large", messages)` |
|
||||
| aisingapore/sea-lion-7b-instruct | `completion(model="nvidia_nim/aisingapore/sea-lion-7b-instruct", messages)` |
|
||||
| databricks/dbrx-instruct | `completion(model="nvidia_nim/databricks/dbrx-instruct", messages)` |
|
||||
| google/gemma-7b | `completion(model="nvidia_nim/google/gemma-7b", messages)` |
|
||||
| google/gemma-2b | `completion(model="nvidia_nim/google/gemma-2b", messages)` |
|
||||
| google/codegemma-1.1-7b | `completion(model="nvidia_nim/google/codegemma-1.1-7b", messages)` |
|
||||
| google/codegemma-7b | `completion(model="nvidia_nim/google/codegemma-7b", messages)` |
|
||||
| google/recurrentgemma-2b | `completion(model="nvidia_nim/google/recurrentgemma-2b", messages)` |
|
||||
| ibm/granite-34b-code-instruct | `completion(model="nvidia_nim/ibm/granite-34b-code-instruct", messages)` |
|
||||
| ibm/granite-8b-code-instruct | `completion(model="nvidia_nim/ibm/granite-8b-code-instruct", messages)` |
|
||||
| mediatek/breeze-7b-instruct | `completion(model="nvidia_nim/mediatek/breeze-7b-instruct", messages)` |
|
||||
| meta/codellama-70b | `completion(model="nvidia_nim/meta/codellama-70b", messages)` |
|
||||
| meta/llama2-70b | `completion(model="nvidia_nim/meta/llama2-70b", messages)` |
|
||||
| meta/llama3-8b | `completion(model="nvidia_nim/meta/llama3-8b", messages)` |
|
||||
| meta/llama3-70b | `completion(model="nvidia_nim/meta/llama3-70b", messages)` |
|
||||
| microsoft/phi-3-medium-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-medium-4k-instruct", messages)` |
|
||||
| microsoft/phi-3-mini-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-128k-instruct", messages)` |
|
||||
| microsoft/phi-3-mini-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-4k-instruct", messages)` |
|
||||
| microsoft/phi-3-small-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-128k-instruct", messages)` |
|
||||
| microsoft/phi-3-small-8k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-8k-instruct", messages)` |
|
||||
| mistralai/codestral-22b-instruct-v0.1 | `completion(model="nvidia_nim/mistralai/codestral-22b-instruct-v0.1", messages)` |
|
||||
| mistralai/mistral-7b-instruct | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct", messages)` |
|
||||
| mistralai/mistral-7b-instruct-v0.3 | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct-v0.3", messages)` |
|
||||
| mistralai/mixtral-8x7b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x7b-instruct", messages)` |
|
||||
| mistralai/mixtral-8x22b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x22b-instruct", messages)` |
|
||||
| mistralai/mistral-large | `completion(model="nvidia_nim/mistralai/mistral-large", messages)` |
|
||||
| nvidia/nemotron-4-340b-instruct | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-instruct", messages)` |
|
||||
| seallms/seallm-7b-v2.5 | `completion(model="nvidia_nim/seallms/seallm-7b-v2.5", messages)` |
|
||||
| snowflake/arctic | `completion(model="nvidia_nim/snowflake/arctic", messages)` |
|
||||
| upstage/solar-10.7b-instruct | `completion(model="nvidia_nim/upstage/solar-10.7b-instruct", messages)` |
|
||||
@@ -14,10 +14,11 @@ Features:
|
||||
- ✅ [SSO for Admin UI](./ui.md#✨-enterprise-features)
|
||||
- ✅ [Audit Logs](#audit-logs)
|
||||
- ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags)
|
||||
- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests)
|
||||
- ✅ [Content Moderation with LLM Guard, LlamaGuard, Google Text Moderations](#content-moderation)
|
||||
- ✅ [Control available public, private routes](#control-available-public-private-routes)
|
||||
- ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](#content-moderation)
|
||||
- ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai)
|
||||
- ✅ [Custom Branding + Routes on Swagger Docs](#swagger-docs---custom-routes--branding)
|
||||
- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests)
|
||||
- ✅ Reject calls from Blocked User list
|
||||
- ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors)
|
||||
|
||||
@@ -448,11 +449,144 @@ Expected Response
|
||||
|
||||
|
||||
|
||||
## Control available public, private routes
|
||||
|
||||
:::info
|
||||
|
||||
❓ Use this when you want to make an existing private route -> public
|
||||
|
||||
Example - Make `/spend/calculate` a publicly available route (by default `/spend/calculate` on LiteLLM Proxy requires authentication)
|
||||
|
||||
:::
|
||||
|
||||
#### Usage - Define public routes
|
||||
|
||||
**Step 1** - set allowed public routes on config.yaml
|
||||
|
||||
`LiteLLMRoutes.public_routes` is an ENUM corresponding to the default public routes on LiteLLM. [You can see this here](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/_types.py)
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
|
||||
```
|
||||
|
||||
**Step 2** - start proxy
|
||||
|
||||
```shell
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
**Step 3** - Test it
|
||||
|
||||
```shell
|
||||
curl --request POST \
|
||||
--url 'http://localhost:4000/spend/calculate' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hey, how'\''s it going?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
🎉 Expect this endpoint to work without an `Authorization / Bearer Token`
|
||||
|
||||
|
||||
|
||||
|
||||
## Content Moderation
|
||||
#### Content Moderation with LLM Guard
|
||||
### Content Moderation - Secret Detection
|
||||
❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM.
|
||||
|
||||
Example if you want to redact the value of `OPENAI_API_KEY` in the following request
|
||||
|
||||
#### Incoming Request
|
||||
|
||||
```json
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'",
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Request after Moderation
|
||||
|
||||
```json
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hey, how's it going, API_KEY = '[REDACTED]'",
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Usage**
|
||||
|
||||
**Step 1** Add this to your config.yaml
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
callbacks: ["hide_secrets"]
|
||||
```
|
||||
|
||||
**Step 2** Run litellm proxy with `--detailed_debug` to see the server logs
|
||||
|
||||
```
|
||||
litellm --config config.yaml --detailed_debug
|
||||
```
|
||||
|
||||
**Step 3** Test it with request
|
||||
|
||||
Send this request
|
||||
```shell
|
||||
curl --location 'http://localhost:4000/chat/completions' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "llama3",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what is the value of my open ai key? openai_api_key=sk-1234998222"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
|
||||
Expect to see the following warning on your litellm server logs
|
||||
|
||||
```shell
|
||||
LiteLLM Proxy:WARNING: secret_detection.py:88 - Detected and redacted secrets in message: ['Secret Keyword']
|
||||
```
|
||||
|
||||
|
||||
You can also see the raw request sent from litellm to the API Provider
|
||||
```json
|
||||
POST Request Sent from LiteLLM:
|
||||
curl -X POST \
|
||||
https://api.groq.com/openai/v1/ \
|
||||
-H 'Authorization: Bearer gsk_mySVchjY********************************************' \
|
||||
-d {
|
||||
"model": "llama3-8b-8192",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what is the time today, openai_api_key=[REDACTED]"
|
||||
}
|
||||
],
|
||||
"stream": false,
|
||||
"extra_body": {}
|
||||
}
|
||||
```
|
||||
|
||||
### Content Moderation with LLM Guard
|
||||
|
||||
Set the LLM Guard API Base in your environment
|
||||
|
||||
@@ -587,7 +721,7 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
#### Content Moderation with LlamaGuard
|
||||
### Content Moderation with LlamaGuard
|
||||
|
||||
Currently works with Sagemaker's LlamaGuard endpoint.
|
||||
|
||||
@@ -621,7 +755,7 @@ callbacks: ["llamaguard_moderations"]
|
||||
|
||||
|
||||
|
||||
#### Content Moderation with Google Text Moderation
|
||||
### Content Moderation with Google Text Moderation
|
||||
|
||||
Requires your GOOGLE_APPLICATION_CREDENTIALS to be set in your .env (same as VertexAI).
|
||||
|
||||
|
||||
@@ -272,6 +272,7 @@ litellm_settings:
|
||||
fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo"]}] # fallback to gpt-3.5-turbo if call fails num_retries
|
||||
context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error
|
||||
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
|
||||
cooldown_time: 30 # how long to cooldown model if fails/min > allowed_fails
|
||||
```
|
||||
### Context Window Fallbacks (Pre-Call Checks + Fallbacks)
|
||||
|
||||
@@ -431,6 +432,67 @@ litellm_settings:
|
||||
content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}]
|
||||
```
|
||||
|
||||
|
||||
|
||||
### Test Fallbacks!
|
||||
|
||||
Check if your fallbacks are working as expected.
|
||||
|
||||
#### **Regular Fallbacks**
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-D '{
|
||||
"model": "my-bad-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "ping"
|
||||
}
|
||||
],
|
||||
"mock_testing_fallbacks": true # 👈 KEY CHANGE
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
#### **Content Policy Fallbacks**
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-D '{
|
||||
"model": "my-bad-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "ping"
|
||||
}
|
||||
],
|
||||
"mock_testing_content_policy_fallbacks": true # 👈 KEY CHANGE
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
#### **Context Window Fallbacks**
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-D '{
|
||||
"model": "my-bad-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "ping"
|
||||
}
|
||||
],
|
||||
"mock_testing_context_window_fallbacks": true # 👈 KEY CHANGE
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
### EU-Region Filtering (Pre-Call Checks)
|
||||
|
||||
**Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**.
|
||||
|
||||
@@ -762,6 +762,9 @@ asyncio.run(router_acompletion())
|
||||
|
||||
Set the limit for how many calls a model is allowed to fail in a minute, before being cooled down for a minute.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import Router
|
||||
|
||||
@@ -779,9 +782,39 @@ messages = [{"content": user_message, "role": "user"}]
|
||||
response = router.completion(model="gpt-3.5-turbo", messages=messages)
|
||||
|
||||
print(f"response: {response}")
|
||||
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
**Set Global Value**
|
||||
|
||||
```yaml
|
||||
router_settings:
|
||||
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
|
||||
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
|
||||
```
|
||||
|
||||
Defaults:
|
||||
- allowed_fails: 0
|
||||
- cooldown_time: 60s
|
||||
|
||||
**Set Per Model**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: predibase/llama-3-8b-instruct
|
||||
api_key: os.environ/PREDIBASE_API_KEY
|
||||
tenant_id: os.environ/PREDIBASE_TENANT_ID
|
||||
max_new_tokens: 256
|
||||
cooldown_time: 0 # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Retries
|
||||
|
||||
For both async + sync functions, we support retrying failed requests.
|
||||
@@ -901,6 +934,39 @@ response = await router.acompletion(
|
||||
|
||||
If a call fails after num_retries, fall back to another model group.
|
||||
|
||||
### Quick Start
|
||||
|
||||
```python
|
||||
from litellm import Router
|
||||
router = Router(
|
||||
model_list=[
|
||||
{ # bad model
|
||||
"model_name": "bad-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/my-bad-model",
|
||||
"api_key": "my-bad-api-key",
|
||||
"mock_response": "Bad call"
|
||||
},
|
||||
},
|
||||
{ # good model
|
||||
"model_name": "my-good-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"mock_response": "Good call"
|
||||
},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"bad-model": ["my-good-model"]}] # 👈 KEY CHANGE
|
||||
)
|
||||
|
||||
response = router.completion(
|
||||
model="bad-model",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
mock_testing_fallbacks=True,
|
||||
)
|
||||
```
|
||||
|
||||
If the error is a context window exceeded error, fall back to a larger model group (if given).
|
||||
|
||||
Fallbacks are done in-order - ["gpt-3.5-turbo, "gpt-4", "gpt-4-32k"], will do 'gpt-3.5-turbo' first, then 'gpt-4', etc.
|
||||
|
||||
@@ -146,13 +146,14 @@ const sidebars = {
|
||||
"providers/databricks",
|
||||
"providers/watsonx",
|
||||
"providers/predibase",
|
||||
"providers/clarifai",
|
||||
"providers/nvidia_nim",
|
||||
"providers/triton-inference-server",
|
||||
"providers/ollama",
|
||||
"providers/perplexity",
|
||||
"providers/groq",
|
||||
"providers/deepseek",
|
||||
"providers/fireworks_ai",
|
||||
"providers/fireworks_ai",
|
||||
"providers/clarifai",
|
||||
"providers/vllm",
|
||||
"providers/xinference",
|
||||
"providers/cloudflare_workers",
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
# +-------------------------------------------------------------+
|
||||
#
|
||||
# Use SecretDetection /moderations for your LLM calls
|
||||
#
|
||||
# +-------------------------------------------------------------+
|
||||
# Thank you users! We ❤️ you! - Krrish & Ishaan
|
||||
|
||||
import sys, os
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
from typing import Optional, Literal, Union
|
||||
import litellm, traceback, sys, uuid
|
||||
from litellm.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from fastapi import HTTPException
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.utils import (
|
||||
ModelResponse,
|
||||
EmbeddingResponse,
|
||||
ImageResponse,
|
||||
StreamingChoices,
|
||||
)
|
||||
from datetime import datetime
|
||||
import aiohttp, asyncio
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
import tempfile
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
|
||||
litellm.set_verbose = True
|
||||
|
||||
|
||||
class _ENTERPRISE_SecretDetection(CustomLogger):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def scan_message_for_secrets(self, message_content: str):
|
||||
from detect_secrets import SecretsCollection
|
||||
from detect_secrets.settings import default_settings
|
||||
|
||||
temp_file = tempfile.NamedTemporaryFile(delete=False)
|
||||
temp_file.write(message_content.encode("utf-8"))
|
||||
temp_file.close()
|
||||
|
||||
secrets = SecretsCollection()
|
||||
with default_settings():
|
||||
secrets.scan_file(temp_file.name)
|
||||
|
||||
os.remove(temp_file.name)
|
||||
|
||||
detected_secrets = []
|
||||
for file in secrets.files:
|
||||
for found_secret in secrets[file]:
|
||||
if found_secret.secret_value is None:
|
||||
continue
|
||||
detected_secrets.append(
|
||||
{"type": found_secret.type, "value": found_secret.secret_value}
|
||||
)
|
||||
|
||||
return detected_secrets
|
||||
|
||||
#### CALL HOOKS - proxy only ####
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
cache: DualCache,
|
||||
data: dict,
|
||||
call_type: str, # "completion", "embeddings", "image_generation", "moderation"
|
||||
):
|
||||
from detect_secrets import SecretsCollection
|
||||
from detect_secrets.settings import default_settings
|
||||
|
||||
if "messages" in data and isinstance(data["messages"], list):
|
||||
for message in data["messages"]:
|
||||
if "content" in message and isinstance(message["content"], str):
|
||||
detected_secrets = self.scan_message_for_secrets(message["content"])
|
||||
|
||||
for secret in detected_secrets:
|
||||
message["content"] = message["content"].replace(
|
||||
secret["value"], "[REDACTED]"
|
||||
)
|
||||
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [secret["type"] for secret in detected_secrets]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in message: {secret_types}"
|
||||
)
|
||||
|
||||
if "prompt" in data:
|
||||
if isinstance(data["prompt"], str):
|
||||
detected_secrets = self.scan_message_for_secrets(data["prompt"])
|
||||
for secret in detected_secrets:
|
||||
data["prompt"] = data["prompt"].replace(
|
||||
secret["value"], "[REDACTED]"
|
||||
)
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [secret["type"] for secret in detected_secrets]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in prompt: {secret_types}"
|
||||
)
|
||||
elif isinstance(data["prompt"], list):
|
||||
for item in data["prompt"]:
|
||||
if isinstance(item, str):
|
||||
detected_secrets = self.scan_message_for_secrets(item)
|
||||
for secret in detected_secrets:
|
||||
item = item.replace(secret["value"], "[REDACTED]")
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [
|
||||
secret["type"] for secret in detected_secrets
|
||||
]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in prompt: {secret_types}"
|
||||
)
|
||||
|
||||
if "input" in data:
|
||||
if isinstance(data["input"], str):
|
||||
detected_secrets = self.scan_message_for_secrets(data["input"])
|
||||
for secret in detected_secrets:
|
||||
data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [secret["type"] for secret in detected_secrets]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in input: {secret_types}"
|
||||
)
|
||||
elif isinstance(data["input"], list):
|
||||
_input_in_request = data["input"]
|
||||
for idx, item in enumerate(_input_in_request):
|
||||
if isinstance(item, str):
|
||||
detected_secrets = self.scan_message_for_secrets(item)
|
||||
for secret in detected_secrets:
|
||||
_input_in_request[idx] = item.replace(
|
||||
secret["value"], "[REDACTED]"
|
||||
)
|
||||
if len(detected_secrets) > 0:
|
||||
secret_types = [
|
||||
secret["type"] for secret in detected_secrets
|
||||
]
|
||||
verbose_proxy_logger.warning(
|
||||
f"Detected and redacted secrets in input: {secret_types}"
|
||||
)
|
||||
verbose_proxy_logger.debug("Data after redacting input %s", data)
|
||||
return
|
||||
@@ -401,6 +401,7 @@ openai_compatible_endpoints: List = [
|
||||
"codestral.mistral.ai/v1/chat/completions",
|
||||
"codestral.mistral.ai/v1/fim/completions",
|
||||
"api.groq.com/openai/v1",
|
||||
"https://integrate.api.nvidia.com/v1",
|
||||
"api.deepseek.com/v1",
|
||||
"api.together.xyz/v1",
|
||||
"inference.friendli.ai/v1",
|
||||
@@ -411,6 +412,7 @@ openai_compatible_providers: List = [
|
||||
"anyscale",
|
||||
"mistral",
|
||||
"groq",
|
||||
"nvidia_nim",
|
||||
"codestral",
|
||||
"deepseek",
|
||||
"deepinfra",
|
||||
@@ -640,6 +642,7 @@ provider_list: List = [
|
||||
"anyscale",
|
||||
"mistral",
|
||||
"groq",
|
||||
"nvidia_nim",
|
||||
"codestral",
|
||||
"text-completion-codestral",
|
||||
"deepseek",
|
||||
@@ -813,6 +816,7 @@ from .llms.openai import (
|
||||
DeepInfraConfig,
|
||||
AzureAIStudioConfig,
|
||||
)
|
||||
from .llms.nvidia_nim import NvidiaNimConfig
|
||||
from .llms.text_completion_codestral import MistralTextCompletionConfig
|
||||
from .llms.azure import (
|
||||
AzureOpenAIConfig,
|
||||
|
||||
+5
-11
@@ -9,10 +9,11 @@
|
||||
|
||||
## LiteLLM versions of the OpenAI Exception Types
|
||||
|
||||
import openai
|
||||
import httpx
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
|
||||
|
||||
class AuthenticationError(openai.AuthenticationError): # type: ignore
|
||||
def __init__(
|
||||
@@ -658,15 +659,8 @@ class APIResponseValidationError(openai.APIResponseValidationError): # type: ig
|
||||
|
||||
|
||||
class OpenAIError(openai.OpenAIError): # type: ignore
|
||||
def __init__(self, original_exception):
|
||||
self.status_code = original_exception.http_status
|
||||
super().__init__(
|
||||
http_body=original_exception.http_body,
|
||||
http_status=original_exception.http_status,
|
||||
json_body=original_exception.json_body,
|
||||
headers=original_exception.headers,
|
||||
code=original_exception.code,
|
||||
)
|
||||
def __init__(self, original_exception=None):
|
||||
super().__init__()
|
||||
self.llm_provider = "openai"
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
#### What this does ####
|
||||
# On success, logs events to Promptlayer
|
||||
import dotenv, os
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.caching import DualCache
|
||||
from typing import Literal, Union, Optional
|
||||
import os
|
||||
import traceback
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import dotenv
|
||||
|
||||
from litellm.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
|
||||
class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
|
||||
|
||||
@@ -108,6 +108,7 @@ class LunaryLogger:
|
||||
try:
|
||||
print_verbose(f"Lunary Logging - Logging request for model {model}")
|
||||
|
||||
template_id = None
|
||||
litellm_params = kwargs.get("litellm_params", {})
|
||||
optional_params = kwargs.get("optional_params", {})
|
||||
metadata = litellm_params.get("metadata", {}) or {}
|
||||
|
||||
@@ -19,8 +19,7 @@ from litellm import (
|
||||
turn_off_message_logging,
|
||||
verbose_logger,
|
||||
)
|
||||
|
||||
from litellm.caching import InMemoryCache, S3Cache, DualCache
|
||||
from litellm.caching import DualCache, InMemoryCache, S3Cache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redact_message_input_output_from_logging,
|
||||
|
||||
@@ -902,7 +902,7 @@ class AzureChatCompletion(BaseLLM):
|
||||
},
|
||||
)
|
||||
|
||||
if aembedding == True:
|
||||
if aembedding is True:
|
||||
response = self.aembedding(
|
||||
data=data,
|
||||
input=input,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Nvidia NIM endpoint: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer
|
||||
|
||||
This is OpenAI compatible
|
||||
|
||||
This file only contains param mapping logic
|
||||
|
||||
API calling is done using the OpenAI SDK with an api_base
|
||||
"""
|
||||
|
||||
import types
|
||||
from typing import Optional, Union
|
||||
|
||||
|
||||
class NvidiaNimConfig:
|
||||
"""
|
||||
Reference: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer
|
||||
|
||||
The class `NvidiaNimConfig` provides configuration for the Nvidia NIM's Chat Completions API interface. Below are the parameters:
|
||||
"""
|
||||
|
||||
temperature: Optional[int] = None
|
||||
top_p: Optional[int] = None
|
||||
frequency_penalty: Optional[int] = None
|
||||
presence_penalty: Optional[int] = None
|
||||
max_tokens: Optional[int] = None
|
||||
stop: Optional[Union[str, list]] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
temperature: Optional[int] = None,
|
||||
top_p: Optional[int] = None,
|
||||
frequency_penalty: Optional[int] = None,
|
||||
presence_penalty: Optional[int] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
stop: Optional[Union[str, list]] = None,
|
||||
) -> None:
|
||||
locals_ = locals().copy()
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
types.FunctionType,
|
||||
types.BuiltinFunctionType,
|
||||
classmethod,
|
||||
staticmethod,
|
||||
),
|
||||
)
|
||||
and v is not None
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self):
|
||||
return [
|
||||
"stream",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"frequency_penalty",
|
||||
"presence_penalty",
|
||||
"max_tokens",
|
||||
"stop",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self, non_default_params: dict, optional_params: dict
|
||||
) -> dict:
|
||||
supported_openai_params = self.get_supported_openai_params()
|
||||
for param, value in non_default_params.items():
|
||||
if param in supported_openai_params:
|
||||
optional_params[param] = value
|
||||
return optional_params
|
||||
@@ -126,7 +126,7 @@ class OllamaConfig:
|
||||
)
|
||||
and v is not None
|
||||
}
|
||||
|
||||
|
||||
def get_required_params(self) -> List[ProviderField]:
|
||||
"""For a given provider, return it's required fields with a description"""
|
||||
return [
|
||||
@@ -451,7 +451,7 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj):
|
||||
{
|
||||
"id": f"call_{str(uuid.uuid4())}",
|
||||
"function": {
|
||||
"name": function_call["name"],
|
||||
"name": function_call.get("name", function_call.get("function", None)),
|
||||
"arguments": json.dumps(function_call["arguments"]),
|
||||
},
|
||||
"type": "function",
|
||||
|
||||
@@ -434,7 +434,7 @@ async def ollama_async_streaming(
|
||||
{
|
||||
"id": f"call_{str(uuid.uuid4())}",
|
||||
"function": {
|
||||
"name": function_call["name"],
|
||||
"name": function_call.get("name", function_call.get("function", None)),
|
||||
"arguments": json.dumps(function_call["arguments"]),
|
||||
},
|
||||
"type": "function",
|
||||
|
||||
+72
-26
@@ -1,27 +1,28 @@
|
||||
# What is this?
|
||||
## Controller file for Predibase Integration - https://predibase.com/
|
||||
|
||||
from functools import partial
|
||||
import os, types
|
||||
import traceback
|
||||
import copy
|
||||
import json
|
||||
from enum import Enum
|
||||
import requests, copy # type: ignore
|
||||
import os
|
||||
import time
|
||||
from typing import Callable, Optional, List, Literal, Union
|
||||
from litellm.utils import (
|
||||
ModelResponse,
|
||||
Usage,
|
||||
CustomStreamWrapper,
|
||||
Message,
|
||||
Choices,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
import litellm
|
||||
from .prompt_templates.factory import prompt_factory, custom_prompt
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from .base import BaseLLM
|
||||
import traceback
|
||||
import types
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
from typing import Callable, List, Literal, Optional, Union
|
||||
|
||||
import httpx # type: ignore
|
||||
import requests # type: ignore
|
||||
|
||||
import litellm
|
||||
import litellm.litellm_core_utils
|
||||
import litellm.litellm_core_utils.litellm_logging
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage
|
||||
|
||||
from .base import BaseLLM
|
||||
from .prompt_templates.factory import custom_prompt, prompt_factory
|
||||
|
||||
|
||||
class PredibaseError(Exception):
|
||||
@@ -146,7 +147,49 @@ class PredibaseConfig:
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self):
|
||||
return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"]
|
||||
return [
|
||||
"stream",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"top_p",
|
||||
"stop",
|
||||
"n",
|
||||
"response_format",
|
||||
]
|
||||
|
||||
def map_openai_params(self, non_default_params: dict, optional_params: dict):
|
||||
for param, value in non_default_params.items():
|
||||
# temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None
|
||||
if param == "temperature":
|
||||
if value == 0.0 or value == 0:
|
||||
# hugging face exception raised when temp==0
|
||||
# Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive
|
||||
value = 0.01
|
||||
optional_params["temperature"] = value
|
||||
if param == "top_p":
|
||||
optional_params["top_p"] = value
|
||||
if param == "n":
|
||||
optional_params["best_of"] = value
|
||||
optional_params["do_sample"] = (
|
||||
True # Need to sample if you want best of for hf inference endpoints
|
||||
)
|
||||
if param == "stream":
|
||||
optional_params["stream"] = value
|
||||
if param == "stop":
|
||||
optional_params["stop"] = value
|
||||
if param == "max_tokens":
|
||||
# HF TGI raises the following exception when max_new_tokens==0
|
||||
# Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive
|
||||
if value == 0:
|
||||
value = 1
|
||||
optional_params["max_new_tokens"] = value
|
||||
if param == "echo":
|
||||
# https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details
|
||||
# Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False
|
||||
optional_params["decoder_input_details"] = True
|
||||
if param == "response_format":
|
||||
optional_params["response_format"] = value
|
||||
return optional_params
|
||||
|
||||
|
||||
class PredibaseChatCompletion(BaseLLM):
|
||||
@@ -225,15 +268,16 @@ class PredibaseChatCompletion(BaseLLM):
|
||||
status_code=response.status_code,
|
||||
)
|
||||
else:
|
||||
if (
|
||||
not isinstance(completion_response, dict)
|
||||
or "generated_text" not in completion_response
|
||||
):
|
||||
if not isinstance(completion_response, dict):
|
||||
raise PredibaseError(
|
||||
status_code=422,
|
||||
message=f"response is not in expected format - {completion_response}",
|
||||
message=f"'completion_response' is not a dictionary - {completion_response}",
|
||||
)
|
||||
elif "generated_text" not in completion_response:
|
||||
raise PredibaseError(
|
||||
status_code=422,
|
||||
message=f"'generated_text' is not a key response dictionary - {completion_response}",
|
||||
)
|
||||
|
||||
if len(completion_response["generated_text"]) > 0:
|
||||
model_response["choices"][0]["message"]["content"] = self.output_parser(
|
||||
completion_response["generated_text"]
|
||||
@@ -496,7 +540,9 @@ class PredibaseChatCompletion(BaseLLM):
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise PredibaseError(
|
||||
status_code=e.response.status_code,
|
||||
message="HTTPStatusError - {}".format(e.response.text),
|
||||
message="HTTPStatusError - received status_code={}, error_message={}".format(
|
||||
e.response.status_code, e.response.text
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
raise PredibaseError(
|
||||
|
||||
@@ -172,14 +172,35 @@ def ollama_pt(
|
||||
images.append(base64_image)
|
||||
return {"prompt": prompt, "images": images}
|
||||
else:
|
||||
prompt = "".join(
|
||||
(
|
||||
m["content"]
|
||||
if isinstance(m["content"], str) is str
|
||||
else "".join(m["content"])
|
||||
)
|
||||
for m in messages
|
||||
)
|
||||
prompt = ""
|
||||
for message in messages:
|
||||
role = message["role"]
|
||||
content = message.get("content", "")
|
||||
|
||||
if "tool_calls" in message:
|
||||
tool_calls = []
|
||||
|
||||
for call in message["tool_calls"]:
|
||||
call_id: str = call["id"]
|
||||
function_name: str = call["function"]["name"]
|
||||
arguments = json.loads(call["function"]["arguments"])
|
||||
|
||||
tool_calls.append(
|
||||
{
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {"name": function_name, "arguments": arguments},
|
||||
}
|
||||
)
|
||||
|
||||
prompt += f"### Assistant:\nTool Calls: {json.dumps(tool_calls, indent=2)}\n\n"
|
||||
|
||||
elif "tool_call_id" in message:
|
||||
prompt += f"### User:\n{message['content']}\n\n"
|
||||
|
||||
elif content:
|
||||
prompt += f"### {role.capitalize()}:\n{content}\n\n"
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
@@ -710,7 +731,7 @@ def convert_to_anthropic_tool_result_xml(message: dict) -> str:
|
||||
|
||||
"""
|
||||
Anthropic tool_results look like:
|
||||
|
||||
|
||||
[Successful results]
|
||||
<function_results>
|
||||
<result>
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
import os, types
|
||||
import asyncio
|
||||
import json
|
||||
import requests # type: ignore
|
||||
import os
|
||||
import time
|
||||
from typing import Callable, Optional, Union, Tuple, Any
|
||||
from litellm.utils import ModelResponse, Usage, CustomStreamWrapper
|
||||
import litellm, asyncio
|
||||
import types
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
|
||||
import httpx # type: ignore
|
||||
from .prompt_templates.factory import prompt_factory, custom_prompt
|
||||
import requests # type: ignore
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
|
||||
|
||||
from .prompt_templates.factory import custom_prompt, prompt_factory
|
||||
|
||||
|
||||
class ReplicateError(Exception):
|
||||
@@ -329,7 +334,15 @@ async def async_handle_prediction_response_streaming(
|
||||
response_data = response.json()
|
||||
status = response_data["status"]
|
||||
if "output" in response_data:
|
||||
output_string = "".join(response_data["output"])
|
||||
try:
|
||||
output_string = "".join(response_data["output"])
|
||||
except Exception as e:
|
||||
raise ReplicateError(
|
||||
status_code=422,
|
||||
message="Unable to parse response. Got={}".format(
|
||||
response_data["output"]
|
||||
),
|
||||
)
|
||||
new_output = output_string[len(previous_output) :]
|
||||
print_verbose(f"New chunk: {new_output}")
|
||||
yield {"output": new_output, "status": status}
|
||||
|
||||
@@ -562,7 +562,47 @@ class VertexLLM(BaseLLM):
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
## GET MODEL ##
|
||||
model_response.model = model
|
||||
|
||||
## CHECK IF RESPONSE FLAGGED
|
||||
if "promptFeedback" in completion_response:
|
||||
if "blockReason" in completion_response["promptFeedback"]:
|
||||
# If set, the prompt was blocked and no candidates are returned. Rephrase your prompt
|
||||
model_response.choices[0].finish_reason = "content_filter"
|
||||
|
||||
chat_completion_message: ChatCompletionResponseMessage = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
}
|
||||
|
||||
choice = litellm.Choices(
|
||||
finish_reason="content_filter",
|
||||
index=0,
|
||||
message=chat_completion_message, # type: ignore
|
||||
logprobs=None,
|
||||
enhancements=None,
|
||||
)
|
||||
|
||||
model_response.choices = [choice]
|
||||
|
||||
## GET USAGE ##
|
||||
usage = litellm.Usage(
|
||||
prompt_tokens=completion_response["usageMetadata"][
|
||||
"promptTokenCount"
|
||||
],
|
||||
completion_tokens=completion_response["usageMetadata"].get(
|
||||
"candidatesTokenCount", 0
|
||||
),
|
||||
total_tokens=completion_response["usageMetadata"][
|
||||
"totalTokenCount"
|
||||
],
|
||||
)
|
||||
|
||||
setattr(model_response, "usage", usage)
|
||||
|
||||
return model_response
|
||||
|
||||
if len(completion_response["candidates"]) > 0:
|
||||
content_policy_violations = (
|
||||
VertexGeminiConfig().get_flagged_finish_reasons()
|
||||
@@ -573,26 +613,45 @@ class VertexLLM(BaseLLM):
|
||||
in content_policy_violations.keys()
|
||||
):
|
||||
## CONTENT POLICY VIOLATION ERROR
|
||||
raise VertexAIError(
|
||||
status_code=400,
|
||||
message="The response was blocked. Reason={}. Raw Response={}".format(
|
||||
content_policy_violations[
|
||||
completion_response["candidates"][0]["finishReason"]
|
||||
],
|
||||
completion_response,
|
||||
),
|
||||
model_response.choices[0].finish_reason = "content_filter"
|
||||
|
||||
chat_completion_message = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
}
|
||||
|
||||
choice = litellm.Choices(
|
||||
finish_reason="content_filter",
|
||||
index=0,
|
||||
message=chat_completion_message, # type: ignore
|
||||
logprobs=None,
|
||||
enhancements=None,
|
||||
)
|
||||
|
||||
model_response.choices = [choice]
|
||||
|
||||
## GET USAGE ##
|
||||
usage = litellm.Usage(
|
||||
prompt_tokens=completion_response["usageMetadata"][
|
||||
"promptTokenCount"
|
||||
],
|
||||
completion_tokens=completion_response["usageMetadata"].get(
|
||||
"candidatesTokenCount", 0
|
||||
),
|
||||
total_tokens=completion_response["usageMetadata"][
|
||||
"totalTokenCount"
|
||||
],
|
||||
)
|
||||
|
||||
setattr(model_response, "usage", usage)
|
||||
|
||||
return model_response
|
||||
|
||||
model_response.choices = [] # type: ignore
|
||||
|
||||
## GET MODEL ##
|
||||
model_response.model = model
|
||||
|
||||
try:
|
||||
## GET TEXT ##
|
||||
chat_completion_message: ChatCompletionResponseMessage = {
|
||||
"role": "assistant"
|
||||
}
|
||||
chat_completion_message = {"role": "assistant"}
|
||||
content_str = ""
|
||||
tools: List[ChatCompletionToolCallChunk] = []
|
||||
for idx, candidate in enumerate(completion_response["candidates"]):
|
||||
@@ -632,9 +691,9 @@ class VertexLLM(BaseLLM):
|
||||
## GET USAGE ##
|
||||
usage = litellm.Usage(
|
||||
prompt_tokens=completion_response["usageMetadata"]["promptTokenCount"],
|
||||
completion_tokens=completion_response["usageMetadata"][
|
||||
"candidatesTokenCount"
|
||||
],
|
||||
completion_tokens=completion_response["usageMetadata"].get(
|
||||
"candidatesTokenCount", 0
|
||||
),
|
||||
total_tokens=completion_response["usageMetadata"]["totalTokenCount"],
|
||||
)
|
||||
|
||||
|
||||
+30
-4
@@ -348,6 +348,7 @@ async def acompletion(
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "nvidia_nim"
|
||||
or custom_llm_provider == "codestral"
|
||||
or custom_llm_provider == "text-completion-codestral"
|
||||
or custom_llm_provider == "deepseek"
|
||||
@@ -428,6 +429,7 @@ def mock_completion(
|
||||
model: str,
|
||||
messages: List,
|
||||
stream: Optional[bool] = False,
|
||||
n: Optional[int] = None,
|
||||
mock_response: Union[str, Exception, dict] = "This is a mock request",
|
||||
mock_tool_calls: Optional[List] = None,
|
||||
logging=None,
|
||||
@@ -486,18 +488,32 @@ def mock_completion(
|
||||
if kwargs.get("acompletion", False) == True:
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=async_mock_completion_streaming_obj(
|
||||
model_response, mock_response=mock_response, model=model
|
||||
model_response, mock_response=mock_response, model=model, n=n
|
||||
),
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging,
|
||||
)
|
||||
response = mock_completion_streaming_obj(
|
||||
model_response, mock_response=mock_response, model=model
|
||||
model_response,
|
||||
mock_response=mock_response,
|
||||
model=model,
|
||||
n=n,
|
||||
)
|
||||
return response
|
||||
|
||||
model_response["choices"][0]["message"]["content"] = mock_response
|
||||
if n is None:
|
||||
model_response["choices"][0]["message"]["content"] = mock_response
|
||||
else:
|
||||
_all_choices = []
|
||||
for i in range(n):
|
||||
_choice = litellm.utils.Choices(
|
||||
index=i,
|
||||
message=litellm.utils.Message(
|
||||
content=mock_response, role="assistant"
|
||||
),
|
||||
)
|
||||
_all_choices.append(_choice)
|
||||
model_response["choices"] = _all_choices
|
||||
model_response["created"] = int(time.time())
|
||||
model_response["model"] = model
|
||||
|
||||
@@ -634,6 +650,7 @@ def completion(
|
||||
headers = kwargs.get("headers", None) or extra_headers
|
||||
num_retries = kwargs.get("num_retries", None) ## deprecated
|
||||
max_retries = kwargs.get("max_retries", None)
|
||||
cooldown_time = kwargs.get("cooldown_time", None)
|
||||
context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None)
|
||||
organization = kwargs.get("organization", None)
|
||||
### CUSTOM MODEL COST ###
|
||||
@@ -747,6 +764,7 @@ def completion(
|
||||
"allowed_model_region",
|
||||
"model_config",
|
||||
"fastest_response",
|
||||
"cooldown_time",
|
||||
]
|
||||
|
||||
default_params = openai_params + litellm_params
|
||||
@@ -931,6 +949,7 @@ def completion(
|
||||
input_cost_per_token=input_cost_per_token,
|
||||
output_cost_per_second=output_cost_per_second,
|
||||
output_cost_per_token=output_cost_per_token,
|
||||
cooldown_time=cooldown_time,
|
||||
)
|
||||
logging.update_environment_variables(
|
||||
model=model,
|
||||
@@ -944,6 +963,7 @@ def completion(
|
||||
model,
|
||||
messages,
|
||||
stream=stream,
|
||||
n=n,
|
||||
mock_response=mock_response,
|
||||
mock_tool_calls=mock_tool_calls,
|
||||
logging=logging,
|
||||
@@ -1171,6 +1191,7 @@ def completion(
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "nvidia_nim"
|
||||
or custom_llm_provider == "codestral"
|
||||
or custom_llm_provider == "deepseek"
|
||||
or custom_llm_provider == "anyscale"
|
||||
@@ -2906,6 +2927,7 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "nvidia_nim"
|
||||
or custom_llm_provider == "deepseek"
|
||||
or custom_llm_provider == "fireworks_ai"
|
||||
or custom_llm_provider == "ollama"
|
||||
@@ -2985,6 +3007,7 @@ def embedding(
|
||||
client = kwargs.pop("client", None)
|
||||
rpm = kwargs.pop("rpm", None)
|
||||
tpm = kwargs.pop("tpm", None)
|
||||
cooldown_time = kwargs.get("cooldown_time", None)
|
||||
max_parallel_requests = kwargs.pop("max_parallel_requests", None)
|
||||
model_info = kwargs.get("model_info", None)
|
||||
metadata = kwargs.get("metadata", None)
|
||||
@@ -3060,6 +3083,7 @@ def embedding(
|
||||
"region_name",
|
||||
"allowed_model_region",
|
||||
"model_config",
|
||||
"cooldown_time",
|
||||
]
|
||||
default_params = openai_params + litellm_params
|
||||
non_default_params = {
|
||||
@@ -3120,6 +3144,7 @@ def embedding(
|
||||
"aembedding": aembedding,
|
||||
"preset_cache_key": None,
|
||||
"stream_response": {},
|
||||
"cooldown_time": cooldown_time,
|
||||
},
|
||||
)
|
||||
if azure == True or custom_llm_provider == "azure":
|
||||
@@ -3481,6 +3506,7 @@ async def atext_completion(
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "nvidia_nim"
|
||||
or custom_llm_provider == "text-completion-codestral"
|
||||
or custom_llm_provider == "deepseek"
|
||||
or custom_llm_provider == "fireworks_ai"
|
||||
|
||||
@@ -887,7 +887,7 @@
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00000005,
|
||||
"output_cost_per_token": 0.00000010,
|
||||
"output_cost_per_token": 0.00000008,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
@@ -906,8 +906,8 @@
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 0.00000027,
|
||||
"output_cost_per_token": 0.00000027,
|
||||
"input_cost_per_token": 0.00000024,
|
||||
"output_cost_per_token": 0.00000024,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
@@ -916,8 +916,8 @@
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00000010,
|
||||
"output_cost_per_token": 0.00000010,
|
||||
"input_cost_per_token": 0.00000007,
|
||||
"output_cost_per_token": 0.00000007,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
|
||||
@@ -1,10 +1,54 @@
|
||||
model_list:
|
||||
- model_name: my-fake-model
|
||||
# model_list:
|
||||
# - model_name: my-fake-model
|
||||
# litellm_params:
|
||||
# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
|
||||
# api_key: my-fake-key
|
||||
# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
|
||||
# mock_response: "Hello world 1"
|
||||
# model_info:
|
||||
# max_input_tokens: 0 # trigger context window fallback
|
||||
# - model_name: my-fake-model
|
||||
# litellm_params:
|
||||
# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
|
||||
# api_key: my-fake-key
|
||||
# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
|
||||
# mock_response: "Hello world 2"
|
||||
# model_info:
|
||||
# max_input_tokens: 0
|
||||
|
||||
# router_settings:
|
||||
# enable_pre_call_checks: True
|
||||
|
||||
|
||||
# litellm_settings:
|
||||
# failure_callback: ["langfuse"]
|
||||
|
||||
model_list:
|
||||
- model_name: summarize
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
|
||||
api_key: my-fake-key
|
||||
aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
|
||||
model: openai/gpt-4o
|
||||
rpm: 10000
|
||||
tpm: 12000000
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
mock_response: Hello world 1
|
||||
|
||||
- model_name: summarize-l
|
||||
litellm_params:
|
||||
model: claude-3-5-sonnet-20240620
|
||||
rpm: 4000
|
||||
tpm: 400000
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
mock_response: Hello world 2
|
||||
|
||||
litellm_settings:
|
||||
success_callback: ["langfuse"]
|
||||
failure_callback: ["langfuse"]
|
||||
num_retries: 3
|
||||
request_timeout: 120
|
||||
allowed_fails: 3
|
||||
# fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
|
||||
# context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
|
||||
|
||||
|
||||
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle
|
||||
enable_pre_call_checks: true.
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
model_list:
|
||||
- model_name: gemini-1.5-flash-gemini
|
||||
litellm_params:
|
||||
model: gemini/gemini-1.5-flash
|
||||
- litellm_params:
|
||||
api_base: http://0.0.0.0:8080
|
||||
api_key: ''
|
||||
@@ -11,13 +14,10 @@ model_list:
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: predibase/llama-3-8b-instruct
|
||||
api_base: "http://0.0.0.0:8000"
|
||||
api_base: "http://0.0.0.0:8081"
|
||||
api_key: os.environ/PREDIBASE_API_KEY
|
||||
tenant_id: os.environ/PREDIBASE_TENANT_ID
|
||||
max_retries: 0
|
||||
temperature: 0.1
|
||||
max_new_tokens: 256
|
||||
return_full_text: false
|
||||
|
||||
# - litellm_params:
|
||||
# api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
|
||||
@@ -70,6 +70,8 @@ model_list:
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["dynamic_rate_limiter"]
|
||||
# success_callback: ["langfuse"]
|
||||
# failure_callback: ["langfuse"]
|
||||
# default_team_settings:
|
||||
# - team_id: proj1
|
||||
# success_callback: ["langfuse"]
|
||||
@@ -91,8 +93,8 @@ assistant_settings:
|
||||
router_settings:
|
||||
enable_pre_call_checks: true
|
||||
|
||||
general_settings:
|
||||
alerting: ["slack"]
|
||||
enable_jwt_auth: True
|
||||
litellm_jwtauth:
|
||||
team_id_jwt_field: "client_id"
|
||||
# general_settings:
|
||||
# # alerting: ["slack"]
|
||||
# enable_jwt_auth: True
|
||||
# litellm_jwtauth:
|
||||
# team_id_jwt_field: "client_id"
|
||||
@@ -1627,3 +1627,9 @@ class CommonProxyErrors(enum.Enum):
|
||||
no_llm_router = "No models configured on proxy"
|
||||
not_allowed_access = "Admin-only endpoint. Not allowed to access this."
|
||||
not_premium_user = "You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
|
||||
|
||||
|
||||
class SpendCalculateRequest(LiteLLMBase):
|
||||
model: Optional[str] = None
|
||||
messages: Optional[List] = None
|
||||
completion_response: Optional[dict] = None
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
|
||||
def route_in_additonal_public_routes(current_route: str):
|
||||
"""
|
||||
Helper to check if the user defined public_routes on config.yaml
|
||||
|
||||
Parameters:
|
||||
- current_route: str - the route the user is trying to call
|
||||
|
||||
Returns:
|
||||
- bool - True if the route is defined in public_routes
|
||||
- bool - False if the route is not defined in public_routes
|
||||
|
||||
|
||||
In order to use this the litellm config.yaml should have the following in general_settings:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
|
||||
```
|
||||
"""
|
||||
|
||||
# check if user is premium_user - if not do nothing
|
||||
from litellm.proxy._types import LiteLLMRoutes
|
||||
from litellm.proxy.proxy_server import general_settings, premium_user
|
||||
|
||||
try:
|
||||
if premium_user is not True:
|
||||
return False
|
||||
# check if this is defined on the config
|
||||
if general_settings is None:
|
||||
return False
|
||||
|
||||
routes_defined = general_settings.get("public_routes", [])
|
||||
if current_route in routes_defined:
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"route_in_additonal_public_routes: {str(e)}")
|
||||
return False
|
||||
@@ -1,6 +1,11 @@
|
||||
# What is this?
|
||||
## If litellm license in env, checks if it's valid
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
|
||||
@@ -15,6 +20,26 @@ class LicenseCheck:
|
||||
def __init__(self) -> None:
|
||||
self.license_str = os.getenv("LITELLM_LICENSE", None)
|
||||
self.http_handler = HTTPHandler()
|
||||
self.public_key = None
|
||||
self.read_public_key()
|
||||
|
||||
def read_public_key(self):
|
||||
try:
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding, rsa
|
||||
|
||||
# current dir
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
# check if public_key.pem exists
|
||||
_path_to_public_key = os.path.join(current_dir, "public_key.pem")
|
||||
if os.path.exists(_path_to_public_key):
|
||||
with open(_path_to_public_key, "rb") as key_file:
|
||||
self.public_key = serialization.load_pem_public_key(key_file.read())
|
||||
else:
|
||||
self.public_key = None
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"Error reading public key: {str(e)}")
|
||||
|
||||
def _verify(self, license_str: str) -> bool:
|
||||
url = "{}/verify_license/{}".format(self.base_url, license_str)
|
||||
@@ -35,11 +60,58 @@ class LicenseCheck:
|
||||
return False
|
||||
|
||||
def is_premium(self) -> bool:
|
||||
"""
|
||||
1. verify_license_without_api_request: checks if license was generate using private / public key pair
|
||||
2. _verify: checks if license is valid calling litellm API. This is the old way we were generating/validating license
|
||||
"""
|
||||
try:
|
||||
if self.license_str is None:
|
||||
return False
|
||||
elif self.verify_license_without_api_request(
|
||||
public_key=self.public_key, license_key=self.license_str
|
||||
):
|
||||
return True
|
||||
elif self._verify(license_str=self.license_str):
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def verify_license_without_api_request(self, public_key, license_key):
|
||||
try:
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding, rsa
|
||||
|
||||
# Decode the license key
|
||||
decoded = base64.b64decode(license_key)
|
||||
message, signature = decoded.split(b".", 1)
|
||||
|
||||
# Verify the signature
|
||||
public_key.verify(
|
||||
signature,
|
||||
message,
|
||||
padding.PSS(
|
||||
mgf=padding.MGF1(hashes.SHA256()),
|
||||
salt_length=padding.PSS.MAX_LENGTH,
|
||||
),
|
||||
hashes.SHA256(),
|
||||
)
|
||||
|
||||
# Decode and parse the data
|
||||
license_data = json.loads(message.decode())
|
||||
|
||||
# debug information provided in license data
|
||||
verbose_proxy_logger.debug("License data: %s", license_data)
|
||||
|
||||
# Check expiration date
|
||||
expiration_date = datetime.strptime(
|
||||
license_data["expiration_date"], "%Y-%m-%d"
|
||||
)
|
||||
if expiration_date < datetime.now():
|
||||
return False, "License has expired"
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(str(e))
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
-----BEGIN PUBLIC KEY-----
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmfBuNiNzDkNWyce23koQ
|
||||
w0vq3bSVHkq7fd9Sw/U1q7FwRwL221daLTyGWssd8xAoQSFXAJKoBwzJQ9wd+o44
|
||||
lfL54E3a61nfjZuF+D9ntpXZFfEAxLVtIahDeQjUz4b/EpgciWIJyUfjCJrQo6LY
|
||||
eyAZPTGSO8V3zHyaU+CFywq5XCuCnfZqCZeCw051St59A2v8W32mXSCJ+A+x0hYP
|
||||
yXJyRRFcefSFG5IBuRHr4Y24Vx7NUIAoco5cnxJho9g2z3J/Hb0GKW+oBNvRVumk
|
||||
nuA2Ljmjh4yI0OoTIW8ZWxemvCCJHSjdfKlMyb+QI4fmeiIUZzP5Au+F561Styqq
|
||||
YQIDAQAB
|
||||
-----END PUBLIC KEY-----
|
||||
@@ -56,6 +56,7 @@ from litellm.proxy.auth.auth_checks import (
|
||||
get_user_object,
|
||||
log_to_opentelemetry,
|
||||
)
|
||||
from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes
|
||||
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
|
||||
from litellm.proxy.utils import _to_ns
|
||||
|
||||
@@ -137,7 +138,10 @@ async def user_api_key_auth(
|
||||
"""
|
||||
route: str = request.url.path
|
||||
|
||||
if route in LiteLLMRoutes.public_routes.value:
|
||||
if (
|
||||
route in LiteLLMRoutes.public_routes.value
|
||||
or route_in_additonal_public_routes(current_route=route)
|
||||
):
|
||||
# check if public endpoint
|
||||
return UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY)
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# Start tracing memory allocations
|
||||
import os
|
||||
import tracemalloc
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
|
||||
tracemalloc.start()
|
||||
|
||||
@router.get("/memory-usage", include_in_schema=False)
|
||||
async def memory_usage():
|
||||
# Take a snapshot of the current memory usage
|
||||
snapshot = tracemalloc.take_snapshot()
|
||||
top_stats = snapshot.statistics("lineno")
|
||||
verbose_proxy_logger.debug("TOP STATS: %s", top_stats)
|
||||
|
||||
# Get the top 50 memory usage lines
|
||||
top_50 = top_stats[:50]
|
||||
result = []
|
||||
for stat in top_50:
|
||||
result.append(f"{stat.traceback.format()}: {stat.size / 1024} KiB")
|
||||
|
||||
return {"top_50_memory_usage": result}
|
||||
@@ -21,10 +21,12 @@ model_list:
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
alerting: ["slack", "email"]
|
||||
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
|
||||
|
||||
|
||||
litellm_settings:
|
||||
success_callback: ["prometheus"]
|
||||
callbacks: ["otel"]
|
||||
callbacks: ["otel", "hide_secrets"]
|
||||
failure_callback: ["prometheus"]
|
||||
store_audit_logs: true
|
||||
redact_messages_in_exceptions: True
|
||||
|
||||
@@ -140,6 +140,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
## Import All Misc routes here ##
|
||||
from litellm.proxy.caching_routes import router as caching_router
|
||||
from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router
|
||||
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
from litellm.proxy.health_endpoints._health_endpoints import router as health_router
|
||||
@@ -1478,6 +1479,21 @@ class ProxyConfig:
|
||||
|
||||
llama_guard_object = _ENTERPRISE_LlamaGuard()
|
||||
imported_list.append(llama_guard_object)
|
||||
elif (
|
||||
isinstance(callback, str) and callback == "hide_secrets"
|
||||
):
|
||||
from enterprise.enterprise_hooks.secret_detection import (
|
||||
_ENTERPRISE_SecretDetection,
|
||||
)
|
||||
|
||||
if premium_user != True:
|
||||
raise Exception(
|
||||
"Trying to use secret hiding"
|
||||
+ CommonProxyErrors.not_premium_user.value
|
||||
)
|
||||
|
||||
_secret_detection_object = _ENTERPRISE_SecretDetection()
|
||||
imported_list.append(_secret_detection_object)
|
||||
elif (
|
||||
isinstance(callback, str)
|
||||
and callback == "openai_moderations"
|
||||
@@ -7508,12 +7524,6 @@ async def login(request: Request):
|
||||
litellm_dashboard_ui += "/ui/"
|
||||
import jwt
|
||||
|
||||
if litellm_master_key_hash is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "No master key set, please set LITELLM_MASTER_KEY"},
|
||||
)
|
||||
|
||||
jwt_token = jwt.encode(
|
||||
{
|
||||
"user_id": user_id,
|
||||
@@ -7523,7 +7533,7 @@ async def login(request: Request):
|
||||
"login_method": "username_password",
|
||||
"premium_user": premium_user,
|
||||
},
|
||||
litellm_master_key_hash,
|
||||
master_key,
|
||||
algorithm="HS256",
|
||||
)
|
||||
litellm_dashboard_ui += "?userID=" + user_id
|
||||
@@ -7578,14 +7588,6 @@ async def login(request: Request):
|
||||
litellm_dashboard_ui += "/ui/"
|
||||
import jwt
|
||||
|
||||
if litellm_master_key_hash is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={
|
||||
"error": "No master key set, please set LITELLM_MASTER_KEY"
|
||||
},
|
||||
)
|
||||
|
||||
jwt_token = jwt.encode(
|
||||
{
|
||||
"user_id": user_id,
|
||||
@@ -7595,7 +7597,7 @@ async def login(request: Request):
|
||||
"login_method": "username_password",
|
||||
"premium_user": premium_user,
|
||||
},
|
||||
litellm_master_key_hash,
|
||||
master_key,
|
||||
algorithm="HS256",
|
||||
)
|
||||
litellm_dashboard_ui += "?userID=" + user_id
|
||||
@@ -7642,7 +7644,14 @@ async def onboarding(invite_link: str):
|
||||
- Get user from db
|
||||
- Pass in user_email if set
|
||||
"""
|
||||
global prisma_client
|
||||
global prisma_client, master_key
|
||||
if master_key is None:
|
||||
raise ProxyException(
|
||||
message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.",
|
||||
type="auth_error",
|
||||
param="master_key",
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
### VALIDATE INVITE LINK ###
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
@@ -7714,12 +7723,6 @@ async def onboarding(invite_link: str):
|
||||
litellm_dashboard_ui += "/ui/onboarding"
|
||||
import jwt
|
||||
|
||||
if litellm_master_key_hash is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "No master key set, please set LITELLM_MASTER_KEY"},
|
||||
)
|
||||
|
||||
jwt_token = jwt.encode(
|
||||
{
|
||||
"user_id": user_obj.user_id,
|
||||
@@ -7729,7 +7732,7 @@ async def onboarding(invite_link: str):
|
||||
"login_method": "username_password",
|
||||
"premium_user": premium_user,
|
||||
},
|
||||
litellm_master_key_hash,
|
||||
master_key,
|
||||
algorithm="HS256",
|
||||
)
|
||||
|
||||
@@ -7862,11 +7865,18 @@ def get_image():
|
||||
@app.get("/sso/callback", tags=["experimental"], include_in_schema=False)
|
||||
async def auth_callback(request: Request):
|
||||
"""Verify login"""
|
||||
global general_settings, ui_access_mode, premium_user
|
||||
global general_settings, ui_access_mode, premium_user, master_key
|
||||
microsoft_client_id = os.getenv("MICROSOFT_CLIENT_ID", None)
|
||||
google_client_id = os.getenv("GOOGLE_CLIENT_ID", None)
|
||||
generic_client_id = os.getenv("GENERIC_CLIENT_ID", None)
|
||||
# get url from request
|
||||
if master_key is None:
|
||||
raise ProxyException(
|
||||
message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.",
|
||||
type="auth_error",
|
||||
param="master_key",
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
redirect_url = os.getenv("PROXY_BASE_URL", str(request.base_url))
|
||||
if redirect_url.endswith("/"):
|
||||
redirect_url += "sso/callback"
|
||||
@@ -8140,12 +8150,6 @@ async def auth_callback(request: Request):
|
||||
|
||||
import jwt
|
||||
|
||||
if litellm_master_key_hash is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "No master key set, please set LITELLM_MASTER_KEY"},
|
||||
)
|
||||
|
||||
jwt_token = jwt.encode(
|
||||
{
|
||||
"user_id": user_id,
|
||||
@@ -8155,7 +8159,7 @@ async def auth_callback(request: Request):
|
||||
"login_method": "sso",
|
||||
"premium_user": premium_user,
|
||||
},
|
||||
litellm_master_key_hash,
|
||||
master_key,
|
||||
algorithm="HS256",
|
||||
)
|
||||
litellm_dashboard_ui += "?userID=" + user_id
|
||||
@@ -9179,3 +9183,4 @@ app.include_router(team_router)
|
||||
app.include_router(spend_management_router)
|
||||
app.include_router(caching_router)
|
||||
app.include_router(analytics_router)
|
||||
app.include_router(debugging_endpoints_router)
|
||||
|
||||
@@ -1199,7 +1199,7 @@ async def _get_spend_report_for_time_range(
|
||||
}
|
||||
},
|
||||
)
|
||||
async def calculate_spend(request: Request):
|
||||
async def calculate_spend(request: SpendCalculateRequest):
|
||||
"""
|
||||
Accepts all the params of completion_cost.
|
||||
|
||||
@@ -1248,14 +1248,93 @@ async def calculate_spend(request: Request):
|
||||
}'
|
||||
```
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
try:
|
||||
from litellm import completion_cost
|
||||
from litellm.cost_calculator import CostPerToken
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
data = await request.json()
|
||||
if "completion_response" in data:
|
||||
data["completion_response"] = litellm.ModelResponse(
|
||||
**data["completion_response"]
|
||||
_cost = None
|
||||
if request.model is not None:
|
||||
if request.messages is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Bad Request - messages must be provided if 'model' is provided",
|
||||
)
|
||||
|
||||
# check if model in llm_router
|
||||
_model_in_llm_router = None
|
||||
cost_per_token: Optional[CostPerToken] = None
|
||||
if llm_router is not None:
|
||||
if (
|
||||
llm_router.model_group_alias is not None
|
||||
and request.model in llm_router.model_group_alias
|
||||
):
|
||||
# lookup alias in llm_router
|
||||
_model_group_name = llm_router.model_group_alias[request.model]
|
||||
for model in llm_router.model_list:
|
||||
if model.get("model_name") == _model_group_name:
|
||||
_model_in_llm_router = model
|
||||
|
||||
else:
|
||||
# no model_group aliases set -> try finding model in llm_router
|
||||
# find model in llm_router
|
||||
for model in llm_router.model_list:
|
||||
if model.get("model_name") == request.model:
|
||||
_model_in_llm_router = model
|
||||
|
||||
"""
|
||||
3 cases for /spend/calculate
|
||||
|
||||
1. user passes model, and model is defined on litellm config.yaml or in DB. use info on config or in DB in this case
|
||||
2. user passes model, and model is not defined on litellm config.yaml or in DB. Pass model as is to litellm.completion_cost
|
||||
3. user passes completion_response
|
||||
|
||||
"""
|
||||
if _model_in_llm_router is not None:
|
||||
_litellm_params = _model_in_llm_router.get("litellm_params")
|
||||
_litellm_model_name = _litellm_params.get("model")
|
||||
input_cost_per_token = _litellm_params.get("input_cost_per_token")
|
||||
output_cost_per_token = _litellm_params.get("output_cost_per_token")
|
||||
if (
|
||||
input_cost_per_token is not None
|
||||
or output_cost_per_token is not None
|
||||
):
|
||||
cost_per_token = CostPerToken(
|
||||
input_cost_per_token=input_cost_per_token,
|
||||
output_cost_per_token=output_cost_per_token,
|
||||
)
|
||||
|
||||
_cost = completion_cost(
|
||||
model=_litellm_model_name,
|
||||
messages=request.messages,
|
||||
custom_cost_per_token=cost_per_token,
|
||||
)
|
||||
else:
|
||||
_cost = completion_cost(model=request.model, messages=request.messages)
|
||||
elif request.completion_response is not None:
|
||||
_completion_response = litellm.ModelResponse(**request.completion_response)
|
||||
_cost = completion_cost(completion_response=_completion_response)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Bad Request - Either 'model' or 'completion_response' must be provided",
|
||||
)
|
||||
return {"cost": _cost}
|
||||
except Exception as e:
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", str(e)),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
error_msg = f"{str(e)}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", 500),
|
||||
)
|
||||
return {"cost": completion_cost(**data)}
|
||||
|
||||
|
||||
@router.get(
|
||||
|
||||
+293
-183
@@ -404,6 +404,7 @@ class Router:
|
||||
litellm.failure_callback = [self.deployment_callback_on_failure]
|
||||
print( # noqa
|
||||
f"Intialized router with Routing strategy: {self.routing_strategy}\n\n"
|
||||
f"Routing enable_pre_call_checks: {self.enable_pre_call_checks}\n\n"
|
||||
f"Routing fallbacks: {self.fallbacks}\n\n"
|
||||
f"Routing content fallbacks: {self.content_policy_fallbacks}\n\n"
|
||||
f"Routing context window fallbacks: {self.context_window_fallbacks}\n\n"
|
||||
@@ -2116,6 +2117,12 @@ class Router:
|
||||
If it fails after num_retries, fall back to another model group
|
||||
"""
|
||||
mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None)
|
||||
mock_testing_context_fallbacks = kwargs.pop(
|
||||
"mock_testing_context_fallbacks", None
|
||||
)
|
||||
mock_testing_content_policy_fallbacks = kwargs.pop(
|
||||
"mock_testing_content_policy_fallbacks", None
|
||||
)
|
||||
model_group = kwargs.get("model")
|
||||
fallbacks = kwargs.get("fallbacks", self.fallbacks)
|
||||
context_window_fallbacks = kwargs.get(
|
||||
@@ -2129,6 +2136,26 @@ class Router:
|
||||
raise Exception(
|
||||
f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}"
|
||||
)
|
||||
elif (
|
||||
mock_testing_context_fallbacks is not None
|
||||
and mock_testing_context_fallbacks is True
|
||||
):
|
||||
raise litellm.ContextWindowExceededError(
|
||||
model=model_group,
|
||||
llm_provider="",
|
||||
message=f"This is a mock exception for model={model_group}, to trigger a fallback. \
|
||||
Context_Window_Fallbacks={context_window_fallbacks}",
|
||||
)
|
||||
elif (
|
||||
mock_testing_content_policy_fallbacks is not None
|
||||
and mock_testing_content_policy_fallbacks is True
|
||||
):
|
||||
raise litellm.ContentPolicyViolationError(
|
||||
model=model_group,
|
||||
llm_provider="",
|
||||
message=f"This is a mock exception for model={model_group}, to trigger a fallback. \
|
||||
Context_Policy_Fallbacks={content_policy_fallbacks}",
|
||||
)
|
||||
|
||||
response = await self.async_function_with_retries(*args, **kwargs)
|
||||
verbose_router_logger.debug(f"Async Response: {response}")
|
||||
@@ -2148,73 +2175,93 @@ class Router:
|
||||
)
|
||||
): # don't retry a malformed request
|
||||
raise e
|
||||
if (
|
||||
isinstance(e, litellm.ContextWindowExceededError)
|
||||
and context_window_fallbacks is not None
|
||||
):
|
||||
fallback_model_group = None
|
||||
for (
|
||||
item
|
||||
) in context_window_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
|
||||
if list(item.keys())[0] == model_group:
|
||||
fallback_model_group = item[model_group]
|
||||
break
|
||||
if isinstance(e, litellm.ContextWindowExceededError):
|
||||
if context_window_fallbacks is not None:
|
||||
fallback_model_group = None
|
||||
for (
|
||||
item
|
||||
) in context_window_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
|
||||
if list(item.keys())[0] == model_group:
|
||||
fallback_model_group = item[model_group]
|
||||
break
|
||||
|
||||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_retries(
|
||||
*args, **kwargs
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_retries(
|
||||
*args, **kwargs
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
pass
|
||||
else:
|
||||
error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
|
||||
model_group, context_window_fallbacks, fallbacks
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \
|
||||
to fallbacks, if available.{}".format(
|
||||
error_message
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
pass
|
||||
elif (
|
||||
isinstance(e, litellm.ContentPolicyViolationError)
|
||||
and content_policy_fallbacks is not None
|
||||
):
|
||||
fallback_model_group = None
|
||||
for (
|
||||
item
|
||||
) in content_policy_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
|
||||
if list(item.keys())[0] == model_group:
|
||||
fallback_model_group = item[model_group]
|
||||
break
|
||||
)
|
||||
|
||||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
e.message += "\n{}".format(error_message)
|
||||
elif isinstance(e, litellm.ContentPolicyViolationError):
|
||||
if content_policy_fallbacks is not None:
|
||||
fallback_model_group = None
|
||||
for (
|
||||
item
|
||||
) in content_policy_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
|
||||
if list(item.keys())[0] == model_group:
|
||||
fallback_model_group = item[model_group]
|
||||
break
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_retries(
|
||||
*args, **kwargs
|
||||
if fallback_model_group is None:
|
||||
raise original_exception
|
||||
|
||||
for mg in fallback_model_group:
|
||||
"""
|
||||
Iterate through the model groups and try calling that deployment
|
||||
"""
|
||||
try:
|
||||
kwargs["model"] = mg
|
||||
kwargs.setdefault("metadata", {}).update(
|
||||
{"model_group": mg}
|
||||
) # update model_group used, if fallbacks are done
|
||||
response = await self.async_function_with_retries(
|
||||
*args, **kwargs
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
pass
|
||||
else:
|
||||
error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
|
||||
model_group, content_policy_fallbacks, fallbacks
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
|
||||
to fallbacks, if available.{}".format(
|
||||
error_message
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
"Successful fallback b/w models."
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
pass
|
||||
elif fallbacks is not None:
|
||||
)
|
||||
|
||||
e.message += "\n{}".format(error_message)
|
||||
if fallbacks is not None:
|
||||
verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
|
||||
generic_fallback_idx: Optional[int] = None
|
||||
## check for specific model group-specific fallbacks
|
||||
@@ -2769,7 +2816,9 @@ class Router:
|
||||
|
||||
exception_response = getattr(exception, "response", {})
|
||||
exception_headers = getattr(exception_response, "headers", None)
|
||||
_time_to_cooldown = self.cooldown_time
|
||||
_time_to_cooldown = kwargs.get("litellm_params", {}).get(
|
||||
"cooldown_time", self.cooldown_time
|
||||
)
|
||||
|
||||
if exception_headers is not None:
|
||||
|
||||
@@ -3915,9 +3964,38 @@ class Router:
|
||||
raise Exception("Model invalid format - {}".format(type(model)))
|
||||
return None
|
||||
|
||||
def get_router_model_info(self, deployment: dict) -> ModelMapInfo:
|
||||
"""
|
||||
For a given model id, return the model info (max tokens, input cost, output cost, etc.).
|
||||
|
||||
Augment litellm info with additional params set in `model_info`.
|
||||
|
||||
Returns
|
||||
- ModelInfo - If found -> typed dict with max tokens, input cost, etc.
|
||||
"""
|
||||
## SET MODEL NAME
|
||||
base_model = deployment.get("model_info", {}).get("base_model", None)
|
||||
if base_model is None:
|
||||
base_model = deployment.get("litellm_params", {}).get("base_model", None)
|
||||
model = base_model or deployment.get("litellm_params", {}).get("model", None)
|
||||
|
||||
## GET LITELLM MODEL INFO
|
||||
model_info = litellm.get_model_info(model=model)
|
||||
|
||||
## CHECK USER SET MODEL INFO
|
||||
user_model_info = deployment.get("model_info", {})
|
||||
|
||||
model_info.update(user_model_info)
|
||||
|
||||
return model_info
|
||||
|
||||
def get_model_info(self, id: str) -> Optional[dict]:
|
||||
"""
|
||||
For a given model id, return the model info
|
||||
|
||||
Returns
|
||||
- dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
|
||||
- None: could not find deployment in list
|
||||
"""
|
||||
for model in self.model_list:
|
||||
if "model_info" in model and "id" in model["model_info"]:
|
||||
@@ -4307,6 +4385,7 @@ class Router:
|
||||
return _returned_deployments
|
||||
|
||||
_context_window_error = False
|
||||
_potential_error_str = ""
|
||||
_rate_limit_error = False
|
||||
|
||||
## get model group RPM ##
|
||||
@@ -4327,7 +4406,7 @@ class Router:
|
||||
model = base_model or deployment.get("litellm_params", {}).get(
|
||||
"model", None
|
||||
)
|
||||
model_info = litellm.get_model_info(model=model)
|
||||
model_info = self.get_router_model_info(deployment=deployment)
|
||||
|
||||
if (
|
||||
isinstance(model_info, dict)
|
||||
@@ -4339,6 +4418,11 @@ class Router:
|
||||
):
|
||||
invalid_model_indices.append(idx)
|
||||
_context_window_error = True
|
||||
_potential_error_str += (
|
||||
"Model={}, Max Input Tokens={}, Got={}".format(
|
||||
model, model_info["max_input_tokens"], input_tokens
|
||||
)
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
verbose_router_logger.debug("An error occurs - {}".format(str(e)))
|
||||
@@ -4438,15 +4522,13 @@ class Router:
|
||||
raise ValueError(
|
||||
f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. Try again in {self.cooldown_time} seconds."
|
||||
)
|
||||
elif _context_window_error == True:
|
||||
elif _context_window_error is True:
|
||||
raise litellm.ContextWindowExceededError(
|
||||
message="Context Window exceeded for given call",
|
||||
message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{}".format(
|
||||
_potential_error_str
|
||||
),
|
||||
model=model,
|
||||
llm_provider="",
|
||||
response=httpx.Response(
|
||||
status_code=400,
|
||||
request=httpx.Request("GET", "https://example.com"),
|
||||
),
|
||||
)
|
||||
if len(invalid_model_indices) > 0:
|
||||
for idx in reversed(invalid_model_indices):
|
||||
@@ -4558,127 +4640,155 @@ class Router:
|
||||
specific_deployment=specific_deployment,
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
model, healthy_deployments = self._common_checks_available_deployment(
|
||||
model=model,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
) # type: ignore
|
||||
|
||||
if isinstance(healthy_deployments, dict):
|
||||
return healthy_deployments
|
||||
|
||||
# filter out the deployments currently cooling down
|
||||
deployments_to_remove = []
|
||||
# cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]
|
||||
cooldown_deployments = await self._async_get_cooldown_deployments()
|
||||
verbose_router_logger.debug(
|
||||
f"async cooldown deployments: {cooldown_deployments}"
|
||||
)
|
||||
# Find deployments in model_list whose model_id is cooling down
|
||||
for deployment in healthy_deployments:
|
||||
deployment_id = deployment["model_info"]["id"]
|
||||
if deployment_id in cooldown_deployments:
|
||||
deployments_to_remove.append(deployment)
|
||||
# remove unhealthy deployments from healthy deployments
|
||||
for deployment in deployments_to_remove:
|
||||
healthy_deployments.remove(deployment)
|
||||
|
||||
# filter pre-call checks
|
||||
_allowed_model_region = (
|
||||
request_kwargs.get("allowed_model_region")
|
||||
if request_kwargs is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if self.enable_pre_call_checks and messages is not None:
|
||||
healthy_deployments = self._pre_call_checks(
|
||||
try:
|
||||
model, healthy_deployments = self._common_checks_available_deployment(
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=messages,
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
if len(healthy_deployments) == 0:
|
||||
if _allowed_model_region is None:
|
||||
_allowed_model_region = "n/a"
|
||||
raise ValueError(
|
||||
f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. pre-call-checks={self.enable_pre_call_checks}, allowed_model_region={_allowed_model_region}"
|
||||
)
|
||||
|
||||
if (
|
||||
self.routing_strategy == "usage-based-routing-v2"
|
||||
and self.lowesttpm_logger_v2 is not None
|
||||
):
|
||||
deployment = await self.lowesttpm_logger_v2.async_get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments, # type: ignore
|
||||
messages=messages,
|
||||
input=input,
|
||||
)
|
||||
if (
|
||||
self.routing_strategy == "cost-based-routing"
|
||||
and self.lowestcost_logger is not None
|
||||
):
|
||||
deployment = await self.lowestcost_logger.async_get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments, # type: ignore
|
||||
messages=messages,
|
||||
input=input,
|
||||
)
|
||||
elif self.routing_strategy == "simple-shuffle":
|
||||
# if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
|
||||
############## Check if we can do a RPM/TPM based weighted pick #################
|
||||
rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
|
||||
if rpm is not None:
|
||||
# use weight-random pick if rpms provided
|
||||
rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
|
||||
verbose_router_logger.debug(f"\nrpms {rpms}")
|
||||
total_rpm = sum(rpms)
|
||||
weights = [rpm / total_rpm for rpm in rpms]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(rpms)), weights=weights)[0]
|
||||
verbose_router_logger.debug(f"\n selected index, {selected_index}")
|
||||
deployment = healthy_deployments[selected_index]
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
|
||||
)
|
||||
return deployment or deployment[0]
|
||||
############## Check if we can do a RPM/TPM based weighted pick #################
|
||||
tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
|
||||
if tpm is not None:
|
||||
# use weight-random pick if rpms provided
|
||||
tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
|
||||
verbose_router_logger.debug(f"\ntpms {tpms}")
|
||||
total_tpm = sum(tpms)
|
||||
weights = [tpm / total_tpm for tpm in tpms]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(tpms)), weights=weights)[0]
|
||||
verbose_router_logger.debug(f"\n selected index, {selected_index}")
|
||||
deployment = healthy_deployments[selected_index]
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
|
||||
)
|
||||
return deployment or deployment[0]
|
||||
specific_deployment=specific_deployment,
|
||||
) # type: ignore
|
||||
|
||||
############## No RPM/TPM passed, we do a random pick #################
|
||||
item = random.choice(healthy_deployments)
|
||||
return item or item[0]
|
||||
if deployment is None:
|
||||
if isinstance(healthy_deployments, dict):
|
||||
return healthy_deployments
|
||||
|
||||
# filter out the deployments currently cooling down
|
||||
deployments_to_remove = []
|
||||
# cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]
|
||||
cooldown_deployments = await self._async_get_cooldown_deployments()
|
||||
verbose_router_logger.debug(
|
||||
f"async cooldown deployments: {cooldown_deployments}"
|
||||
)
|
||||
# Find deployments in model_list whose model_id is cooling down
|
||||
for deployment in healthy_deployments:
|
||||
deployment_id = deployment["model_info"]["id"]
|
||||
if deployment_id in cooldown_deployments:
|
||||
deployments_to_remove.append(deployment)
|
||||
# remove unhealthy deployments from healthy deployments
|
||||
for deployment in deployments_to_remove:
|
||||
healthy_deployments.remove(deployment)
|
||||
|
||||
# filter pre-call checks
|
||||
_allowed_model_region = (
|
||||
request_kwargs.get("allowed_model_region")
|
||||
if request_kwargs is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if self.enable_pre_call_checks and messages is not None:
|
||||
healthy_deployments = self._pre_call_checks(
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=messages,
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
if len(healthy_deployments) == 0:
|
||||
if _allowed_model_region is None:
|
||||
_allowed_model_region = "n/a"
|
||||
raise ValueError(
|
||||
f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. pre-call-checks={self.enable_pre_call_checks}, allowed_model_region={_allowed_model_region}"
|
||||
)
|
||||
|
||||
if (
|
||||
self.routing_strategy == "usage-based-routing-v2"
|
||||
and self.lowesttpm_logger_v2 is not None
|
||||
):
|
||||
deployment = (
|
||||
await self.lowesttpm_logger_v2.async_get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments, # type: ignore
|
||||
messages=messages,
|
||||
input=input,
|
||||
)
|
||||
)
|
||||
if (
|
||||
self.routing_strategy == "cost-based-routing"
|
||||
and self.lowestcost_logger is not None
|
||||
):
|
||||
deployment = (
|
||||
await self.lowestcost_logger.async_get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments, # type: ignore
|
||||
messages=messages,
|
||||
input=input,
|
||||
)
|
||||
)
|
||||
elif self.routing_strategy == "simple-shuffle":
|
||||
# if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
|
||||
############## Check if we can do a RPM/TPM based weighted pick #################
|
||||
rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
|
||||
if rpm is not None:
|
||||
# use weight-random pick if rpms provided
|
||||
rpms = [
|
||||
m["litellm_params"].get("rpm", 0) for m in healthy_deployments
|
||||
]
|
||||
verbose_router_logger.debug(f"\nrpms {rpms}")
|
||||
total_rpm = sum(rpms)
|
||||
weights = [rpm / total_rpm for rpm in rpms]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(rpms)), weights=weights)[
|
||||
0
|
||||
]
|
||||
verbose_router_logger.debug(f"\n selected index, {selected_index}")
|
||||
deployment = healthy_deployments[selected_index]
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
|
||||
)
|
||||
return deployment or deployment[0]
|
||||
############## Check if we can do a RPM/TPM based weighted pick #################
|
||||
tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
|
||||
if tpm is not None:
|
||||
# use weight-random pick if rpms provided
|
||||
tpms = [
|
||||
m["litellm_params"].get("tpm", 0) for m in healthy_deployments
|
||||
]
|
||||
verbose_router_logger.debug(f"\ntpms {tpms}")
|
||||
total_tpm = sum(tpms)
|
||||
weights = [tpm / total_tpm for tpm in tpms]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(tpms)), weights=weights)[
|
||||
0
|
||||
]
|
||||
verbose_router_logger.debug(f"\n selected index, {selected_index}")
|
||||
deployment = healthy_deployments[selected_index]
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
|
||||
)
|
||||
return deployment or deployment[0]
|
||||
|
||||
############## No RPM/TPM passed, we do a random pick #################
|
||||
item = random.choice(healthy_deployments)
|
||||
return item or item[0]
|
||||
if deployment is None:
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, No deployment available"
|
||||
)
|
||||
raise ValueError(
|
||||
f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}"
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, No deployment available"
|
||||
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}"
|
||||
)
|
||||
raise ValueError(
|
||||
f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}"
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}"
|
||||
)
|
||||
|
||||
return deployment
|
||||
return deployment
|
||||
except Exception as e:
|
||||
traceback_exception = traceback.format_exc()
|
||||
# if router rejects call -> log to langfuse/otel/etc.
|
||||
if request_kwargs is not None:
|
||||
logging_obj = request_kwargs.get("litellm_logging_obj", None)
|
||||
if logging_obj is not None:
|
||||
## LOGGING
|
||||
threading.Thread(
|
||||
target=logging_obj.failure_handler,
|
||||
args=(e, traceback_exception),
|
||||
).start() # log response
|
||||
# Handle any exceptions that might occur during streaming
|
||||
asyncio.create_task(
|
||||
logging_obj.async_failure_handler(e, traceback_exception) # type: ignore
|
||||
)
|
||||
raise e
|
||||
|
||||
def get_available_deployment(
|
||||
self,
|
||||
|
||||
@@ -696,6 +696,18 @@ async def test_gemini_pro_function_calling_httpx(provider, sync_mode):
|
||||
pytest.fail("An unexpected exception occurred - {}".format(str(e)))
|
||||
|
||||
|
||||
def vertex_httpx_mock_reject_prompt_post(*args, **kwargs):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"Content-Type": "application/json"}
|
||||
mock_response.json.return_value = {
|
||||
"promptFeedback": {"blockReason": "OTHER"},
|
||||
"usageMetadata": {"promptTokenCount": 6285, "totalTokenCount": 6285},
|
||||
}
|
||||
|
||||
return mock_response
|
||||
|
||||
|
||||
# @pytest.mark.skip(reason="exhausted vertex quota. need to refactor to mock the call")
|
||||
def vertex_httpx_mock_post(url, data=None, json=None, headers=None):
|
||||
mock_response = MagicMock()
|
||||
@@ -817,8 +829,11 @@ def vertex_httpx_mock_post(url, data=None, json=None, headers=None):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["vertex_ai_beta"]) # "vertex_ai",
|
||||
@pytest.mark.parametrize("content_filter_type", ["prompt", "response"]) # "vertex_ai",
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_pro_json_schema_httpx_content_policy_error(provider):
|
||||
async def test_gemini_pro_json_schema_httpx_content_policy_error(
|
||||
provider, content_filter_type
|
||||
):
|
||||
load_vertex_ai_credentials()
|
||||
litellm.set_verbose = True
|
||||
messages = [
|
||||
@@ -839,16 +854,20 @@ Using this JSON schema:
|
||||
|
||||
client = HTTPHandler()
|
||||
|
||||
with patch.object(client, "post", side_effect=vertex_httpx_mock_post) as mock_call:
|
||||
try:
|
||||
response = completion(
|
||||
model="vertex_ai_beta/gemini-1.5-flash",
|
||||
messages=messages,
|
||||
response_format={"type": "json_object"},
|
||||
client=client,
|
||||
)
|
||||
except litellm.ContentPolicyViolationError as e:
|
||||
pass
|
||||
if content_filter_type == "prompt":
|
||||
_side_effect = vertex_httpx_mock_reject_prompt_post
|
||||
else:
|
||||
_side_effect = vertex_httpx_mock_post
|
||||
|
||||
with patch.object(client, "post", side_effect=_side_effect) as mock_call:
|
||||
response = completion(
|
||||
model="vertex_ai_beta/gemini-1.5-flash",
|
||||
messages=messages,
|
||||
response_format={"type": "json_object"},
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response.choices[0].finish_reason == "content_filter"
|
||||
|
||||
mock_call.assert_called_once()
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.prompt_templates.factory import anthropic_messages_pt
|
||||
|
||||
# litellm.num_retries=3
|
||||
# litellm.num_retries = 3
|
||||
litellm.cache = None
|
||||
litellm.success_callback = []
|
||||
user_message = "Write a short poem about the sky"
|
||||
@@ -3470,6 +3470,28 @@ def test_completion_deep_infra_mistral():
|
||||
# test_completion_deep_infra_mistral()
|
||||
|
||||
|
||||
def test_completion_nvidia_nim():
|
||||
model_name = "nvidia_nim/databricks/dbrx-instruct"
|
||||
try:
|
||||
response = completion(
|
||||
model=model_name,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Boston today in Fahrenheit?",
|
||||
}
|
||||
],
|
||||
)
|
||||
# Add any assertions here to check the response
|
||||
print(response)
|
||||
assert response.choices[0].message.content is not None
|
||||
assert len(response.choices[0].message.content) > 0
|
||||
except litellm.exceptions.Timeout as e:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# Gemini tests
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
|
||||
@@ -58,3 +58,37 @@ async def test_async_mock_streaming_request():
|
||||
assert (
|
||||
complete_response == "LiteLLM is awesome"
|
||||
), f"Unexpected response got {complete_response}"
|
||||
|
||||
|
||||
def test_mock_request_n_greater_than_1():
|
||||
try:
|
||||
model = "gpt-3.5-turbo"
|
||||
messages = [{"role": "user", "content": "Hey, I'm a mock request"}]
|
||||
response = litellm.mock_completion(model=model, messages=messages, n=5)
|
||||
print("response: ", response)
|
||||
|
||||
assert len(response.choices) == 5
|
||||
for choice in response.choices:
|
||||
assert choice.message.content == "This is a mock request"
|
||||
|
||||
except:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_async_mock_streaming_request_n_greater_than_1():
|
||||
generator = await litellm.acompletion(
|
||||
messages=[{"role": "user", "content": "Why is LiteLLM amazing?"}],
|
||||
mock_response="LiteLLM is awesome",
|
||||
stream=True,
|
||||
model="gpt-3.5-turbo",
|
||||
n=5,
|
||||
)
|
||||
complete_response = ""
|
||||
async for chunk in generator:
|
||||
print(chunk)
|
||||
# complete_response += chunk["choices"][0]["delta"]["content"] or ""
|
||||
|
||||
# assert (
|
||||
# complete_response == "LiteLLM is awesome"
|
||||
# ), f"Unexpected response got {complete_response}"
|
||||
|
||||
@@ -732,7 +732,7 @@ def test_router_rpm_pre_call_check():
|
||||
pytest.fail(f"Got unexpected exception on router! - {str(e)}")
|
||||
|
||||
|
||||
def test_router_context_window_check_pre_call_check_in_group():
|
||||
def test_router_context_window_check_pre_call_check_in_group_custom_model_info():
|
||||
"""
|
||||
- Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k)
|
||||
- Send a 5k prompt
|
||||
@@ -755,6 +755,61 @@ def test_router_context_window_check_pre_call_check_in_group():
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_API_BASE"),
|
||||
"base_model": "azure/gpt-35-turbo",
|
||||
"mock_response": "Hello world 1!",
|
||||
},
|
||||
"model_info": {"max_input_tokens": 100},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo-1106",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"mock_response": "Hello world 2!",
|
||||
},
|
||||
"model_info": {"max_input_tokens": 0},
|
||||
},
|
||||
]
|
||||
|
||||
router = Router(model_list=model_list, set_verbose=True, enable_pre_call_checks=True, num_retries=0) # type: ignore
|
||||
|
||||
response = router.completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[
|
||||
{"role": "user", "content": "Who was Alexander?"},
|
||||
],
|
||||
)
|
||||
|
||||
print(f"response: {response}")
|
||||
|
||||
assert response.choices[0].message.content == "Hello world 1!"
|
||||
except Exception as e:
|
||||
pytest.fail(f"Got unexpected exception on router! - {str(e)}")
|
||||
|
||||
|
||||
def test_router_context_window_check_pre_call_check():
|
||||
"""
|
||||
- Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k)
|
||||
- Send a 5k prompt
|
||||
- Assert it works
|
||||
"""
|
||||
import os
|
||||
|
||||
from large_text import text
|
||||
|
||||
litellm.set_verbose = False
|
||||
|
||||
print(f"len(text): {len(text)}")
|
||||
try:
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/chatgpt-v-2",
|
||||
"api_key": os.getenv("AZURE_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_API_BASE"),
|
||||
"base_model": "azure/gpt-35-turbo",
|
||||
"mock_response": "Hello world 1!",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -762,6 +817,7 @@ def test_router_context_window_check_pre_call_check_in_group():
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo-1106",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"mock_response": "Hello world 2!",
|
||||
},
|
||||
},
|
||||
]
|
||||
@@ -777,6 +833,8 @@ def test_router_context_window_check_pre_call_check_in_group():
|
||||
)
|
||||
|
||||
print(f"response: {response}")
|
||||
|
||||
assert response.choices[0].message.content == "Hello world 2!"
|
||||
except Exception as e:
|
||||
pytest.fail(f"Got unexpected exception on router! - {str(e)}")
|
||||
|
||||
|
||||
@@ -1,18 +1,26 @@
|
||||
#### What this tests ####
|
||||
# This tests calling router with fallback models
|
||||
|
||||
import sys, os, time
|
||||
import traceback, asyncio
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
import openai, httpx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -62,3 +70,45 @@ async def test_cooldown_badrequest_error():
|
||||
assert response is not None
|
||||
|
||||
print(response)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_cooldowns():
|
||||
"""
|
||||
Assert kwargs for completion/embedding have 'cooldown_time' as a litellm_param
|
||||
"""
|
||||
# litellm.set_verbose = True
|
||||
tmp_mock = MagicMock()
|
||||
|
||||
litellm.failure_callback = [tmp_mock]
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-fake-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-1",
|
||||
"api_key": "my-key",
|
||||
"mock_response": Exception("this is an error"),
|
||||
},
|
||||
}
|
||||
],
|
||||
cooldown_time=60,
|
||||
)
|
||||
|
||||
try:
|
||||
_ = router.completion(
|
||||
model="my-fake-model",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
cooldown_time=0,
|
||||
num_retries=0,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tmp_mock.assert_called_once()
|
||||
|
||||
print(tmp_mock.call_count)
|
||||
|
||||
assert "cooldown_time" in tmp_mock.call_args[0][0]["litellm_params"]
|
||||
assert tmp_mock.call_args[0][0]["litellm_params"]["cooldown_time"] == 0
|
||||
|
||||
@@ -1129,7 +1129,9 @@ async def test_router_content_policy_fallbacks(
|
||||
mock_response = Exception("content filtering policy")
|
||||
else:
|
||||
mock_response = litellm.ModelResponse(
|
||||
choices=[litellm.Choices(finish_reason="content_filter")]
|
||||
choices=[litellm.Choices(finish_reason="content_filter")],
|
||||
model="gpt-3.5-turbo",
|
||||
usage=litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10),
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
# What is this?
|
||||
## This tests the llm guard integration
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import random
|
||||
|
||||
# What is this?
|
||||
## Unit test for presidio pii masking
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
import os
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router, mock_completion
|
||||
from litellm.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.enterprise.enterprise_hooks.secret_detection import (
|
||||
_ENTERPRISE_SecretDetection,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging, hash_token
|
||||
|
||||
### UNIT TESTS FOR OpenAI Moderation ###
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_secret_detection_chat():
|
||||
"""
|
||||
Tests to see if secret detection hook will mask api keys
|
||||
|
||||
|
||||
It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef'
|
||||
"""
|
||||
secret_instance = _ENTERPRISE_SecretDetection()
|
||||
_api_key = "sk-12345"
|
||||
_api_key = hash_token("sk-12345")
|
||||
user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
|
||||
local_cache = DualCache()
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
test_data = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Hello! I'm doing well. How can I assist you today?",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is my OPENAI_API_KEY = 'sk_1234567890abcdef'",
|
||||
},
|
||||
{"role": "user", "content": "i think it is +1 412-555-5555"},
|
||||
],
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
|
||||
await secret_instance.async_pre_call_hook(
|
||||
cache=local_cache,
|
||||
data=test_data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
call_type="completion",
|
||||
)
|
||||
print(
|
||||
"test data after running pre_call_hook: Expect all API Keys to be masked",
|
||||
test_data,
|
||||
)
|
||||
|
||||
assert test_data == {
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hey, how's it going, API_KEY = '[REDACTED]'"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Hello! I'm doing well. How can I assist you today?",
|
||||
},
|
||||
{"role": "user", "content": "this is my OPENAI_API_KEY = '[REDACTED]'"},
|
||||
{"role": "user", "content": "i think it is +1 412-555-5555"},
|
||||
],
|
||||
"model": "gpt-3.5-turbo",
|
||||
}, "Expect all API Keys to be masked"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_secret_detection_text_completion():
|
||||
"""
|
||||
Tests to see if secret detection hook will mask api keys
|
||||
|
||||
|
||||
It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef'
|
||||
"""
|
||||
secret_instance = _ENTERPRISE_SecretDetection()
|
||||
_api_key = "sk-12345"
|
||||
_api_key = hash_token("sk-12345")
|
||||
user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
|
||||
local_cache = DualCache()
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
test_data = {
|
||||
"prompt": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef', my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather",
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
|
||||
await secret_instance.async_pre_call_hook(
|
||||
cache=local_cache,
|
||||
data=test_data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
test_data == {
|
||||
"prompt": "Hey, how's it going, API_KEY = '[REDACTED]', my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather",
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
print(
|
||||
"test data after running pre_call_hook: Expect all API Keys to be masked",
|
||||
test_data,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_secret_detection_embeddings():
|
||||
"""
|
||||
Tests to see if secret detection hook will mask api keys
|
||||
|
||||
|
||||
It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef'
|
||||
"""
|
||||
secret_instance = _ENTERPRISE_SecretDetection()
|
||||
_api_key = "sk-12345"
|
||||
_api_key = hash_token("sk-12345")
|
||||
user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
|
||||
local_cache = DualCache()
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
test_data = {
|
||||
"input": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef', my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather",
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
|
||||
await secret_instance.async_pre_call_hook(
|
||||
cache=local_cache,
|
||||
data=test_data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
call_type="embedding",
|
||||
)
|
||||
|
||||
assert test_data == {
|
||||
"input": "Hey, how's it going, API_KEY = '[REDACTED]', my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather",
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
print(
|
||||
"test data after running pre_call_hook: Expect all API Keys to be masked",
|
||||
test_data,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_secret_detection_embeddings_list():
|
||||
"""
|
||||
Tests to see if secret detection hook will mask api keys
|
||||
|
||||
|
||||
It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef'
|
||||
"""
|
||||
secret_instance = _ENTERPRISE_SecretDetection()
|
||||
_api_key = "sk-12345"
|
||||
_api_key = hash_token("sk-12345")
|
||||
user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
|
||||
local_cache = DualCache()
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
test_data = {
|
||||
"input": [
|
||||
"hey",
|
||||
"how's it going, API_KEY = 'sk_1234567890abcdef'",
|
||||
"my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather",
|
||||
],
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
|
||||
await secret_instance.async_pre_call_hook(
|
||||
cache=local_cache,
|
||||
data=test_data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
call_type="embedding",
|
||||
)
|
||||
|
||||
print(
|
||||
"test data after running pre_call_hook: Expect all API Keys to be masked",
|
||||
test_data,
|
||||
)
|
||||
assert test_data == {
|
||||
"input": [
|
||||
"hey",
|
||||
"how's it going, API_KEY = '[REDACTED]'",
|
||||
"my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather",
|
||||
],
|
||||
"model": "gpt-3.5-turbo",
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import Request
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import SpendCalculateRequest
|
||||
from litellm.proxy.spend_tracking.spend_management_endpoints import calculate_spend
|
||||
from litellm.router import Router
|
||||
|
||||
# this file is to test litellm/proxy
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_calc_model_messages():
|
||||
cost_obj = await calculate_spend(
|
||||
request=SpendCalculateRequest(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
print("calculated cost", cost_obj)
|
||||
cost = cost_obj["cost"]
|
||||
assert cost > 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_calc_model_on_router_messages():
|
||||
from litellm.proxy.proxy_server import llm_router as init_llm_router
|
||||
|
||||
temp_llm_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "special-llama-model",
|
||||
"litellm_params": {
|
||||
"model": "groq/llama3-8b-8192",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "llm_router", temp_llm_router)
|
||||
|
||||
cost_obj = await calculate_spend(
|
||||
request=SpendCalculateRequest(
|
||||
model="special-llama-model",
|
||||
messages=[
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
print("calculated cost", cost_obj)
|
||||
_cost = cost_obj["cost"]
|
||||
|
||||
assert _cost > 0.0
|
||||
|
||||
# set router to init value
|
||||
setattr(litellm.proxy.proxy_server, "llm_router", init_llm_router)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_calc_using_response():
|
||||
cost_obj = await calculate_spend(
|
||||
request=SpendCalculateRequest(
|
||||
completion_response={
|
||||
"id": "chatcmpl-3bc7abcd-f70b-48ab-a16c-dfba0b286c86",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "Yooo! What's good?",
|
||||
"role": "assistant",
|
||||
},
|
||||
}
|
||||
],
|
||||
"created": "1677652288",
|
||||
"model": "groq/llama3-8b-8192",
|
||||
"object": "chat.completion",
|
||||
"system_fingerprint": "fp_873a560973",
|
||||
"usage": {
|
||||
"completion_tokens": 8,
|
||||
"prompt_tokens": 12,
|
||||
"total_tokens": 20,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
print("calculated cost", cost_obj)
|
||||
cost = cost_obj["cost"]
|
||||
assert cost > 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_calc_model_alias_on_router_messages():
|
||||
from litellm.proxy.proxy_server import llm_router as init_llm_router
|
||||
|
||||
temp_llm_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
},
|
||||
}
|
||||
],
|
||||
model_group_alias={
|
||||
"gpt4o": "gpt-4o",
|
||||
},
|
||||
)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "llm_router", temp_llm_router)
|
||||
|
||||
cost_obj = await calculate_spend(
|
||||
request=SpendCalculateRequest(
|
||||
model="gpt4o",
|
||||
messages=[
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
print("calculated cost", cost_obj)
|
||||
_cost = cost_obj["cost"]
|
||||
|
||||
assert _cost > 0.0
|
||||
|
||||
# set router to init value
|
||||
setattr(litellm.proxy.proxy_server, "llm_router", init_llm_router)
|
||||
@@ -227,9 +227,9 @@ class PromptFeedback(TypedDict):
|
||||
blockReasonMessage: str
|
||||
|
||||
|
||||
class UsageMetadata(TypedDict):
|
||||
promptTokenCount: int
|
||||
totalTokenCount: int
|
||||
class UsageMetadata(TypedDict, total=False):
|
||||
promptTokenCount: Required[int]
|
||||
totalTokenCount: Required[int]
|
||||
candidatesTokenCount: int
|
||||
|
||||
|
||||
|
||||
+88
-16
@@ -2017,6 +2017,7 @@ def get_litellm_params(
|
||||
input_cost_per_token=None,
|
||||
output_cost_per_token=None,
|
||||
output_cost_per_second=None,
|
||||
cooldown_time=None,
|
||||
):
|
||||
litellm_params = {
|
||||
"acompletion": acompletion,
|
||||
@@ -2039,6 +2040,7 @@ def get_litellm_params(
|
||||
"input_cost_per_second": input_cost_per_second,
|
||||
"output_cost_per_token": output_cost_per_token,
|
||||
"output_cost_per_second": output_cost_per_second,
|
||||
"cooldown_time": cooldown_time,
|
||||
}
|
||||
|
||||
return litellm_params
|
||||
@@ -2410,6 +2412,7 @@ def get_optional_params(
|
||||
and custom_llm_provider != "anyscale"
|
||||
and custom_llm_provider != "together_ai"
|
||||
and custom_llm_provider != "groq"
|
||||
and custom_llm_provider != "nvidia_nim"
|
||||
and custom_llm_provider != "deepseek"
|
||||
and custom_llm_provider != "codestral"
|
||||
and custom_llm_provider != "mistral"
|
||||
@@ -2608,7 +2611,15 @@ def get_optional_params(
|
||||
optional_params["top_p"] = top_p
|
||||
if stop is not None:
|
||||
optional_params["stop_sequences"] = stop
|
||||
elif custom_llm_provider == "huggingface" or custom_llm_provider == "predibase":
|
||||
elif custom_llm_provider == "predibase":
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
_check_valid_arg(supported_params=supported_params)
|
||||
optional_params = litellm.PredibaseConfig().map_openai_params(
|
||||
non_default_params=non_default_params, optional_params=optional_params
|
||||
)
|
||||
elif custom_llm_provider == "huggingface":
|
||||
## check if unsupported param passed in
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
@@ -3060,6 +3071,14 @@ def get_optional_params(
|
||||
optional_params = litellm.DatabricksConfig().map_openai_params(
|
||||
non_default_params=non_default_params, optional_params=optional_params
|
||||
)
|
||||
elif custom_llm_provider == "nvidia_nim":
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
_check_valid_arg(supported_params=supported_params)
|
||||
optional_params = litellm.NvidiaNimConfig().map_openai_params(
|
||||
non_default_params=non_default_params, optional_params=optional_params
|
||||
)
|
||||
elif custom_llm_provider == "groq":
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
@@ -3626,6 +3645,8 @@ def get_supported_openai_params(
|
||||
return litellm.OllamaChatConfig().get_supported_openai_params()
|
||||
elif custom_llm_provider == "anthropic":
|
||||
return litellm.AnthropicConfig().get_supported_openai_params()
|
||||
elif custom_llm_provider == "nvidia_nim":
|
||||
return litellm.NvidiaNimConfig().get_supported_openai_params()
|
||||
elif custom_llm_provider == "groq":
|
||||
return [
|
||||
"temperature",
|
||||
@@ -3986,6 +4007,10 @@ def get_llm_provider(
|
||||
# groq is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.groq.com/openai/v1
|
||||
api_base = "https://api.groq.com/openai/v1"
|
||||
dynamic_api_key = get_secret("GROQ_API_KEY")
|
||||
elif custom_llm_provider == "nvidia_nim":
|
||||
# nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
|
||||
api_base = "https://integrate.api.nvidia.com/v1"
|
||||
dynamic_api_key = get_secret("NVIDIA_NIM_API_KEY")
|
||||
elif custom_llm_provider == "codestral":
|
||||
# codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1
|
||||
api_base = "https://codestral.mistral.ai/v1"
|
||||
@@ -4087,6 +4112,9 @@ def get_llm_provider(
|
||||
elif endpoint == "api.groq.com/openai/v1":
|
||||
custom_llm_provider = "groq"
|
||||
dynamic_api_key = get_secret("GROQ_API_KEY")
|
||||
elif endpoint == "https://integrate.api.nvidia.com/v1":
|
||||
custom_llm_provider = "nvidia_nim"
|
||||
dynamic_api_key = get_secret("NVIDIA_NIM_API_KEY")
|
||||
elif endpoint == "https://codestral.mistral.ai/v1":
|
||||
custom_llm_provider = "codestral"
|
||||
dynamic_api_key = get_secret("CODESTRAL_API_KEY")
|
||||
@@ -4900,6 +4928,11 @@ def validate_environment(model: Optional[str] = None) -> dict:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("GROQ_API_KEY")
|
||||
elif custom_llm_provider == "nvidia_nim":
|
||||
if "NVIDIA_NIM_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("NVIDIA_NIM_API_KEY")
|
||||
elif (
|
||||
custom_llm_provider == "codestral"
|
||||
or custom_llm_provider == "text-completion-codestral"
|
||||
@@ -5914,6 +5947,7 @@ def exception_type(
|
||||
)
|
||||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
# exception_mapping_worked = True
|
||||
raise APIConnectionError(
|
||||
message=f"APIConnectionError: {exception_provider} - {message}",
|
||||
llm_provider=custom_llm_provider,
|
||||
@@ -6067,6 +6101,14 @@ def exception_type(
|
||||
model=model,
|
||||
llm_provider="replicate",
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
exception_mapping_worked = True
|
||||
raise UnprocessableEntityError(
|
||||
message=f"ReplicateException - {original_exception.message}",
|
||||
llm_provider="replicate",
|
||||
model=model,
|
||||
response=original_exception.response,
|
||||
)
|
||||
elif original_exception.status_code == 429:
|
||||
exception_mapping_worked = True
|
||||
raise RateLimitError(
|
||||
@@ -6125,13 +6167,6 @@ def exception_type(
|
||||
response=original_exception.response,
|
||||
litellm_debug_info=extra_information,
|
||||
)
|
||||
if "Request failed during generation" in error_str:
|
||||
# this is an internal server error from predibase
|
||||
raise litellm.InternalServerError(
|
||||
message=f"PredibaseException - {error_str}",
|
||||
llm_provider="predibase",
|
||||
model=model,
|
||||
)
|
||||
elif hasattr(original_exception, "status_code"):
|
||||
if original_exception.status_code == 500:
|
||||
exception_mapping_worked = True
|
||||
@@ -6169,7 +6204,10 @@ def exception_type(
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
elif (
|
||||
original_exception.status_code == 422
|
||||
or original_exception.status_code == 424
|
||||
):
|
||||
exception_mapping_worked = True
|
||||
raise BadRequestError(
|
||||
message=f"PredibaseException - {original_exception.message}",
|
||||
@@ -6438,7 +6476,11 @@ def exception_type(
|
||||
),
|
||||
litellm_debug_info=extra_information,
|
||||
)
|
||||
elif "The response was blocked." in error_str:
|
||||
elif (
|
||||
"The response was blocked." in error_str
|
||||
or "Output blocked by content filtering policy"
|
||||
in error_str # anthropic on vertex ai
|
||||
):
|
||||
exception_mapping_worked = True
|
||||
raise ContentPolicyViolationError(
|
||||
message=f"VertexAIException ContentPolicyViolationError - {error_str}",
|
||||
@@ -7460,6 +7502,9 @@ def exception_type(
|
||||
if exception_mapping_worked:
|
||||
raise e
|
||||
else:
|
||||
for error_type in litellm.LITELLM_EXCEPTION_TYPES:
|
||||
if isinstance(e, error_type):
|
||||
raise e # it's already mapped
|
||||
raise APIConnectionError(
|
||||
message="{}\n{}".format(original_exception, traceback.format_exc()),
|
||||
llm_provider="",
|
||||
@@ -9696,18 +9741,45 @@ class TextCompletionStreamWrapper:
|
||||
raise StopAsyncIteration
|
||||
|
||||
|
||||
def mock_completion_streaming_obj(model_response, mock_response, model):
|
||||
def mock_completion_streaming_obj(
|
||||
model_response, mock_response, model, n: Optional[int] = None
|
||||
):
|
||||
for i in range(0, len(mock_response), 3):
|
||||
completion_obj = {"role": "assistant", "content": mock_response[i : i + 3]}
|
||||
model_response.choices[0].delta = completion_obj
|
||||
completion_obj = Delta(role="assistant", content=mock_response[i : i + 3])
|
||||
if n is None:
|
||||
model_response.choices[0].delta = completion_obj
|
||||
else:
|
||||
_all_choices = []
|
||||
for j in range(n):
|
||||
_streaming_choice = litellm.utils.StreamingChoices(
|
||||
index=j,
|
||||
delta=litellm.utils.Delta(
|
||||
role="assistant", content=mock_response[i : i + 3]
|
||||
),
|
||||
)
|
||||
_all_choices.append(_streaming_choice)
|
||||
model_response.choices = _all_choices
|
||||
yield model_response
|
||||
|
||||
|
||||
async def async_mock_completion_streaming_obj(model_response, mock_response, model):
|
||||
async def async_mock_completion_streaming_obj(
|
||||
model_response, mock_response, model, n: Optional[int] = None
|
||||
):
|
||||
for i in range(0, len(mock_response), 3):
|
||||
completion_obj = Delta(role="assistant", content=mock_response[i : i + 3])
|
||||
model_response.choices[0].delta = completion_obj
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
if n is None:
|
||||
model_response.choices[0].delta = completion_obj
|
||||
else:
|
||||
_all_choices = []
|
||||
for j in range(n):
|
||||
_streaming_choice = litellm.utils.StreamingChoices(
|
||||
index=j,
|
||||
delta=litellm.utils.Delta(
|
||||
role="assistant", content=mock_response[i : i + 3]
|
||||
),
|
||||
)
|
||||
_all_choices.append(_streaming_choice)
|
||||
model_response.choices = _all_choices
|
||||
yield model_response
|
||||
|
||||
|
||||
|
||||
@@ -887,7 +887,7 @@
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00000005,
|
||||
"output_cost_per_token": 0.00000010,
|
||||
"output_cost_per_token": 0.00000008,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
@@ -906,8 +906,8 @@
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 0.00000027,
|
||||
"output_cost_per_token": 0.00000027,
|
||||
"input_cost_per_token": 0.00000024,
|
||||
"output_cost_per_token": 0.00000024,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
@@ -916,8 +916,8 @@
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00000010,
|
||||
"output_cost_per_token": 0.00000010,
|
||||
"input_cost_per_token": 0.00000007,
|
||||
"output_cost_per_token": 0.00000007,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true
|
||||
@@ -2073,6 +2073,30 @@
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/anthropic/claude-3-haiku-20240307": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.00000025,
|
||||
"output_cost_per_token": 0.00000125,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 264
|
||||
},
|
||||
"openrouter/anthropic/claude-3.5-sonnet": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
"input_cost_per_token": 0.000003,
|
||||
"output_cost_per_token": 0.000015,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 159
|
||||
},
|
||||
"openrouter/anthropic/claude-3-sonnet": {
|
||||
"max_tokens": 200000,
|
||||
"input_cost_per_token": 0.000003,
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "litellm"
|
||||
version = "1.40.25"
|
||||
version = "1.40.27"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
authors = ["BerriAI"]
|
||||
license = "MIT"
|
||||
@@ -90,7 +90,7 @@ requires = ["poetry-core", "wheel"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.40.25"
|
||||
version = "1.40.27"
|
||||
version_files = [
|
||||
"pyproject.toml:^version"
|
||||
]
|
||||
|
||||
@@ -31,6 +31,8 @@ azure-identity==1.16.1 # for azure content safety
|
||||
opentelemetry-api==1.25.0
|
||||
opentelemetry-sdk==1.25.0
|
||||
opentelemetry-exporter-otlp==1.25.0
|
||||
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
|
||||
cryptography==42.0.7
|
||||
|
||||
### LITELLM PACKAGE DEPENDENCIES
|
||||
python-dotenv==1.0.0 # for env
|
||||
|
||||
Reference in New Issue
Block a user