diff --git a/.circleci/config.yml b/.circleci/config.yml index fd1b48a9c6..5dfeedcaa2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,7 +48,8 @@ jobs: pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 pip install openai - pip install prisma + pip install prisma + pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" pip install fastapi pip install "gunicorn==21.2.0" diff --git a/.gitignore b/.gitignore index b633e1d3d8..8a9095b840 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ litellm/proxy/_experimental/out/model_hub/index.html litellm/proxy/_experimental/out/onboarding/index.html litellm/tests/log.txt litellm/tests/langfuse.log +litellm/tests/langfuse.log diff --git a/README.md b/README.md index 91b709442b..6d26e92c2b 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,8 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature # Usage ([**Docs**](https://docs.litellm.ai/docs/)) > [!IMPORTANT] -> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration) +> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration) +> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required. Open In Colab diff --git a/docs/my-website/docs/completion/function_call.md b/docs/my-website/docs/completion/function_call.md index 5daccf7232..514e8cda1a 100644 --- a/docs/my-website/docs/completion/function_call.md +++ b/docs/my-website/docs/completion/function_call.md @@ -502,10 +502,10 @@ response = completion(model="gpt-3.5-turbo-0613", messages=messages, functions=f print(response) ``` -## Function calling for Non-OpenAI LLMs +## Function calling for Models w/out function-calling support ### Adding Function to prompt -For Non OpenAI LLMs LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True` +For Models/providers without function calling support, LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True` #### Usage ```python diff --git a/docs/my-website/docs/completion/reliable_completions.md b/docs/my-website/docs/completion/reliable_completions.md index 2656f9a4fb..94102e1944 100644 --- a/docs/my-website/docs/completion/reliable_completions.md +++ b/docs/my-website/docs/completion/reliable_completions.md @@ -31,9 +31,15 @@ response = completion( ) ``` -## Fallbacks +## Fallbacks (SDK) -### Context Window Fallbacks +:::info + +[See how to do on PROXY](../proxy/reliability.md) + +::: + +### Context Window Fallbacks (SDK) ```python from litellm import completion @@ -43,7 +49,7 @@ messages = [{"content": "how does a court case get to the Supreme Court?" * 500, completion(model="gpt-3.5-turbo", messages=messages, context_window_fallback_dict=ctx_window_fallback_dict) ``` -### Fallbacks - Switch Models/API Keys/API Bases +### Fallbacks - Switch Models/API Keys/API Bases (SDK) LLM APIs can be unstable, completion() with fallbacks ensures you'll always get a response from your calls @@ -69,7 +75,7 @@ response = completion(model="azure/gpt-4", messages=messages, api_key=api_key, [Check out this section for implementation details](#fallbacks-1) -## Implementation Details +## Implementation Details (SDK) ### Fallbacks #### Output from calls diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 0edf937ed3..875aec57f0 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -12,7 +12,9 @@ This covers: - ✅ [**Secure UI access with Single Sign-On**](../docs/proxy/ui.md#setup-ssoauth-for-ui) - ✅ [**Audit Logs with retention policy**](../docs/proxy/enterprise.md#audit-logs) - ✅ [**JWT-Auth**](../docs/proxy/token_auth.md) -- ✅ [**Prompt Injection Detection**](#prompt-injection-detection-lakeraai) +- ✅ [**Control available public, private routes**](../docs/proxy/enterprise.md#control-available-public-private-routes) +- ✅ [**Guardrails, Content Moderation, PII Masking, Secret/API Key Masking**](../docs/proxy/enterprise.md#prompt-injection-detection---lakeraai) +- ✅ [**Prompt Injection Detection**](../docs/proxy/enterprise.md#prompt-injection-detection---lakeraai) - ✅ [**Invite Team Members to access `/spend` Routes**](../docs/proxy/cost_tracking#allowing-non-proxy-admins-to-access-spend-endpoints) - ✅ **Feature Prioritization** - ✅ **Custom Integrations** diff --git a/docs/my-website/docs/observability/telemetry.md b/docs/my-website/docs/observability/telemetry.md index 78267b9c56..2322955662 100644 --- a/docs/my-website/docs/observability/telemetry.md +++ b/docs/my-website/docs/observability/telemetry.md @@ -1,13 +1,8 @@ # Telemetry -LiteLLM contains a telemetry feature that tells us what models are used, and what errors are hit. +There is no Telemetry on LiteLLM - no data is stored by us ## What is logged? -Only the model name and exception raised is logged. +NOTHING - no data is sent to LiteLLM Servers -## Why? -We use this information to help us understand how LiteLLM is used, and improve stability. - -## Opting out -If you prefer to opt out of telemetry, you can do this by setting `litellm.telemetry = False`. \ No newline at end of file diff --git a/docs/my-website/docs/providers/nvidia_nim.md b/docs/my-website/docs/providers/nvidia_nim.md new file mode 100644 index 0000000000..7f895aa337 --- /dev/null +++ b/docs/my-website/docs/providers/nvidia_nim.md @@ -0,0 +1,103 @@ +# Nvidia NIM +https://docs.api.nvidia.com/nim/reference/ + +:::tip + +**We support ALL Nvidia NIM models, just set `model=nvidia_nim/` as a prefix when sending litellm requests** + +::: + +## API Key +```python +# env variable +os.environ['NVIDIA_NIM_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['NVIDIA_NIM_API_KEY'] = "" +response = completion( + model="nvidia_nim/meta/llama3-70b-instruct", + messages=[ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ], + temperature=0.2, # optional + top_p=0.9, # optional + frequency_penalty=0.1, # optional + presence_penalty=0.1, # optional + max_tokens=10, # optional + stop=["\n\n"], # optional +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['NVIDIA_NIM_API_KEY'] = "" +response = completion( + model="nvidia_nim/meta/llama3-70b-instruct", + messages=[ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ], + stream=True, + temperature=0.2, # optional + top_p=0.9, # optional + frequency_penalty=0.1, # optional + presence_penalty=0.1, # optional + max_tokens=10, # optional + stop=["\n\n"], # optional +) + +for chunk in response: + print(chunk) +``` + + +## Supported Models - 💥 ALL Nvidia NIM Models Supported! +We support ALL `nvidia_nim` models, just set `nvidia_nim/` as a prefix when sending completion requests + +| Model Name | Function Call | +|------------|---------------| +| nvidia/nemotron-4-340b-reward | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-reward", messages)` | +| 01-ai/yi-large | `completion(model="nvidia_nim/01-ai/yi-large", messages)` | +| aisingapore/sea-lion-7b-instruct | `completion(model="nvidia_nim/aisingapore/sea-lion-7b-instruct", messages)` | +| databricks/dbrx-instruct | `completion(model="nvidia_nim/databricks/dbrx-instruct", messages)` | +| google/gemma-7b | `completion(model="nvidia_nim/google/gemma-7b", messages)` | +| google/gemma-2b | `completion(model="nvidia_nim/google/gemma-2b", messages)` | +| google/codegemma-1.1-7b | `completion(model="nvidia_nim/google/codegemma-1.1-7b", messages)` | +| google/codegemma-7b | `completion(model="nvidia_nim/google/codegemma-7b", messages)` | +| google/recurrentgemma-2b | `completion(model="nvidia_nim/google/recurrentgemma-2b", messages)` | +| ibm/granite-34b-code-instruct | `completion(model="nvidia_nim/ibm/granite-34b-code-instruct", messages)` | +| ibm/granite-8b-code-instruct | `completion(model="nvidia_nim/ibm/granite-8b-code-instruct", messages)` | +| mediatek/breeze-7b-instruct | `completion(model="nvidia_nim/mediatek/breeze-7b-instruct", messages)` | +| meta/codellama-70b | `completion(model="nvidia_nim/meta/codellama-70b", messages)` | +| meta/llama2-70b | `completion(model="nvidia_nim/meta/llama2-70b", messages)` | +| meta/llama3-8b | `completion(model="nvidia_nim/meta/llama3-8b", messages)` | +| meta/llama3-70b | `completion(model="nvidia_nim/meta/llama3-70b", messages)` | +| microsoft/phi-3-medium-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-medium-4k-instruct", messages)` | +| microsoft/phi-3-mini-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-128k-instruct", messages)` | +| microsoft/phi-3-mini-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-4k-instruct", messages)` | +| microsoft/phi-3-small-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-128k-instruct", messages)` | +| microsoft/phi-3-small-8k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-8k-instruct", messages)` | +| mistralai/codestral-22b-instruct-v0.1 | `completion(model="nvidia_nim/mistralai/codestral-22b-instruct-v0.1", messages)` | +| mistralai/mistral-7b-instruct | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct", messages)` | +| mistralai/mistral-7b-instruct-v0.3 | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct-v0.3", messages)` | +| mistralai/mixtral-8x7b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x7b-instruct", messages)` | +| mistralai/mixtral-8x22b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x22b-instruct", messages)` | +| mistralai/mistral-large | `completion(model="nvidia_nim/mistralai/mistral-large", messages)` | +| nvidia/nemotron-4-340b-instruct | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-instruct", messages)` | +| seallms/seallm-7b-v2.5 | `completion(model="nvidia_nim/seallms/seallm-7b-v2.5", messages)` | +| snowflake/arctic | `completion(model="nvidia_nim/snowflake/arctic", messages)` | +| upstage/solar-10.7b-instruct | `completion(model="nvidia_nim/upstage/solar-10.7b-instruct", messages)` | \ No newline at end of file diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index e657d3b73e..9fff879e54 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -14,10 +14,11 @@ Features: - ✅ [SSO for Admin UI](./ui.md#✨-enterprise-features) - ✅ [Audit Logs](#audit-logs) - ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags) -- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests) -- ✅ [Content Moderation with LLM Guard, LlamaGuard, Google Text Moderations](#content-moderation) +- ✅ [Control available public, private routes](#control-available-public-private-routes) +- ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](#content-moderation) - ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai) - ✅ [Custom Branding + Routes on Swagger Docs](#swagger-docs---custom-routes--branding) +- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests) - ✅ Reject calls from Blocked User list - ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors) @@ -448,11 +449,144 @@ Expected Response +## Control available public, private routes + +:::info + +❓ Use this when you want to make an existing private route -> public + +Example - Make `/spend/calculate` a publicly available route (by default `/spend/calculate` on LiteLLM Proxy requires authentication) + +::: + +#### Usage - Define public routes + +**Step 1** - set allowed public routes on config.yaml + +`LiteLLMRoutes.public_routes` is an ENUM corresponding to the default public routes on LiteLLM. [You can see this here](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/_types.py) + +```yaml +general_settings: + master_key: sk-1234 + public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] +``` + +**Step 2** - start proxy + +```shell +litellm --config config.yaml +``` + +**Step 3** - Test it + +```shell +curl --request POST \ + --url 'http://localhost:4000/spend/calculate' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hey, how'\''s it going?"}] + }' +``` + +🎉 Expect this endpoint to work without an `Authorization / Bearer Token` + ## Content Moderation -#### Content Moderation with LLM Guard +### Content Moderation - Secret Detection +❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM. + +Example if you want to redact the value of `OPENAI_API_KEY` in the following request + +#### Incoming Request + +```json +{ + "messages": [ + { + "role": "user", + "content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'", + } + ] +} +``` + +#### Request after Moderation + +```json +{ + "messages": [ + { + "role": "user", + "content": "Hey, how's it going, API_KEY = '[REDACTED]'", + } + ] +} +``` + +**Usage** + +**Step 1** Add this to your config.yaml + +```yaml +litellm_settings: + callbacks: ["hide_secrets"] +``` + +**Step 2** Run litellm proxy with `--detailed_debug` to see the server logs + +``` +litellm --config config.yaml --detailed_debug +``` + +**Step 3** Test it with request + +Send this request +```shell +curl --location 'http://localhost:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "llama3", + "messages": [ + { + "role": "user", + "content": "what is the value of my open ai key? openai_api_key=sk-1234998222" + } + ] +}' +``` + + +Expect to see the following warning on your litellm server logs + +```shell +LiteLLM Proxy:WARNING: secret_detection.py:88 - Detected and redacted secrets in message: ['Secret Keyword'] +``` + + +You can also see the raw request sent from litellm to the API Provider +```json +POST Request Sent from LiteLLM: +curl -X POST \ +https://api.groq.com/openai/v1/ \ +-H 'Authorization: Bearer gsk_mySVchjY********************************************' \ +-d { + "model": "llama3-8b-8192", + "messages": [ + { + "role": "user", + "content": "what is the time today, openai_api_key=[REDACTED]" + } + ], + "stream": false, + "extra_body": {} +} +``` + +### Content Moderation with LLM Guard Set the LLM Guard API Base in your environment @@ -587,7 +721,7 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \ -#### Content Moderation with LlamaGuard +### Content Moderation with LlamaGuard Currently works with Sagemaker's LlamaGuard endpoint. @@ -621,7 +755,7 @@ callbacks: ["llamaguard_moderations"] -#### Content Moderation with Google Text Moderation +### Content Moderation with Google Text Moderation Requires your GOOGLE_APPLICATION_CREDENTIALS to be set in your .env (same as VertexAI). diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index a2d24da69b..9228071b0d 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -272,6 +272,7 @@ litellm_settings: fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo"]}] # fallback to gpt-3.5-turbo if call fails num_retries context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. + cooldown_time: 30 # how long to cooldown model if fails/min > allowed_fails ``` ### Context Window Fallbacks (Pre-Call Checks + Fallbacks) @@ -431,6 +432,67 @@ litellm_settings: content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] ``` + + +### Test Fallbacks! + +Check if your fallbacks are working as expected. + +#### **Regular Fallbacks** +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "my-bad-model", + "messages": [ + { + "role": "user", + "content": "ping" + } + ], + "mock_testing_fallbacks": true # 👈 KEY CHANGE +} +' +``` + +#### **Content Policy Fallbacks** +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "my-bad-model", + "messages": [ + { + "role": "user", + "content": "ping" + } + ], + "mock_testing_content_policy_fallbacks": true # 👈 KEY CHANGE +} +' +``` + +#### **Context Window Fallbacks** + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "my-bad-model", + "messages": [ + { + "role": "user", + "content": "ping" + } + ], + "mock_testing_context_window_fallbacks": true # 👈 KEY CHANGE +} +' +``` + ### EU-Region Filtering (Pre-Call Checks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index fd4fb86588..240e6c8e04 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -762,6 +762,9 @@ asyncio.run(router_acompletion()) Set the limit for how many calls a model is allowed to fail in a minute, before being cooled down for a minute. + + + ```python from litellm import Router @@ -779,9 +782,39 @@ messages = [{"content": user_message, "role": "user"}] response = router.completion(model="gpt-3.5-turbo", messages=messages) print(f"response: {response}") - ``` + + + +**Set Global Value** + +```yaml +router_settings: + allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. + cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails +``` + +Defaults: +- allowed_fails: 0 +- cooldown_time: 60s + +**Set Per Model** + +```yaml +model_list: +- model_name: fake-openai-endpoint + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + tenant_id: os.environ/PREDIBASE_TENANT_ID + max_new_tokens: 256 + cooldown_time: 0 # 👈 KEY CHANGE +``` + + + + ### Retries For both async + sync functions, we support retrying failed requests. @@ -901,6 +934,39 @@ response = await router.acompletion( If a call fails after num_retries, fall back to another model group. +### Quick Start + +```python +from litellm import Router +router = Router( + model_list=[ + { # bad model + "model_name": "bad-model", + "litellm_params": { + "model": "openai/my-bad-model", + "api_key": "my-bad-api-key", + "mock_response": "Bad call" + }, + }, + { # good model + "model_name": "my-good-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": os.getenv("OPENAI_API_KEY"), + "mock_response": "Good call" + }, + }, + ], + fallbacks=[{"bad-model": ["my-good-model"]}] # 👈 KEY CHANGE +) + +response = router.completion( + model="bad-model", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + mock_testing_fallbacks=True, +) +``` + If the error is a context window exceeded error, fall back to a larger model group (if given). Fallbacks are done in-order - ["gpt-3.5-turbo, "gpt-4", "gpt-4-32k"], will do 'gpt-3.5-turbo' first, then 'gpt-4', etc. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 2673933f4c..9835a260b3 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -146,13 +146,14 @@ const sidebars = { "providers/databricks", "providers/watsonx", "providers/predibase", - "providers/clarifai", + "providers/nvidia_nim", "providers/triton-inference-server", "providers/ollama", "providers/perplexity", "providers/groq", "providers/deepseek", - "providers/fireworks_ai", + "providers/fireworks_ai", + "providers/clarifai", "providers/vllm", "providers/xinference", "providers/cloudflare_workers", diff --git a/enterprise/enterprise_hooks/secret_detection.py b/enterprise/enterprise_hooks/secret_detection.py new file mode 100644 index 0000000000..ded9f27c17 --- /dev/null +++ b/enterprise/enterprise_hooks/secret_detection.py @@ -0,0 +1,145 @@ +# +-------------------------------------------------------------+ +# +# Use SecretDetection /moderations for your LLM calls +# +# +-------------------------------------------------------------+ +# Thank you users! We ❤️ you! - Krrish & Ishaan + +import sys, os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +from typing import Optional, Literal, Union +import litellm, traceback, sys, uuid +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.integrations.custom_logger import CustomLogger +from fastapi import HTTPException +from litellm._logging import verbose_proxy_logger +from litellm.utils import ( + ModelResponse, + EmbeddingResponse, + ImageResponse, + StreamingChoices, +) +from datetime import datetime +import aiohttp, asyncio +from litellm._logging import verbose_proxy_logger +import tempfile +from litellm._logging import verbose_proxy_logger + + +litellm.set_verbose = True + + +class _ENTERPRISE_SecretDetection(CustomLogger): + def __init__(self): + pass + + def scan_message_for_secrets(self, message_content: str): + from detect_secrets import SecretsCollection + from detect_secrets.settings import default_settings + + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.write(message_content.encode("utf-8")) + temp_file.close() + + secrets = SecretsCollection() + with default_settings(): + secrets.scan_file(temp_file.name) + + os.remove(temp_file.name) + + detected_secrets = [] + for file in secrets.files: + for found_secret in secrets[file]: + if found_secret.secret_value is None: + continue + detected_secrets.append( + {"type": found_secret.type, "value": found_secret.secret_value} + ) + + return detected_secrets + + #### CALL HOOKS - proxy only #### + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: str, # "completion", "embeddings", "image_generation", "moderation" + ): + from detect_secrets import SecretsCollection + from detect_secrets.settings import default_settings + + if "messages" in data and isinstance(data["messages"], list): + for message in data["messages"]: + if "content" in message and isinstance(message["content"], str): + detected_secrets = self.scan_message_for_secrets(message["content"]) + + for secret in detected_secrets: + message["content"] = message["content"].replace( + secret["value"], "[REDACTED]" + ) + + if len(detected_secrets) > 0: + secret_types = [secret["type"] for secret in detected_secrets] + verbose_proxy_logger.warning( + f"Detected and redacted secrets in message: {secret_types}" + ) + + if "prompt" in data: + if isinstance(data["prompt"], str): + detected_secrets = self.scan_message_for_secrets(data["prompt"]) + for secret in detected_secrets: + data["prompt"] = data["prompt"].replace( + secret["value"], "[REDACTED]" + ) + if len(detected_secrets) > 0: + secret_types = [secret["type"] for secret in detected_secrets] + verbose_proxy_logger.warning( + f"Detected and redacted secrets in prompt: {secret_types}" + ) + elif isinstance(data["prompt"], list): + for item in data["prompt"]: + if isinstance(item, str): + detected_secrets = self.scan_message_for_secrets(item) + for secret in detected_secrets: + item = item.replace(secret["value"], "[REDACTED]") + if len(detected_secrets) > 0: + secret_types = [ + secret["type"] for secret in detected_secrets + ] + verbose_proxy_logger.warning( + f"Detected and redacted secrets in prompt: {secret_types}" + ) + + if "input" in data: + if isinstance(data["input"], str): + detected_secrets = self.scan_message_for_secrets(data["input"]) + for secret in detected_secrets: + data["input"] = data["input"].replace(secret["value"], "[REDACTED]") + if len(detected_secrets) > 0: + secret_types = [secret["type"] for secret in detected_secrets] + verbose_proxy_logger.warning( + f"Detected and redacted secrets in input: {secret_types}" + ) + elif isinstance(data["input"], list): + _input_in_request = data["input"] + for idx, item in enumerate(_input_in_request): + if isinstance(item, str): + detected_secrets = self.scan_message_for_secrets(item) + for secret in detected_secrets: + _input_in_request[idx] = item.replace( + secret["value"], "[REDACTED]" + ) + if len(detected_secrets) > 0: + secret_types = [ + secret["type"] for secret in detected_secrets + ] + verbose_proxy_logger.warning( + f"Detected and redacted secrets in input: {secret_types}" + ) + verbose_proxy_logger.debug("Data after redacting input %s", data) + return diff --git a/litellm/__init__.py b/litellm/__init__.py index f07ce88092..08ee84aaad 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -401,6 +401,7 @@ openai_compatible_endpoints: List = [ "codestral.mistral.ai/v1/chat/completions", "codestral.mistral.ai/v1/fim/completions", "api.groq.com/openai/v1", + "https://integrate.api.nvidia.com/v1", "api.deepseek.com/v1", "api.together.xyz/v1", "inference.friendli.ai/v1", @@ -411,6 +412,7 @@ openai_compatible_providers: List = [ "anyscale", "mistral", "groq", + "nvidia_nim", "codestral", "deepseek", "deepinfra", @@ -640,6 +642,7 @@ provider_list: List = [ "anyscale", "mistral", "groq", + "nvidia_nim", "codestral", "text-completion-codestral", "deepseek", @@ -813,6 +816,7 @@ from .llms.openai import ( DeepInfraConfig, AzureAIStudioConfig, ) +from .llms.nvidia_nim import NvidiaNimConfig from .llms.text_completion_codestral import MistralTextCompletionConfig from .llms.azure import ( AzureOpenAIConfig, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 9674d48b12..98b5192784 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,10 +9,11 @@ ## LiteLLM versions of the OpenAI Exception Types -import openai -import httpx from typing import Optional +import httpx +import openai + class AuthenticationError(openai.AuthenticationError): # type: ignore def __init__( @@ -658,15 +659,8 @@ class APIResponseValidationError(openai.APIResponseValidationError): # type: ig class OpenAIError(openai.OpenAIError): # type: ignore - def __init__(self, original_exception): - self.status_code = original_exception.http_status - super().__init__( - http_body=original_exception.http_body, - http_status=original_exception.http_status, - json_body=original_exception.json_body, - headers=original_exception.headers, - code=original_exception.code, - ) + def __init__(self, original_exception=None): + super().__init__() self.llm_provider = "openai" diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 5a6282994c..da9826b9b5 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -1,11 +1,13 @@ #### What this does #### # On success, logs events to Promptlayer -import dotenv, os - -from litellm.proxy._types import UserAPIKeyAuth -from litellm.caching import DualCache -from typing import Literal, Union, Optional +import os import traceback +from typing import Literal, Optional, Union + +import dotenv + +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index f9b2f25e70..b0cc069c40 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -108,6 +108,7 @@ class LunaryLogger: try: print_verbose(f"Lunary Logging - Logging request for model {model}") + template_id = None litellm_params = kwargs.get("litellm_params", {}) optional_params = kwargs.get("optional_params", {}) metadata = litellm_params.get("metadata", {}) or {} diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index aa22b51534..add281e43f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -19,8 +19,7 @@ from litellm import ( turn_off_message_logging, verbose_logger, ) - -from litellm.caching import InMemoryCache, S3Cache, DualCache +from litellm.caching import DualCache, InMemoryCache, S3Cache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index c292c3423f..b763a7c955 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -902,7 +902,7 @@ class AzureChatCompletion(BaseLLM): }, ) - if aembedding == True: + if aembedding is True: response = self.aembedding( data=data, input=input, diff --git a/litellm/llms/nvidia_nim.py b/litellm/llms/nvidia_nim.py new file mode 100644 index 0000000000..ebcc84c13e --- /dev/null +++ b/litellm/llms/nvidia_nim.py @@ -0,0 +1,79 @@ +""" +Nvidia NIM endpoint: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer + +This is OpenAI compatible + +This file only contains param mapping logic + +API calling is done using the OpenAI SDK with an api_base +""" + +import types +from typing import Optional, Union + + +class NvidiaNimConfig: + """ + Reference: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer + + The class `NvidiaNimConfig` provides configuration for the Nvidia NIM's Chat Completions API interface. Below are the parameters: + """ + + temperature: Optional[int] = None + top_p: Optional[int] = None + frequency_penalty: Optional[int] = None + presence_penalty: Optional[int] = None + max_tokens: Optional[int] = None + stop: Optional[Union[str, list]] = None + + def __init__( + self, + temperature: Optional[int] = None, + top_p: Optional[int] = None, + frequency_penalty: Optional[int] = None, + presence_penalty: Optional[int] = None, + max_tokens: Optional[int] = None, + stop: Optional[Union[str, list]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return [ + "stream", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "max_tokens", + "stop", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + supported_openai_params = self.get_supported_openai_params() + for param, value in non_default_params.items(): + if param in supported_openai_params: + optional_params[param] = value + return optional_params diff --git a/litellm/llms/ollama.py b/litellm/llms/ollama.py index e7dd1d5f55..1939715b35 100644 --- a/litellm/llms/ollama.py +++ b/litellm/llms/ollama.py @@ -126,7 +126,7 @@ class OllamaConfig: ) and v is not None } - + def get_required_params(self) -> List[ProviderField]: """For a given provider, return it's required fields with a description""" return [ @@ -451,7 +451,7 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj): { "id": f"call_{str(uuid.uuid4())}", "function": { - "name": function_call["name"], + "name": function_call.get("name", function_call.get("function", None)), "arguments": json.dumps(function_call["arguments"]), }, "type": "function", diff --git a/litellm/llms/ollama_chat.py b/litellm/llms/ollama_chat.py index a7439bbcc0..af6fd5b806 100644 --- a/litellm/llms/ollama_chat.py +++ b/litellm/llms/ollama_chat.py @@ -434,7 +434,7 @@ async def ollama_async_streaming( { "id": f"call_{str(uuid.uuid4())}", "function": { - "name": function_call["name"], + "name": function_call.get("name", function_call.get("function", None)), "arguments": json.dumps(function_call["arguments"]), }, "type": "function", diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index 8ad294457e..534f8e26f2 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -1,27 +1,28 @@ # What is this? ## Controller file for Predibase Integration - https://predibase.com/ -from functools import partial -import os, types -import traceback +import copy import json -from enum import Enum -import requests, copy # type: ignore +import os import time -from typing import Callable, Optional, List, Literal, Union -from litellm.utils import ( - ModelResponse, - Usage, - CustomStreamWrapper, - Message, - Choices, -) -from litellm.litellm_core_utils.core_helpers import map_finish_reason -import litellm -from .prompt_templates.factory import prompt_factory, custom_prompt -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from .base import BaseLLM +import traceback +import types +from enum import Enum +from functools import partial +from typing import Callable, List, Literal, Optional, Union + import httpx # type: ignore +import requests # type: ignore + +import litellm +import litellm.litellm_core_utils +import litellm.litellm_core_utils.litellm_logging +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage + +from .base import BaseLLM +from .prompt_templates.factory import custom_prompt, prompt_factory class PredibaseError(Exception): @@ -146,7 +147,49 @@ class PredibaseConfig: } def get_supported_openai_params(self): - return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"] + return [ + "stream", + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "response_format", + ] + + def map_openai_params(self, non_default_params: dict, optional_params: dict): + for param, value in non_default_params.items(): + # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None + if param == "temperature": + if value == 0.0 or value == 0: + # hugging face exception raised when temp==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive + value = 0.01 + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "n": + optional_params["best_of"] = value + optional_params["do_sample"] = ( + True # Need to sample if you want best of for hf inference endpoints + ) + if param == "stream": + optional_params["stream"] = value + if param == "stop": + optional_params["stop"] = value + if param == "max_tokens": + # HF TGI raises the following exception when max_new_tokens==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive + if value == 0: + value = 1 + optional_params["max_new_tokens"] = value + if param == "echo": + # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details + # Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False + optional_params["decoder_input_details"] = True + if param == "response_format": + optional_params["response_format"] = value + return optional_params class PredibaseChatCompletion(BaseLLM): @@ -225,15 +268,16 @@ class PredibaseChatCompletion(BaseLLM): status_code=response.status_code, ) else: - if ( - not isinstance(completion_response, dict) - or "generated_text" not in completion_response - ): + if not isinstance(completion_response, dict): raise PredibaseError( status_code=422, - message=f"response is not in expected format - {completion_response}", + message=f"'completion_response' is not a dictionary - {completion_response}", + ) + elif "generated_text" not in completion_response: + raise PredibaseError( + status_code=422, + message=f"'generated_text' is not a key response dictionary - {completion_response}", ) - if len(completion_response["generated_text"]) > 0: model_response["choices"][0]["message"]["content"] = self.output_parser( completion_response["generated_text"] @@ -496,7 +540,9 @@ class PredibaseChatCompletion(BaseLLM): except httpx.HTTPStatusError as e: raise PredibaseError( status_code=e.response.status_code, - message="HTTPStatusError - {}".format(e.response.text), + message="HTTPStatusError - received status_code={}, error_message={}".format( + e.response.status_code, e.response.text + ), ) except Exception as e: raise PredibaseError( diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 398e96af7e..a97d6812c8 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -172,14 +172,35 @@ def ollama_pt( images.append(base64_image) return {"prompt": prompt, "images": images} else: - prompt = "".join( - ( - m["content"] - if isinstance(m["content"], str) is str - else "".join(m["content"]) - ) - for m in messages - ) + prompt = "" + for message in messages: + role = message["role"] + content = message.get("content", "") + + if "tool_calls" in message: + tool_calls = [] + + for call in message["tool_calls"]: + call_id: str = call["id"] + function_name: str = call["function"]["name"] + arguments = json.loads(call["function"]["arguments"]) + + tool_calls.append( + { + "id": call_id, + "type": "function", + "function": {"name": function_name, "arguments": arguments}, + } + ) + + prompt += f"### Assistant:\nTool Calls: {json.dumps(tool_calls, indent=2)}\n\n" + + elif "tool_call_id" in message: + prompt += f"### User:\n{message['content']}\n\n" + + elif content: + prompt += f"### {role.capitalize()}:\n{content}\n\n" + return prompt @@ -710,7 +731,7 @@ def convert_to_anthropic_tool_result_xml(message: dict) -> str: """ Anthropic tool_results look like: - + [Successful results] diff --git a/litellm/llms/replicate.py b/litellm/llms/replicate.py index ce62e51e90..56549cfd4a 100644 --- a/litellm/llms/replicate.py +++ b/litellm/llms/replicate.py @@ -1,13 +1,18 @@ -import os, types +import asyncio import json -import requests # type: ignore +import os import time -from typing import Callable, Optional, Union, Tuple, Any -from litellm.utils import ModelResponse, Usage, CustomStreamWrapper -import litellm, asyncio +import types +from typing import Any, Callable, Optional, Tuple, Union + import httpx # type: ignore -from .prompt_templates.factory import prompt_factory, custom_prompt +import requests # type: ignore + +import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.utils import CustomStreamWrapper, ModelResponse, Usage + +from .prompt_templates.factory import custom_prompt, prompt_factory class ReplicateError(Exception): @@ -329,7 +334,15 @@ async def async_handle_prediction_response_streaming( response_data = response.json() status = response_data["status"] if "output" in response_data: - output_string = "".join(response_data["output"]) + try: + output_string = "".join(response_data["output"]) + except Exception as e: + raise ReplicateError( + status_code=422, + message="Unable to parse response. Got={}".format( + response_data["output"] + ), + ) new_output = output_string[len(previous_output) :] print_verbose(f"New chunk: {new_output}") yield {"output": new_output, "status": status} diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 63bcd9f4f5..856b05f61c 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -562,7 +562,47 @@ class VertexLLM(BaseLLM): status_code=422, ) + ## GET MODEL ## + model_response.model = model + ## CHECK IF RESPONSE FLAGGED + if "promptFeedback" in completion_response: + if "blockReason" in completion_response["promptFeedback"]: + # If set, the prompt was blocked and no candidates are returned. Rephrase your prompt + model_response.choices[0].finish_reason = "content_filter" + + chat_completion_message: ChatCompletionResponseMessage = { + "role": "assistant", + "content": None, + } + + choice = litellm.Choices( + finish_reason="content_filter", + index=0, + message=chat_completion_message, # type: ignore + logprobs=None, + enhancements=None, + ) + + model_response.choices = [choice] + + ## GET USAGE ## + usage = litellm.Usage( + prompt_tokens=completion_response["usageMetadata"][ + "promptTokenCount" + ], + completion_tokens=completion_response["usageMetadata"].get( + "candidatesTokenCount", 0 + ), + total_tokens=completion_response["usageMetadata"][ + "totalTokenCount" + ], + ) + + setattr(model_response, "usage", usage) + + return model_response + if len(completion_response["candidates"]) > 0: content_policy_violations = ( VertexGeminiConfig().get_flagged_finish_reasons() @@ -573,26 +613,45 @@ class VertexLLM(BaseLLM): in content_policy_violations.keys() ): ## CONTENT POLICY VIOLATION ERROR - raise VertexAIError( - status_code=400, - message="The response was blocked. Reason={}. Raw Response={}".format( - content_policy_violations[ - completion_response["candidates"][0]["finishReason"] - ], - completion_response, - ), + model_response.choices[0].finish_reason = "content_filter" + + chat_completion_message = { + "role": "assistant", + "content": None, + } + + choice = litellm.Choices( + finish_reason="content_filter", + index=0, + message=chat_completion_message, # type: ignore + logprobs=None, + enhancements=None, ) + model_response.choices = [choice] + + ## GET USAGE ## + usage = litellm.Usage( + prompt_tokens=completion_response["usageMetadata"][ + "promptTokenCount" + ], + completion_tokens=completion_response["usageMetadata"].get( + "candidatesTokenCount", 0 + ), + total_tokens=completion_response["usageMetadata"][ + "totalTokenCount" + ], + ) + + setattr(model_response, "usage", usage) + + return model_response + model_response.choices = [] # type: ignore - ## GET MODEL ## - model_response.model = model - try: ## GET TEXT ## - chat_completion_message: ChatCompletionResponseMessage = { - "role": "assistant" - } + chat_completion_message = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] for idx, candidate in enumerate(completion_response["candidates"]): @@ -632,9 +691,9 @@ class VertexLLM(BaseLLM): ## GET USAGE ## usage = litellm.Usage( prompt_tokens=completion_response["usageMetadata"]["promptTokenCount"], - completion_tokens=completion_response["usageMetadata"][ - "candidatesTokenCount" - ], + completion_tokens=completion_response["usageMetadata"].get( + "candidatesTokenCount", 0 + ), total_tokens=completion_response["usageMetadata"]["totalTokenCount"], ) diff --git a/litellm/main.py b/litellm/main.py index 92cd2fee11..5436c49a86 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -348,6 +348,7 @@ async def acompletion( or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "codestral" or custom_llm_provider == "text-completion-codestral" or custom_llm_provider == "deepseek" @@ -428,6 +429,7 @@ def mock_completion( model: str, messages: List, stream: Optional[bool] = False, + n: Optional[int] = None, mock_response: Union[str, Exception, dict] = "This is a mock request", mock_tool_calls: Optional[List] = None, logging=None, @@ -486,18 +488,32 @@ def mock_completion( if kwargs.get("acompletion", False) == True: return CustomStreamWrapper( completion_stream=async_mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model + model_response, mock_response=mock_response, model=model, n=n ), model=model, custom_llm_provider="openai", logging_obj=logging, ) response = mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model + model_response, + mock_response=mock_response, + model=model, + n=n, ) return response - - model_response["choices"][0]["message"]["content"] = mock_response + if n is None: + model_response["choices"][0]["message"]["content"] = mock_response + else: + _all_choices = [] + for i in range(n): + _choice = litellm.utils.Choices( + index=i, + message=litellm.utils.Message( + content=mock_response, role="assistant" + ), + ) + _all_choices.append(_choice) + model_response["choices"] = _all_choices model_response["created"] = int(time.time()) model_response["model"] = model @@ -634,6 +650,7 @@ def completion( headers = kwargs.get("headers", None) or extra_headers num_retries = kwargs.get("num_retries", None) ## deprecated max_retries = kwargs.get("max_retries", None) + cooldown_time = kwargs.get("cooldown_time", None) context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None) organization = kwargs.get("organization", None) ### CUSTOM MODEL COST ### @@ -747,6 +764,7 @@ def completion( "allowed_model_region", "model_config", "fastest_response", + "cooldown_time", ] default_params = openai_params + litellm_params @@ -931,6 +949,7 @@ def completion( input_cost_per_token=input_cost_per_token, output_cost_per_second=output_cost_per_second, output_cost_per_token=output_cost_per_token, + cooldown_time=cooldown_time, ) logging.update_environment_variables( model=model, @@ -944,6 +963,7 @@ def completion( model, messages, stream=stream, + n=n, mock_response=mock_response, mock_tool_calls=mock_tool_calls, logging=logging, @@ -1171,6 +1191,7 @@ def completion( or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "codestral" or custom_llm_provider == "deepseek" or custom_llm_provider == "anyscale" @@ -2906,6 +2927,7 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "deepseek" or custom_llm_provider == "fireworks_ai" or custom_llm_provider == "ollama" @@ -2985,6 +3007,7 @@ def embedding( client = kwargs.pop("client", None) rpm = kwargs.pop("rpm", None) tpm = kwargs.pop("tpm", None) + cooldown_time = kwargs.get("cooldown_time", None) max_parallel_requests = kwargs.pop("max_parallel_requests", None) model_info = kwargs.get("model_info", None) metadata = kwargs.get("metadata", None) @@ -3060,6 +3083,7 @@ def embedding( "region_name", "allowed_model_region", "model_config", + "cooldown_time", ] default_params = openai_params + litellm_params non_default_params = { @@ -3120,6 +3144,7 @@ def embedding( "aembedding": aembedding, "preset_cache_key": None, "stream_response": {}, + "cooldown_time": cooldown_time, }, ) if azure == True or custom_llm_provider == "azure": @@ -3481,6 +3506,7 @@ async def atext_completion( or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "text-completion-codestral" or custom_llm_provider == "deepseek" or custom_llm_provider == "fireworks_ai" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef07d87ccb..415d220f21 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -887,7 +887,7 @@ "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000010, + "output_cost_per_token": 0.00000008, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -906,8 +906,8 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 0.00000027, - "output_cost_per_token": 0.00000027, + "input_cost_per_token": 0.00000024, + "output_cost_per_token": 0.00000024, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -916,8 +916,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000010, - "output_cost_per_token": 0.00000010, + "input_cost_per_token": 0.00000007, + "output_cost_per_token": 0.00000007, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 640a3b2cf2..938e74b5e7 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,10 +1,54 @@ -model_list: - - model_name: my-fake-model +# model_list: +# - model_name: my-fake-model +# litellm_params: +# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 +# api_key: my-fake-key +# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000 +# mock_response: "Hello world 1" +# model_info: +# max_input_tokens: 0 # trigger context window fallback +# - model_name: my-fake-model +# litellm_params: +# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 +# api_key: my-fake-key +# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000 +# mock_response: "Hello world 2" +# model_info: +# max_input_tokens: 0 + +# router_settings: +# enable_pre_call_checks: True + + +# litellm_settings: +# failure_callback: ["langfuse"] + +model_list: + - model_name: summarize litellm_params: - model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 - api_key: my-fake-key - aws_bedrock_runtime_endpoint: http://127.0.0.1:8000 + model: openai/gpt-4o + rpm: 10000 + tpm: 12000000 + api_key: os.environ/OPENAI_API_KEY + mock_response: Hello world 1 + + - model_name: summarize-l + litellm_params: + model: claude-3-5-sonnet-20240620 + rpm: 4000 + tpm: 400000 + api_key: os.environ/ANTHROPIC_API_KEY + mock_response: Hello world 2 litellm_settings: - success_callback: ["langfuse"] - failure_callback: ["langfuse"] + num_retries: 3 + request_timeout: 120 + allowed_fails: 3 + # fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}] + # context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}] + + + +router_settings: + routing_strategy: simple-shuffle + enable_pre_call_checks: true. diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index 04a4806c12..2060f61ca4 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -1,4 +1,7 @@ model_list: +- model_name: gemini-1.5-flash-gemini + litellm_params: + model: gemini/gemini-1.5-flash - litellm_params: api_base: http://0.0.0.0:8080 api_key: '' @@ -11,13 +14,10 @@ model_list: - model_name: fake-openai-endpoint litellm_params: model: predibase/llama-3-8b-instruct - api_base: "http://0.0.0.0:8000" + api_base: "http://0.0.0.0:8081" api_key: os.environ/PREDIBASE_API_KEY tenant_id: os.environ/PREDIBASE_TENANT_ID - max_retries: 0 - temperature: 0.1 max_new_tokens: 256 - return_full_text: false # - litellm_params: # api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ @@ -70,6 +70,8 @@ model_list: litellm_settings: callbacks: ["dynamic_rate_limiter"] + # success_callback: ["langfuse"] + # failure_callback: ["langfuse"] # default_team_settings: # - team_id: proj1 # success_callback: ["langfuse"] @@ -91,8 +93,8 @@ assistant_settings: router_settings: enable_pre_call_checks: true -general_settings: - alerting: ["slack"] - enable_jwt_auth: True - litellm_jwtauth: - team_id_jwt_field: "client_id" \ No newline at end of file +# general_settings: +# # alerting: ["slack"] +# enable_jwt_auth: True +# litellm_jwtauth: +# team_id_jwt_field: "client_id" \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0883763d1c..640c7695a0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1627,3 +1627,9 @@ class CommonProxyErrors(enum.Enum): no_llm_router = "No models configured on proxy" not_allowed_access = "Admin-only endpoint. Not allowed to access this." not_premium_user = "You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" + + +class SpendCalculateRequest(LiteLLMBase): + model: Optional[str] = None + messages: Optional[List] = None + completion_response: Optional[dict] = None diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py new file mode 100644 index 0000000000..cc09a9689b --- /dev/null +++ b/litellm/proxy/auth/auth_utils.py @@ -0,0 +1,43 @@ +from litellm._logging import verbose_proxy_logger + + +def route_in_additonal_public_routes(current_route: str): + """ + Helper to check if the user defined public_routes on config.yaml + + Parameters: + - current_route: str - the route the user is trying to call + + Returns: + - bool - True if the route is defined in public_routes + - bool - False if the route is not defined in public_routes + + + In order to use this the litellm config.yaml should have the following in general_settings: + + ```yaml + general_settings: + master_key: sk-1234 + public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] + ``` + """ + + # check if user is premium_user - if not do nothing + from litellm.proxy._types import LiteLLMRoutes + from litellm.proxy.proxy_server import general_settings, premium_user + + try: + if premium_user is not True: + return False + # check if this is defined on the config + if general_settings is None: + return False + + routes_defined = general_settings.get("public_routes", []) + if current_route in routes_defined: + return True + + return False + except Exception as e: + verbose_proxy_logger.error(f"route_in_additonal_public_routes: {str(e)}") + return False diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py index ffd9f5273e..0310dcaf58 100644 --- a/litellm/proxy/auth/litellm_license.py +++ b/litellm/proxy/auth/litellm_license.py @@ -1,6 +1,11 @@ # What is this? ## If litellm license in env, checks if it's valid +import base64 +import json import os +from datetime import datetime + +from litellm._logging import verbose_proxy_logger from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -15,6 +20,26 @@ class LicenseCheck: def __init__(self) -> None: self.license_str = os.getenv("LITELLM_LICENSE", None) self.http_handler = HTTPHandler() + self.public_key = None + self.read_public_key() + + def read_public_key(self): + try: + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding, rsa + + # current dir + current_dir = os.path.dirname(os.path.realpath(__file__)) + + # check if public_key.pem exists + _path_to_public_key = os.path.join(current_dir, "public_key.pem") + if os.path.exists(_path_to_public_key): + with open(_path_to_public_key, "rb") as key_file: + self.public_key = serialization.load_pem_public_key(key_file.read()) + else: + self.public_key = None + except Exception as e: + verbose_proxy_logger.error(f"Error reading public key: {str(e)}") def _verify(self, license_str: str) -> bool: url = "{}/verify_license/{}".format(self.base_url, license_str) @@ -35,11 +60,58 @@ class LicenseCheck: return False def is_premium(self) -> bool: + """ + 1. verify_license_without_api_request: checks if license was generate using private / public key pair + 2. _verify: checks if license is valid calling litellm API. This is the old way we were generating/validating license + """ try: if self.license_str is None: return False + elif self.verify_license_without_api_request( + public_key=self.public_key, license_key=self.license_str + ): + return True elif self._verify(license_str=self.license_str): return True return False except Exception as e: return False + + def verify_license_without_api_request(self, public_key, license_key): + try: + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding, rsa + + # Decode the license key + decoded = base64.b64decode(license_key) + message, signature = decoded.split(b".", 1) + + # Verify the signature + public_key.verify( + signature, + message, + padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), + salt_length=padding.PSS.MAX_LENGTH, + ), + hashes.SHA256(), + ) + + # Decode and parse the data + license_data = json.loads(message.decode()) + + # debug information provided in license data + verbose_proxy_logger.debug("License data: %s", license_data) + + # Check expiration date + expiration_date = datetime.strptime( + license_data["expiration_date"], "%Y-%m-%d" + ) + if expiration_date < datetime.now(): + return False, "License has expired" + + return True + + except Exception as e: + verbose_proxy_logger.error(str(e)) + return False diff --git a/litellm/proxy/auth/public_key.pem b/litellm/proxy/auth/public_key.pem new file mode 100644 index 0000000000..12a69dde27 --- /dev/null +++ b/litellm/proxy/auth/public_key.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmfBuNiNzDkNWyce23koQ +w0vq3bSVHkq7fd9Sw/U1q7FwRwL221daLTyGWssd8xAoQSFXAJKoBwzJQ9wd+o44 +lfL54E3a61nfjZuF+D9ntpXZFfEAxLVtIahDeQjUz4b/EpgciWIJyUfjCJrQo6LY +eyAZPTGSO8V3zHyaU+CFywq5XCuCnfZqCZeCw051St59A2v8W32mXSCJ+A+x0hYP +yXJyRRFcefSFG5IBuRHr4Y24Vx7NUIAoco5cnxJho9g2z3J/Hb0GKW+oBNvRVumk +nuA2Ljmjh4yI0OoTIW8ZWxemvCCJHSjdfKlMyb+QI4fmeiIUZzP5Au+F561Styqq +YQIDAQAB +-----END PUBLIC KEY----- diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 3d14f53000..d3e937734c 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -56,6 +56,7 @@ from litellm.proxy.auth.auth_checks import ( get_user_object, log_to_opentelemetry, ) +from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.utils import _to_ns @@ -137,7 +138,10 @@ async def user_api_key_auth( """ route: str = request.url.path - if route in LiteLLMRoutes.public_routes.value: + if ( + route in LiteLLMRoutes.public_routes.value + or route_in_additonal_public_routes(current_route=route) + ): # check if public endpoint return UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY) diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py new file mode 100644 index 0000000000..dc77958a62 --- /dev/null +++ b/litellm/proxy/common_utils/debug_utils.py @@ -0,0 +1,27 @@ +# Start tracing memory allocations +import os +import tracemalloc + +from fastapi import APIRouter + +from litellm._logging import verbose_proxy_logger + +router = APIRouter() + +if os.environ.get("LITELLM_PROFILE", "false").lower() == "true": + tracemalloc.start() + + @router.get("/memory-usage", include_in_schema=False) + async def memory_usage(): + # Take a snapshot of the current memory usage + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics("lineno") + verbose_proxy_logger.debug("TOP STATS: %s", top_stats) + + # Get the top 50 memory usage lines + top_50 = top_stats[:50] + result = [] + for stat in top_50: + result.append(f"{stat.traceback.format()}: {stat.size / 1024} KiB") + + return {"top_50_memory_usage": result} diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index d5190455f1..0c0365f43d 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -21,10 +21,12 @@ model_list: general_settings: master_key: sk-1234 alerting: ["slack", "email"] + public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] + litellm_settings: success_callback: ["prometheus"] - callbacks: ["otel"] + callbacks: ["otel", "hide_secrets"] failure_callback: ["prometheus"] store_audit_logs: true redact_messages_in_exceptions: True diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 30b90abe64..c3b855c5f5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -140,6 +140,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth ## Import All Misc routes here ## from litellm.proxy.caching_routes import router as caching_router +from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router @@ -1478,6 +1479,21 @@ class ProxyConfig: llama_guard_object = _ENTERPRISE_LlamaGuard() imported_list.append(llama_guard_object) + elif ( + isinstance(callback, str) and callback == "hide_secrets" + ): + from enterprise.enterprise_hooks.secret_detection import ( + _ENTERPRISE_SecretDetection, + ) + + if premium_user != True: + raise Exception( + "Trying to use secret hiding" + + CommonProxyErrors.not_premium_user.value + ) + + _secret_detection_object = _ENTERPRISE_SecretDetection() + imported_list.append(_secret_detection_object) elif ( isinstance(callback, str) and callback == "openai_moderations" @@ -7508,12 +7524,6 @@ async def login(request: Request): litellm_dashboard_ui += "/ui/" import jwt - if litellm_master_key_hash is None: - raise HTTPException( - status_code=500, - detail={"error": "No master key set, please set LITELLM_MASTER_KEY"}, - ) - jwt_token = jwt.encode( { "user_id": user_id, @@ -7523,7 +7533,7 @@ async def login(request: Request): "login_method": "username_password", "premium_user": premium_user, }, - litellm_master_key_hash, + master_key, algorithm="HS256", ) litellm_dashboard_ui += "?userID=" + user_id @@ -7578,14 +7588,6 @@ async def login(request: Request): litellm_dashboard_ui += "/ui/" import jwt - if litellm_master_key_hash is None: - raise HTTPException( - status_code=500, - detail={ - "error": "No master key set, please set LITELLM_MASTER_KEY" - }, - ) - jwt_token = jwt.encode( { "user_id": user_id, @@ -7595,7 +7597,7 @@ async def login(request: Request): "login_method": "username_password", "premium_user": premium_user, }, - litellm_master_key_hash, + master_key, algorithm="HS256", ) litellm_dashboard_ui += "?userID=" + user_id @@ -7642,7 +7644,14 @@ async def onboarding(invite_link: str): - Get user from db - Pass in user_email if set """ - global prisma_client + global prisma_client, master_key + if master_key is None: + raise ProxyException( + message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.", + type="auth_error", + param="master_key", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) ### VALIDATE INVITE LINK ### if prisma_client is None: raise HTTPException( @@ -7714,12 +7723,6 @@ async def onboarding(invite_link: str): litellm_dashboard_ui += "/ui/onboarding" import jwt - if litellm_master_key_hash is None: - raise HTTPException( - status_code=500, - detail={"error": "No master key set, please set LITELLM_MASTER_KEY"}, - ) - jwt_token = jwt.encode( { "user_id": user_obj.user_id, @@ -7729,7 +7732,7 @@ async def onboarding(invite_link: str): "login_method": "username_password", "premium_user": premium_user, }, - litellm_master_key_hash, + master_key, algorithm="HS256", ) @@ -7862,11 +7865,18 @@ def get_image(): @app.get("/sso/callback", tags=["experimental"], include_in_schema=False) async def auth_callback(request: Request): """Verify login""" - global general_settings, ui_access_mode, premium_user + global general_settings, ui_access_mode, premium_user, master_key microsoft_client_id = os.getenv("MICROSOFT_CLIENT_ID", None) google_client_id = os.getenv("GOOGLE_CLIENT_ID", None) generic_client_id = os.getenv("GENERIC_CLIENT_ID", None) # get url from request + if master_key is None: + raise ProxyException( + message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.", + type="auth_error", + param="master_key", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) redirect_url = os.getenv("PROXY_BASE_URL", str(request.base_url)) if redirect_url.endswith("/"): redirect_url += "sso/callback" @@ -8140,12 +8150,6 @@ async def auth_callback(request: Request): import jwt - if litellm_master_key_hash is None: - raise HTTPException( - status_code=500, - detail={"error": "No master key set, please set LITELLM_MASTER_KEY"}, - ) - jwt_token = jwt.encode( { "user_id": user_id, @@ -8155,7 +8159,7 @@ async def auth_callback(request: Request): "login_method": "sso", "premium_user": premium_user, }, - litellm_master_key_hash, + master_key, algorithm="HS256", ) litellm_dashboard_ui += "?userID=" + user_id @@ -9179,3 +9183,4 @@ app.include_router(team_router) app.include_router(spend_management_router) app.include_router(caching_router) app.include_router(analytics_router) +app.include_router(debugging_endpoints_router) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 11edd18873..1fbd95b3cf 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1199,7 +1199,7 @@ async def _get_spend_report_for_time_range( } }, ) -async def calculate_spend(request: Request): +async def calculate_spend(request: SpendCalculateRequest): """ Accepts all the params of completion_cost. @@ -1248,14 +1248,93 @@ async def calculate_spend(request: Request): }' ``` """ - from litellm import completion_cost + try: + from litellm import completion_cost + from litellm.cost_calculator import CostPerToken + from litellm.proxy.proxy_server import llm_router - data = await request.json() - if "completion_response" in data: - data["completion_response"] = litellm.ModelResponse( - **data["completion_response"] + _cost = None + if request.model is not None: + if request.messages is None: + raise HTTPException( + status_code=400, + detail="Bad Request - messages must be provided if 'model' is provided", + ) + + # check if model in llm_router + _model_in_llm_router = None + cost_per_token: Optional[CostPerToken] = None + if llm_router is not None: + if ( + llm_router.model_group_alias is not None + and request.model in llm_router.model_group_alias + ): + # lookup alias in llm_router + _model_group_name = llm_router.model_group_alias[request.model] + for model in llm_router.model_list: + if model.get("model_name") == _model_group_name: + _model_in_llm_router = model + + else: + # no model_group aliases set -> try finding model in llm_router + # find model in llm_router + for model in llm_router.model_list: + if model.get("model_name") == request.model: + _model_in_llm_router = model + + """ + 3 cases for /spend/calculate + + 1. user passes model, and model is defined on litellm config.yaml or in DB. use info on config or in DB in this case + 2. user passes model, and model is not defined on litellm config.yaml or in DB. Pass model as is to litellm.completion_cost + 3. user passes completion_response + + """ + if _model_in_llm_router is not None: + _litellm_params = _model_in_llm_router.get("litellm_params") + _litellm_model_name = _litellm_params.get("model") + input_cost_per_token = _litellm_params.get("input_cost_per_token") + output_cost_per_token = _litellm_params.get("output_cost_per_token") + if ( + input_cost_per_token is not None + or output_cost_per_token is not None + ): + cost_per_token = CostPerToken( + input_cost_per_token=input_cost_per_token, + output_cost_per_token=output_cost_per_token, + ) + + _cost = completion_cost( + model=_litellm_model_name, + messages=request.messages, + custom_cost_per_token=cost_per_token, + ) + else: + _cost = completion_cost(model=request.model, messages=request.messages) + elif request.completion_response is not None: + _completion_response = litellm.ModelResponse(**request.completion_response) + _cost = completion_cost(completion_response=_completion_response) + else: + raise HTTPException( + status_code=400, + detail="Bad Request - Either 'model' or 'completion_response' must be provided", + ) + return {"cost": _cost} + except Exception as e: + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", str(e)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), ) - return {"cost": completion_cost(**data)} @router.get( diff --git a/litellm/router.py b/litellm/router.py index e9b0cc00a9..e2f7ce8b21 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -404,6 +404,7 @@ class Router: litellm.failure_callback = [self.deployment_callback_on_failure] print( # noqa f"Intialized router with Routing strategy: {self.routing_strategy}\n\n" + f"Routing enable_pre_call_checks: {self.enable_pre_call_checks}\n\n" f"Routing fallbacks: {self.fallbacks}\n\n" f"Routing content fallbacks: {self.content_policy_fallbacks}\n\n" f"Routing context window fallbacks: {self.context_window_fallbacks}\n\n" @@ -2116,6 +2117,12 @@ class Router: If it fails after num_retries, fall back to another model group """ mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None) + mock_testing_context_fallbacks = kwargs.pop( + "mock_testing_context_fallbacks", None + ) + mock_testing_content_policy_fallbacks = kwargs.pop( + "mock_testing_content_policy_fallbacks", None + ) model_group = kwargs.get("model") fallbacks = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.get( @@ -2129,6 +2136,26 @@ class Router: raise Exception( f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}" ) + elif ( + mock_testing_context_fallbacks is not None + and mock_testing_context_fallbacks is True + ): + raise litellm.ContextWindowExceededError( + model=model_group, + llm_provider="", + message=f"This is a mock exception for model={model_group}, to trigger a fallback. \ + Context_Window_Fallbacks={context_window_fallbacks}", + ) + elif ( + mock_testing_content_policy_fallbacks is not None + and mock_testing_content_policy_fallbacks is True + ): + raise litellm.ContentPolicyViolationError( + model=model_group, + llm_provider="", + message=f"This is a mock exception for model={model_group}, to trigger a fallback. \ + Context_Policy_Fallbacks={content_policy_fallbacks}", + ) response = await self.async_function_with_retries(*args, **kwargs) verbose_router_logger.debug(f"Async Response: {response}") @@ -2148,73 +2175,93 @@ class Router: ) ): # don't retry a malformed request raise e - if ( - isinstance(e, litellm.ContextWindowExceededError) - and context_window_fallbacks is not None - ): - fallback_model_group = None - for ( - item - ) in context_window_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}] - if list(item.keys())[0] == model_group: - fallback_model_group = item[model_group] - break + if isinstance(e, litellm.ContextWindowExceededError): + if context_window_fallbacks is not None: + fallback_model_group = None + for ( + item + ) in context_window_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}] + if list(item.keys())[0] == model_group: + fallback_model_group = item[model_group] + break - if fallback_model_group is None: - raise original_exception + if fallback_model_group is None: + raise original_exception - for mg in fallback_model_group: - """ - Iterate through the model groups and try calling that deployment - """ - try: - kwargs["model"] = mg - kwargs.setdefault("metadata", {}).update( - {"model_group": mg} - ) # update model_group used, if fallbacks are done - response = await self.async_function_with_retries( - *args, **kwargs + for mg in fallback_model_group: + """ + Iterate through the model groups and try calling that deployment + """ + try: + kwargs["model"] = mg + kwargs.setdefault("metadata", {}).update( + {"model_group": mg} + ) # update model_group used, if fallbacks are done + response = await self.async_function_with_retries( + *args, **kwargs + ) + verbose_router_logger.info( + "Successful fallback b/w models." + ) + return response + except Exception as e: + pass + else: + error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format( + model_group, context_window_fallbacks, fallbacks + ) + verbose_router_logger.info( + msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \ + to fallbacks, if available.{}".format( + error_message ) - verbose_router_logger.info( - "Successful fallback b/w models." - ) - return response - except Exception as e: - pass - elif ( - isinstance(e, litellm.ContentPolicyViolationError) - and content_policy_fallbacks is not None - ): - fallback_model_group = None - for ( - item - ) in content_policy_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}] - if list(item.keys())[0] == model_group: - fallback_model_group = item[model_group] - break + ) - if fallback_model_group is None: - raise original_exception + e.message += "\n{}".format(error_message) + elif isinstance(e, litellm.ContentPolicyViolationError): + if content_policy_fallbacks is not None: + fallback_model_group = None + for ( + item + ) in content_policy_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}] + if list(item.keys())[0] == model_group: + fallback_model_group = item[model_group] + break - for mg in fallback_model_group: - """ - Iterate through the model groups and try calling that deployment - """ - try: - kwargs["model"] = mg - kwargs.setdefault("metadata", {}).update( - {"model_group": mg} - ) # update model_group used, if fallbacks are done - response = await self.async_function_with_retries( - *args, **kwargs + if fallback_model_group is None: + raise original_exception + + for mg in fallback_model_group: + """ + Iterate through the model groups and try calling that deployment + """ + try: + kwargs["model"] = mg + kwargs.setdefault("metadata", {}).update( + {"model_group": mg} + ) # update model_group used, if fallbacks are done + response = await self.async_function_with_retries( + *args, **kwargs + ) + verbose_router_logger.info( + "Successful fallback b/w models." + ) + return response + except Exception as e: + pass + else: + error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format( + model_group, content_policy_fallbacks, fallbacks + ) + verbose_router_logger.info( + msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \ + to fallbacks, if available.{}".format( + error_message ) - verbose_router_logger.info( - "Successful fallback b/w models." - ) - return response - except Exception as e: - pass - elif fallbacks is not None: + ) + + e.message += "\n{}".format(error_message) + if fallbacks is not None: verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") generic_fallback_idx: Optional[int] = None ## check for specific model group-specific fallbacks @@ -2769,7 +2816,9 @@ class Router: exception_response = getattr(exception, "response", {}) exception_headers = getattr(exception_response, "headers", None) - _time_to_cooldown = self.cooldown_time + _time_to_cooldown = kwargs.get("litellm_params", {}).get( + "cooldown_time", self.cooldown_time + ) if exception_headers is not None: @@ -3915,9 +3964,38 @@ class Router: raise Exception("Model invalid format - {}".format(type(model))) return None + def get_router_model_info(self, deployment: dict) -> ModelMapInfo: + """ + For a given model id, return the model info (max tokens, input cost, output cost, etc.). + + Augment litellm info with additional params set in `model_info`. + + Returns + - ModelInfo - If found -> typed dict with max tokens, input cost, etc. + """ + ## SET MODEL NAME + base_model = deployment.get("model_info", {}).get("base_model", None) + if base_model is None: + base_model = deployment.get("litellm_params", {}).get("base_model", None) + model = base_model or deployment.get("litellm_params", {}).get("model", None) + + ## GET LITELLM MODEL INFO + model_info = litellm.get_model_info(model=model) + + ## CHECK USER SET MODEL INFO + user_model_info = deployment.get("model_info", {}) + + model_info.update(user_model_info) + + return model_info + def get_model_info(self, id: str) -> Optional[dict]: """ For a given model id, return the model info + + Returns + - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info'] + - None: could not find deployment in list """ for model in self.model_list: if "model_info" in model and "id" in model["model_info"]: @@ -4307,6 +4385,7 @@ class Router: return _returned_deployments _context_window_error = False + _potential_error_str = "" _rate_limit_error = False ## get model group RPM ## @@ -4327,7 +4406,7 @@ class Router: model = base_model or deployment.get("litellm_params", {}).get( "model", None ) - model_info = litellm.get_model_info(model=model) + model_info = self.get_router_model_info(deployment=deployment) if ( isinstance(model_info, dict) @@ -4339,6 +4418,11 @@ class Router: ): invalid_model_indices.append(idx) _context_window_error = True + _potential_error_str += ( + "Model={}, Max Input Tokens={}, Got={}".format( + model, model_info["max_input_tokens"], input_tokens + ) + ) continue except Exception as e: verbose_router_logger.debug("An error occurs - {}".format(str(e))) @@ -4438,15 +4522,13 @@ class Router: raise ValueError( f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. Try again in {self.cooldown_time} seconds." ) - elif _context_window_error == True: + elif _context_window_error is True: raise litellm.ContextWindowExceededError( - message="Context Window exceeded for given call", + message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{}".format( + _potential_error_str + ), model=model, llm_provider="", - response=httpx.Response( - status_code=400, - request=httpx.Request("GET", "https://example.com"), - ), ) if len(invalid_model_indices) > 0: for idx in reversed(invalid_model_indices): @@ -4558,127 +4640,155 @@ class Router: specific_deployment=specific_deployment, request_kwargs=request_kwargs, ) - - model, healthy_deployments = self._common_checks_available_deployment( - model=model, - messages=messages, - input=input, - specific_deployment=specific_deployment, - ) # type: ignore - - if isinstance(healthy_deployments, dict): - return healthy_deployments - - # filter out the deployments currently cooling down - deployments_to_remove = [] - # cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"] - cooldown_deployments = await self._async_get_cooldown_deployments() - verbose_router_logger.debug( - f"async cooldown deployments: {cooldown_deployments}" - ) - # Find deployments in model_list whose model_id is cooling down - for deployment in healthy_deployments: - deployment_id = deployment["model_info"]["id"] - if deployment_id in cooldown_deployments: - deployments_to_remove.append(deployment) - # remove unhealthy deployments from healthy deployments - for deployment in deployments_to_remove: - healthy_deployments.remove(deployment) - - # filter pre-call checks - _allowed_model_region = ( - request_kwargs.get("allowed_model_region") - if request_kwargs is not None - else None - ) - - if self.enable_pre_call_checks and messages is not None: - healthy_deployments = self._pre_call_checks( + try: + model, healthy_deployments = self._common_checks_available_deployment( model=model, - healthy_deployments=healthy_deployments, - messages=messages, - request_kwargs=request_kwargs, - ) - - if len(healthy_deployments) == 0: - if _allowed_model_region is None: - _allowed_model_region = "n/a" - raise ValueError( - f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. pre-call-checks={self.enable_pre_call_checks}, allowed_model_region={_allowed_model_region}" - ) - - if ( - self.routing_strategy == "usage-based-routing-v2" - and self.lowesttpm_logger_v2 is not None - ): - deployment = await self.lowesttpm_logger_v2.async_get_available_deployments( - model_group=model, - healthy_deployments=healthy_deployments, # type: ignore messages=messages, input=input, - ) - if ( - self.routing_strategy == "cost-based-routing" - and self.lowestcost_logger is not None - ): - deployment = await self.lowestcost_logger.async_get_available_deployments( - model_group=model, - healthy_deployments=healthy_deployments, # type: ignore - messages=messages, - input=input, - ) - elif self.routing_strategy == "simple-shuffle": - # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm - ############## Check if we can do a RPM/TPM based weighted pick ################# - rpm = healthy_deployments[0].get("litellm_params").get("rpm", None) - if rpm is not None: - # use weight-random pick if rpms provided - rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments] - verbose_router_logger.debug(f"\nrpms {rpms}") - total_rpm = sum(rpms) - weights = [rpm / total_rpm for rpm in rpms] - verbose_router_logger.debug(f"\n weights {weights}") - # Perform weighted random pick - selected_index = random.choices(range(len(rpms)), weights=weights)[0] - verbose_router_logger.debug(f"\n selected index, {selected_index}") - deployment = healthy_deployments[selected_index] - verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}" - ) - return deployment or deployment[0] - ############## Check if we can do a RPM/TPM based weighted pick ################# - tpm = healthy_deployments[0].get("litellm_params").get("tpm", None) - if tpm is not None: - # use weight-random pick if rpms provided - tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments] - verbose_router_logger.debug(f"\ntpms {tpms}") - total_tpm = sum(tpms) - weights = [tpm / total_tpm for tpm in tpms] - verbose_router_logger.debug(f"\n weights {weights}") - # Perform weighted random pick - selected_index = random.choices(range(len(tpms)), weights=weights)[0] - verbose_router_logger.debug(f"\n selected index, {selected_index}") - deployment = healthy_deployments[selected_index] - verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}" - ) - return deployment or deployment[0] + specific_deployment=specific_deployment, + ) # type: ignore - ############## No RPM/TPM passed, we do a random pick ################# - item = random.choice(healthy_deployments) - return item or item[0] - if deployment is None: + if isinstance(healthy_deployments, dict): + return healthy_deployments + + # filter out the deployments currently cooling down + deployments_to_remove = [] + # cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"] + cooldown_deployments = await self._async_get_cooldown_deployments() + verbose_router_logger.debug( + f"async cooldown deployments: {cooldown_deployments}" + ) + # Find deployments in model_list whose model_id is cooling down + for deployment in healthy_deployments: + deployment_id = deployment["model_info"]["id"] + if deployment_id in cooldown_deployments: + deployments_to_remove.append(deployment) + # remove unhealthy deployments from healthy deployments + for deployment in deployments_to_remove: + healthy_deployments.remove(deployment) + + # filter pre-call checks + _allowed_model_region = ( + request_kwargs.get("allowed_model_region") + if request_kwargs is not None + else None + ) + + if self.enable_pre_call_checks and messages is not None: + healthy_deployments = self._pre_call_checks( + model=model, + healthy_deployments=healthy_deployments, + messages=messages, + request_kwargs=request_kwargs, + ) + + if len(healthy_deployments) == 0: + if _allowed_model_region is None: + _allowed_model_region = "n/a" + raise ValueError( + f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. pre-call-checks={self.enable_pre_call_checks}, allowed_model_region={_allowed_model_region}" + ) + + if ( + self.routing_strategy == "usage-based-routing-v2" + and self.lowesttpm_logger_v2 is not None + ): + deployment = ( + await self.lowesttpm_logger_v2.async_get_available_deployments( + model_group=model, + healthy_deployments=healthy_deployments, # type: ignore + messages=messages, + input=input, + ) + ) + if ( + self.routing_strategy == "cost-based-routing" + and self.lowestcost_logger is not None + ): + deployment = ( + await self.lowestcost_logger.async_get_available_deployments( + model_group=model, + healthy_deployments=healthy_deployments, # type: ignore + messages=messages, + input=input, + ) + ) + elif self.routing_strategy == "simple-shuffle": + # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm + ############## Check if we can do a RPM/TPM based weighted pick ################# + rpm = healthy_deployments[0].get("litellm_params").get("rpm", None) + if rpm is not None: + # use weight-random pick if rpms provided + rpms = [ + m["litellm_params"].get("rpm", 0) for m in healthy_deployments + ] + verbose_router_logger.debug(f"\nrpms {rpms}") + total_rpm = sum(rpms) + weights = [rpm / total_rpm for rpm in rpms] + verbose_router_logger.debug(f"\n weights {weights}") + # Perform weighted random pick + selected_index = random.choices(range(len(rpms)), weights=weights)[ + 0 + ] + verbose_router_logger.debug(f"\n selected index, {selected_index}") + deployment = healthy_deployments[selected_index] + verbose_router_logger.info( + f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}" + ) + return deployment or deployment[0] + ############## Check if we can do a RPM/TPM based weighted pick ################# + tpm = healthy_deployments[0].get("litellm_params").get("tpm", None) + if tpm is not None: + # use weight-random pick if rpms provided + tpms = [ + m["litellm_params"].get("tpm", 0) for m in healthy_deployments + ] + verbose_router_logger.debug(f"\ntpms {tpms}") + total_tpm = sum(tpms) + weights = [tpm / total_tpm for tpm in tpms] + verbose_router_logger.debug(f"\n weights {weights}") + # Perform weighted random pick + selected_index = random.choices(range(len(tpms)), weights=weights)[ + 0 + ] + verbose_router_logger.debug(f"\n selected index, {selected_index}") + deployment = healthy_deployments[selected_index] + verbose_router_logger.info( + f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}" + ) + return deployment or deployment[0] + + ############## No RPM/TPM passed, we do a random pick ################# + item = random.choice(healthy_deployments) + return item or item[0] + if deployment is None: + verbose_router_logger.info( + f"get_available_deployment for model: {model}, No deployment available" + ) + raise ValueError( + f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}" + ) verbose_router_logger.info( - f"get_available_deployment for model: {model}, No deployment available" + f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" ) - raise ValueError( - f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}" - ) - verbose_router_logger.info( - f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" - ) - return deployment + return deployment + except Exception as e: + traceback_exception = traceback.format_exc() + # if router rejects call -> log to langfuse/otel/etc. + if request_kwargs is not None: + logging_obj = request_kwargs.get("litellm_logging_obj", None) + if logging_obj is not None: + ## LOGGING + threading.Thread( + target=logging_obj.failure_handler, + args=(e, traceback_exception), + ).start() # log response + # Handle any exceptions that might occur during streaming + asyncio.create_task( + logging_obj.async_failure_handler(e, traceback_exception) # type: ignore + ) + raise e def get_available_deployment( self, diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index fb28912493..c9e5501a8c 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -696,6 +696,18 @@ async def test_gemini_pro_function_calling_httpx(provider, sync_mode): pytest.fail("An unexpected exception occurred - {}".format(str(e))) +def vertex_httpx_mock_reject_prompt_post(*args, **kwargs): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.json.return_value = { + "promptFeedback": {"blockReason": "OTHER"}, + "usageMetadata": {"promptTokenCount": 6285, "totalTokenCount": 6285}, + } + + return mock_response + + # @pytest.mark.skip(reason="exhausted vertex quota. need to refactor to mock the call") def vertex_httpx_mock_post(url, data=None, json=None, headers=None): mock_response = MagicMock() @@ -817,8 +829,11 @@ def vertex_httpx_mock_post(url, data=None, json=None, headers=None): @pytest.mark.parametrize("provider", ["vertex_ai_beta"]) # "vertex_ai", +@pytest.mark.parametrize("content_filter_type", ["prompt", "response"]) # "vertex_ai", @pytest.mark.asyncio -async def test_gemini_pro_json_schema_httpx_content_policy_error(provider): +async def test_gemini_pro_json_schema_httpx_content_policy_error( + provider, content_filter_type +): load_vertex_ai_credentials() litellm.set_verbose = True messages = [ @@ -839,16 +854,20 @@ Using this JSON schema: client = HTTPHandler() - with patch.object(client, "post", side_effect=vertex_httpx_mock_post) as mock_call: - try: - response = completion( - model="vertex_ai_beta/gemini-1.5-flash", - messages=messages, - response_format={"type": "json_object"}, - client=client, - ) - except litellm.ContentPolicyViolationError as e: - pass + if content_filter_type == "prompt": + _side_effect = vertex_httpx_mock_reject_prompt_post + else: + _side_effect = vertex_httpx_mock_post + + with patch.object(client, "post", side_effect=_side_effect) as mock_call: + response = completion( + model="vertex_ai_beta/gemini-1.5-flash", + messages=messages, + response_format={"type": "json_object"}, + client=client, + ) + + assert response.choices[0].finish_reason == "content_filter" mock_call.assert_called_once() diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 830b3acd38..30ae1d0ab1 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries=3 +# litellm.num_retries = 3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" @@ -3470,6 +3470,28 @@ def test_completion_deep_infra_mistral(): # test_completion_deep_infra_mistral() +def test_completion_nvidia_nim(): + model_name = "nvidia_nim/databricks/dbrx-instruct" + try: + response = completion( + model=model_name, + messages=[ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ], + ) + # Add any assertions here to check the response + print(response) + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + except litellm.exceptions.Timeout as e: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + # Gemini tests @pytest.mark.parametrize( "model", diff --git a/litellm/tests/test_mock_request.py b/litellm/tests/test_mock_request.py index 7d670feb5b..48b054371f 100644 --- a/litellm/tests/test_mock_request.py +++ b/litellm/tests/test_mock_request.py @@ -58,3 +58,37 @@ async def test_async_mock_streaming_request(): assert ( complete_response == "LiteLLM is awesome" ), f"Unexpected response got {complete_response}" + + +def test_mock_request_n_greater_than_1(): + try: + model = "gpt-3.5-turbo" + messages = [{"role": "user", "content": "Hey, I'm a mock request"}] + response = litellm.mock_completion(model=model, messages=messages, n=5) + print("response: ", response) + + assert len(response.choices) == 5 + for choice in response.choices: + assert choice.message.content == "This is a mock request" + + except: + traceback.print_exc() + + +@pytest.mark.asyncio() +async def test_async_mock_streaming_request_n_greater_than_1(): + generator = await litellm.acompletion( + messages=[{"role": "user", "content": "Why is LiteLLM amazing?"}], + mock_response="LiteLLM is awesome", + stream=True, + model="gpt-3.5-turbo", + n=5, + ) + complete_response = "" + async for chunk in generator: + print(chunk) + # complete_response += chunk["choices"][0]["delta"]["content"] or "" + + # assert ( + # complete_response == "LiteLLM is awesome" + # ), f"Unexpected response got {complete_response}" diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py index 2e88143273..3237c8084a 100644 --- a/litellm/tests/test_router.py +++ b/litellm/tests/test_router.py @@ -732,7 +732,7 @@ def test_router_rpm_pre_call_check(): pytest.fail(f"Got unexpected exception on router! - {str(e)}") -def test_router_context_window_check_pre_call_check_in_group(): +def test_router_context_window_check_pre_call_check_in_group_custom_model_info(): """ - Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k) - Send a 5k prompt @@ -755,6 +755,61 @@ def test_router_context_window_check_pre_call_check_in_group(): "api_version": os.getenv("AZURE_API_VERSION"), "api_base": os.getenv("AZURE_API_BASE"), "base_model": "azure/gpt-35-turbo", + "mock_response": "Hello world 1!", + }, + "model_info": {"max_input_tokens": 100}, + }, + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "gpt-3.5-turbo-1106", + "api_key": os.getenv("OPENAI_API_KEY"), + "mock_response": "Hello world 2!", + }, + "model_info": {"max_input_tokens": 0}, + }, + ] + + router = Router(model_list=model_list, set_verbose=True, enable_pre_call_checks=True, num_retries=0) # type: ignore + + response = router.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Who was Alexander?"}, + ], + ) + + print(f"response: {response}") + + assert response.choices[0].message.content == "Hello world 1!" + except Exception as e: + pytest.fail(f"Got unexpected exception on router! - {str(e)}") + + +def test_router_context_window_check_pre_call_check(): + """ + - Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k) + - Send a 5k prompt + - Assert it works + """ + import os + + from large_text import text + + litellm.set_verbose = False + + print(f"len(text): {len(text)}") + try: + model_list = [ + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + "base_model": "azure/gpt-35-turbo", + "mock_response": "Hello world 1!", }, }, { @@ -762,6 +817,7 @@ def test_router_context_window_check_pre_call_check_in_group(): "litellm_params": { # params for litellm completion/embedding call "model": "gpt-3.5-turbo-1106", "api_key": os.getenv("OPENAI_API_KEY"), + "mock_response": "Hello world 2!", }, }, ] @@ -777,6 +833,8 @@ def test_router_context_window_check_pre_call_check_in_group(): ) print(f"response: {response}") + + assert response.choices[0].message.content == "Hello world 2!" except Exception as e: pytest.fail(f"Got unexpected exception on router! - {str(e)}") diff --git a/litellm/tests/test_router_cooldowns.py b/litellm/tests/test_router_cooldowns.py index 35095bb2cf..3eef6e5423 100644 --- a/litellm/tests/test_router_cooldowns.py +++ b/litellm/tests/test_router_cooldowns.py @@ -1,18 +1,26 @@ #### What this tests #### # This tests calling router with fallback models -import sys, os, time -import traceback, asyncio +import asyncio +import os +import sys +import time +import traceback + import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import openai + import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger -import openai, httpx @pytest.mark.asyncio @@ -62,3 +70,45 @@ async def test_cooldown_badrequest_error(): assert response is not None print(response) + + +@pytest.mark.asyncio +async def test_dynamic_cooldowns(): + """ + Assert kwargs for completion/embedding have 'cooldown_time' as a litellm_param + """ + # litellm.set_verbose = True + tmp_mock = MagicMock() + + litellm.failure_callback = [tmp_mock] + + router = Router( + model_list=[ + { + "model_name": "my-fake-model", + "litellm_params": { + "model": "openai/gpt-1", + "api_key": "my-key", + "mock_response": Exception("this is an error"), + }, + } + ], + cooldown_time=60, + ) + + try: + _ = router.completion( + model="my-fake-model", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + cooldown_time=0, + num_retries=0, + ) + except Exception: + pass + + tmp_mock.assert_called_once() + + print(tmp_mock.call_count) + + assert "cooldown_time" in tmp_mock.call_args[0][0]["litellm_params"] + assert tmp_mock.call_args[0][0]["litellm_params"]["cooldown_time"] == 0 diff --git a/litellm/tests/test_router_fallbacks.py b/litellm/tests/test_router_fallbacks.py index 99d2a600c8..2c552a64bf 100644 --- a/litellm/tests/test_router_fallbacks.py +++ b/litellm/tests/test_router_fallbacks.py @@ -1129,7 +1129,9 @@ async def test_router_content_policy_fallbacks( mock_response = Exception("content filtering policy") else: mock_response = litellm.ModelResponse( - choices=[litellm.Choices(finish_reason="content_filter")] + choices=[litellm.Choices(finish_reason="content_filter")], + model="gpt-3.5-turbo", + usage=litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10), ) router = Router( model_list=[ diff --git a/litellm/tests/test_secret_detect_hook.py b/litellm/tests/test_secret_detect_hook.py new file mode 100644 index 0000000000..a1bf10ebad --- /dev/null +++ b/litellm/tests/test_secret_detect_hook.py @@ -0,0 +1,216 @@ +# What is this? +## This tests the llm guard integration + +import asyncio +import os +import random + +# What is this? +## Unit test for presidio pii masking +import sys +import time +import traceback +from datetime import datetime + +from dotenv import load_dotenv + +load_dotenv() +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import pytest + +import litellm +from litellm import Router, mock_completion +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.enterprise.enterprise_hooks.secret_detection import ( + _ENTERPRISE_SecretDetection, +) +from litellm.proxy.utils import ProxyLogging, hash_token + +### UNIT TESTS FOR OpenAI Moderation ### + + +@pytest.mark.asyncio +async def test_basic_secret_detection_chat(): + """ + Tests to see if secret detection hook will mask api keys + + + It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef' + """ + secret_instance = _ENTERPRISE_SecretDetection() + _api_key = "sk-12345" + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key) + local_cache = DualCache() + + from litellm.proxy.proxy_server import llm_router + + test_data = { + "messages": [ + { + "role": "user", + "content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'", + }, + { + "role": "assistant", + "content": "Hello! I'm doing well. How can I assist you today?", + }, + { + "role": "user", + "content": "this is my OPENAI_API_KEY = 'sk_1234567890abcdef'", + }, + {"role": "user", "content": "i think it is +1 412-555-5555"}, + ], + "model": "gpt-3.5-turbo", + } + + await secret_instance.async_pre_call_hook( + cache=local_cache, + data=test_data, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + print( + "test data after running pre_call_hook: Expect all API Keys to be masked", + test_data, + ) + + assert test_data == { + "messages": [ + {"role": "user", "content": "Hey, how's it going, API_KEY = '[REDACTED]'"}, + { + "role": "assistant", + "content": "Hello! I'm doing well. How can I assist you today?", + }, + {"role": "user", "content": "this is my OPENAI_API_KEY = '[REDACTED]'"}, + {"role": "user", "content": "i think it is +1 412-555-5555"}, + ], + "model": "gpt-3.5-turbo", + }, "Expect all API Keys to be masked" + + +@pytest.mark.asyncio +async def test_basic_secret_detection_text_completion(): + """ + Tests to see if secret detection hook will mask api keys + + + It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef' + """ + secret_instance = _ENTERPRISE_SecretDetection() + _api_key = "sk-12345" + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key) + local_cache = DualCache() + + from litellm.proxy.proxy_server import llm_router + + test_data = { + "prompt": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef', my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather", + "model": "gpt-3.5-turbo", + } + + await secret_instance.async_pre_call_hook( + cache=local_cache, + data=test_data, + user_api_key_dict=user_api_key_dict, + call_type="completion", + ) + + test_data == { + "prompt": "Hey, how's it going, API_KEY = '[REDACTED]', my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather", + "model": "gpt-3.5-turbo", + } + print( + "test data after running pre_call_hook: Expect all API Keys to be masked", + test_data, + ) + + +@pytest.mark.asyncio +async def test_basic_secret_detection_embeddings(): + """ + Tests to see if secret detection hook will mask api keys + + + It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef' + """ + secret_instance = _ENTERPRISE_SecretDetection() + _api_key = "sk-12345" + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key) + local_cache = DualCache() + + from litellm.proxy.proxy_server import llm_router + + test_data = { + "input": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef', my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather", + "model": "gpt-3.5-turbo", + } + + await secret_instance.async_pre_call_hook( + cache=local_cache, + data=test_data, + user_api_key_dict=user_api_key_dict, + call_type="embedding", + ) + + assert test_data == { + "input": "Hey, how's it going, API_KEY = '[REDACTED]', my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather", + "model": "gpt-3.5-turbo", + } + print( + "test data after running pre_call_hook: Expect all API Keys to be masked", + test_data, + ) + + +@pytest.mark.asyncio +async def test_basic_secret_detection_embeddings_list(): + """ + Tests to see if secret detection hook will mask api keys + + + It should mask the following API_KEY = 'sk_1234567890abcdef' and OPENAI_API_KEY = 'sk_1234567890abcdef' + """ + secret_instance = _ENTERPRISE_SecretDetection() + _api_key = "sk-12345" + _api_key = hash_token("sk-12345") + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key) + local_cache = DualCache() + + from litellm.proxy.proxy_server import llm_router + + test_data = { + "input": [ + "hey", + "how's it going, API_KEY = 'sk_1234567890abcdef'", + "my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather", + ], + "model": "gpt-3.5-turbo", + } + + await secret_instance.async_pre_call_hook( + cache=local_cache, + data=test_data, + user_api_key_dict=user_api_key_dict, + call_type="embedding", + ) + + print( + "test data after running pre_call_hook: Expect all API Keys to be masked", + test_data, + ) + assert test_data == { + "input": [ + "hey", + "how's it going, API_KEY = '[REDACTED]'", + "my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather", + ], + "model": "gpt-3.5-turbo", + } diff --git a/litellm/tests/test_spend_calculate_endpoint.py b/litellm/tests/test_spend_calculate_endpoint.py new file mode 100644 index 0000000000..8bdd4a54d8 --- /dev/null +++ b/litellm/tests/test_spend_calculate_endpoint.py @@ -0,0 +1,141 @@ +import os +import sys + +import pytest +from dotenv import load_dotenv +from fastapi import Request +from fastapi.routing import APIRoute + +import litellm +from litellm.proxy._types import SpendCalculateRequest +from litellm.proxy.spend_tracking.spend_management_endpoints import calculate_spend +from litellm.router import Router + +# this file is to test litellm/proxy + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + + +@pytest.mark.asyncio +async def test_spend_calc_model_messages(): + cost_obj = await calculate_spend( + request=SpendCalculateRequest( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "What is the capital of France?"}, + ], + ) + ) + + print("calculated cost", cost_obj) + cost = cost_obj["cost"] + assert cost > 0.0 + + +@pytest.mark.asyncio +async def test_spend_calc_model_on_router_messages(): + from litellm.proxy.proxy_server import llm_router as init_llm_router + + temp_llm_router = Router( + model_list=[ + { + "model_name": "special-llama-model", + "litellm_params": { + "model": "groq/llama3-8b-8192", + }, + } + ] + ) + + setattr(litellm.proxy.proxy_server, "llm_router", temp_llm_router) + + cost_obj = await calculate_spend( + request=SpendCalculateRequest( + model="special-llama-model", + messages=[ + {"role": "user", "content": "What is the capital of France?"}, + ], + ) + ) + + print("calculated cost", cost_obj) + _cost = cost_obj["cost"] + + assert _cost > 0.0 + + # set router to init value + setattr(litellm.proxy.proxy_server, "llm_router", init_llm_router) + + +@pytest.mark.asyncio +async def test_spend_calc_using_response(): + cost_obj = await calculate_spend( + request=SpendCalculateRequest( + completion_response={ + "id": "chatcmpl-3bc7abcd-f70b-48ab-a16c-dfba0b286c86", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Yooo! What's good?", + "role": "assistant", + }, + } + ], + "created": "1677652288", + "model": "groq/llama3-8b-8192", + "object": "chat.completion", + "system_fingerprint": "fp_873a560973", + "usage": { + "completion_tokens": 8, + "prompt_tokens": 12, + "total_tokens": 20, + }, + } + ) + ) + + print("calculated cost", cost_obj) + cost = cost_obj["cost"] + assert cost > 0.0 + + +@pytest.mark.asyncio +async def test_spend_calc_model_alias_on_router_messages(): + from litellm.proxy.proxy_server import llm_router as init_llm_router + + temp_llm_router = Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": { + "model": "gpt-4o", + }, + } + ], + model_group_alias={ + "gpt4o": "gpt-4o", + }, + ) + + setattr(litellm.proxy.proxy_server, "llm_router", temp_llm_router) + + cost_obj = await calculate_spend( + request=SpendCalculateRequest( + model="gpt4o", + messages=[ + {"role": "user", "content": "What is the capital of France?"}, + ], + ) + ) + + print("calculated cost", cost_obj) + _cost = cost_obj["cost"] + + assert _cost > 0.0 + + # set router to init value + setattr(litellm.proxy.proxy_server, "llm_router", init_llm_router) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 1612f8761f..2dda57c2e9 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -227,9 +227,9 @@ class PromptFeedback(TypedDict): blockReasonMessage: str -class UsageMetadata(TypedDict): - promptTokenCount: int - totalTokenCount: int +class UsageMetadata(TypedDict, total=False): + promptTokenCount: Required[int] + totalTokenCount: Required[int] candidatesTokenCount: int diff --git a/litellm/utils.py b/litellm/utils.py index 0849ba3a26..beae7ba4ab 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2017,6 +2017,7 @@ def get_litellm_params( input_cost_per_token=None, output_cost_per_token=None, output_cost_per_second=None, + cooldown_time=None, ): litellm_params = { "acompletion": acompletion, @@ -2039,6 +2040,7 @@ def get_litellm_params( "input_cost_per_second": input_cost_per_second, "output_cost_per_token": output_cost_per_token, "output_cost_per_second": output_cost_per_second, + "cooldown_time": cooldown_time, } return litellm_params @@ -2410,6 +2412,7 @@ def get_optional_params( and custom_llm_provider != "anyscale" and custom_llm_provider != "together_ai" and custom_llm_provider != "groq" + and custom_llm_provider != "nvidia_nim" and custom_llm_provider != "deepseek" and custom_llm_provider != "codestral" and custom_llm_provider != "mistral" @@ -2608,7 +2611,15 @@ def get_optional_params( optional_params["top_p"] = top_p if stop is not None: optional_params["stop_sequences"] = stop - elif custom_llm_provider == "huggingface" or custom_llm_provider == "predibase": + elif custom_llm_provider == "predibase": + supported_params = get_supported_openai_params( + model=model, custom_llm_provider=custom_llm_provider + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.PredibaseConfig().map_openai_params( + non_default_params=non_default_params, optional_params=optional_params + ) + elif custom_llm_provider == "huggingface": ## check if unsupported param passed in supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider @@ -3060,6 +3071,14 @@ def get_optional_params( optional_params = litellm.DatabricksConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params ) + elif custom_llm_provider == "nvidia_nim": + supported_params = get_supported_openai_params( + model=model, custom_llm_provider=custom_llm_provider + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.NvidiaNimConfig().map_openai_params( + non_default_params=non_default_params, optional_params=optional_params + ) elif custom_llm_provider == "groq": supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider @@ -3626,6 +3645,8 @@ def get_supported_openai_params( return litellm.OllamaChatConfig().get_supported_openai_params() elif custom_llm_provider == "anthropic": return litellm.AnthropicConfig().get_supported_openai_params() + elif custom_llm_provider == "nvidia_nim": + return litellm.NvidiaNimConfig().get_supported_openai_params() elif custom_llm_provider == "groq": return [ "temperature", @@ -3986,6 +4007,10 @@ def get_llm_provider( # groq is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.groq.com/openai/v1 api_base = "https://api.groq.com/openai/v1" dynamic_api_key = get_secret("GROQ_API_KEY") + elif custom_llm_provider == "nvidia_nim": + # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 + api_base = "https://integrate.api.nvidia.com/v1" + dynamic_api_key = get_secret("NVIDIA_NIM_API_KEY") elif custom_llm_provider == "codestral": # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1 api_base = "https://codestral.mistral.ai/v1" @@ -4087,6 +4112,9 @@ def get_llm_provider( elif endpoint == "api.groq.com/openai/v1": custom_llm_provider = "groq" dynamic_api_key = get_secret("GROQ_API_KEY") + elif endpoint == "https://integrate.api.nvidia.com/v1": + custom_llm_provider = "nvidia_nim" + dynamic_api_key = get_secret("NVIDIA_NIM_API_KEY") elif endpoint == "https://codestral.mistral.ai/v1": custom_llm_provider = "codestral" dynamic_api_key = get_secret("CODESTRAL_API_KEY") @@ -4900,6 +4928,11 @@ def validate_environment(model: Optional[str] = None) -> dict: keys_in_environment = True else: missing_keys.append("GROQ_API_KEY") + elif custom_llm_provider == "nvidia_nim": + if "NVIDIA_NIM_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("NVIDIA_NIM_API_KEY") elif ( custom_llm_provider == "codestral" or custom_llm_provider == "text-completion-codestral" @@ -5914,6 +5947,7 @@ def exception_type( ) else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors + # exception_mapping_worked = True raise APIConnectionError( message=f"APIConnectionError: {exception_provider} - {message}", llm_provider=custom_llm_provider, @@ -6067,6 +6101,14 @@ def exception_type( model=model, llm_provider="replicate", ) + elif original_exception.status_code == 422: + exception_mapping_worked = True + raise UnprocessableEntityError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) elif original_exception.status_code == 429: exception_mapping_worked = True raise RateLimitError( @@ -6125,13 +6167,6 @@ def exception_type( response=original_exception.response, litellm_debug_info=extra_information, ) - if "Request failed during generation" in error_str: - # this is an internal server error from predibase - raise litellm.InternalServerError( - message=f"PredibaseException - {error_str}", - llm_provider="predibase", - model=model, - ) elif hasattr(original_exception, "status_code"): if original_exception.status_code == 500: exception_mapping_worked = True @@ -6169,7 +6204,10 @@ def exception_type( llm_provider=custom_llm_provider, litellm_debug_info=extra_information, ) - elif original_exception.status_code == 422: + elif ( + original_exception.status_code == 422 + or original_exception.status_code == 424 + ): exception_mapping_worked = True raise BadRequestError( message=f"PredibaseException - {original_exception.message}", @@ -6438,7 +6476,11 @@ def exception_type( ), litellm_debug_info=extra_information, ) - elif "The response was blocked." in error_str: + elif ( + "The response was blocked." in error_str + or "Output blocked by content filtering policy" + in error_str # anthropic on vertex ai + ): exception_mapping_worked = True raise ContentPolicyViolationError( message=f"VertexAIException ContentPolicyViolationError - {error_str}", @@ -7460,6 +7502,9 @@ def exception_type( if exception_mapping_worked: raise e else: + for error_type in litellm.LITELLM_EXCEPTION_TYPES: + if isinstance(e, error_type): + raise e # it's already mapped raise APIConnectionError( message="{}\n{}".format(original_exception, traceback.format_exc()), llm_provider="", @@ -9696,18 +9741,45 @@ class TextCompletionStreamWrapper: raise StopAsyncIteration -def mock_completion_streaming_obj(model_response, mock_response, model): +def mock_completion_streaming_obj( + model_response, mock_response, model, n: Optional[int] = None +): for i in range(0, len(mock_response), 3): - completion_obj = {"role": "assistant", "content": mock_response[i : i + 3]} - model_response.choices[0].delta = completion_obj + completion_obj = Delta(role="assistant", content=mock_response[i : i + 3]) + if n is None: + model_response.choices[0].delta = completion_obj + else: + _all_choices = [] + for j in range(n): + _streaming_choice = litellm.utils.StreamingChoices( + index=j, + delta=litellm.utils.Delta( + role="assistant", content=mock_response[i : i + 3] + ), + ) + _all_choices.append(_streaming_choice) + model_response.choices = _all_choices yield model_response -async def async_mock_completion_streaming_obj(model_response, mock_response, model): +async def async_mock_completion_streaming_obj( + model_response, mock_response, model, n: Optional[int] = None +): for i in range(0, len(mock_response), 3): completion_obj = Delta(role="assistant", content=mock_response[i : i + 3]) - model_response.choices[0].delta = completion_obj - model_response.choices[0].finish_reason = "stop" + if n is None: + model_response.choices[0].delta = completion_obj + else: + _all_choices = [] + for j in range(n): + _streaming_choice = litellm.utils.StreamingChoices( + index=j, + delta=litellm.utils.Delta( + role="assistant", content=mock_response[i : i + 3] + ), + ) + _all_choices.append(_streaming_choice) + model_response.choices = _all_choices yield model_response diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ef07d87ccb..d7a7a7dc80 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -887,7 +887,7 @@ "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 0.00000005, - "output_cost_per_token": 0.00000010, + "output_cost_per_token": 0.00000008, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -906,8 +906,8 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 0.00000027, - "output_cost_per_token": 0.00000027, + "input_cost_per_token": 0.00000024, + "output_cost_per_token": 0.00000024, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -916,8 +916,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.00000010, - "output_cost_per_token": 0.00000010, + "input_cost_per_token": 0.00000007, + "output_cost_per_token": 0.00000007, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true @@ -2073,6 +2073,30 @@ "supports_function_calling": true, "supports_vision": true }, + "openrouter/anthropic/claude-3-haiku-20240307": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 264 + }, + "openrouter/anthropic/claude-3.5-sonnet": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "openrouter/anthropic/claude-3-sonnet": { "max_tokens": 200000, "input_cost_per_token": 0.000003, diff --git a/pyproject.toml b/pyproject.toml index fc3526dcc5..321f44b23b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.40.25" +version = "1.40.27" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -90,7 +90,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.40.25" +version = "1.40.27" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index fbf2bfc1d1..00d3802da5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,8 @@ azure-identity==1.16.1 # for azure content safety opentelemetry-api==1.25.0 opentelemetry-sdk==1.25.0 opentelemetry-exporter-otlp==1.25.0 +detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests +cryptography==42.0.7 ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.0 # for env