diff --git a/.circleci/config.yml b/.circleci/config.yml index 4cfb68f647..40d498d6e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,6 +212,7 @@ jobs: -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index cfab07c22a..e3758266a1 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -10,22 +10,23 @@ Interested in Enterprise? Schedule a meeting with us here 👉 This covers: - **Enterprise Features** - **Security** - - ✅ [SSO for Admin UI](./ui.md#✨-enterprise-features) - - ✅ [Audit Logs with retention policy](#audit-logs) + - ✅ [SSO for Admin UI](./proxy/ui#✨-enterprise-features) + - ✅ [Audit Logs with retention policy](./proxy/enterprise#audit-logs) - ✅ [JWT-Auth](../docs/proxy/token_auth.md) - - ✅ [Control available public, private routes](#control-available-public-private-routes) - - ✅ [[BETA] AWS Key Manager v2 - Key Decryption](#beta-aws-key-manager---key-decryption) - - ✅ [Use LiteLLM keys/authentication on Pass Through Endpoints](pass_through#✨-enterprise---use-litellm-keysauthentication-on-pass-through-endpoints) - - ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests) + - ✅ [Control available public, private routes](./proxy/enterprise#control-available-public-private-routes) + - ✅ [[BETA] AWS Key Manager v2 - Key Decryption](./proxy/enterprise#beta-aws-key-manager---key-decryption) + - ✅ [Use LiteLLM keys/authentication on Pass Through Endpoints](./proxy/pass_through#✨-enterprise---use-litellm-keysauthentication-on-pass-through-endpoints) + - ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](./proxy/enterprise#enforce-required-params-for-llm-requests) - **Spend Tracking** - - ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags) + - ✅ [Tracking Spend for Custom Tags](./proxy/enterprise#tracking-spend-for-custom-tags) + - ✅ [API Endpoints to get Spend Reports per Team, API Key, Customer](./proxy/cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend) - **Guardrails, PII Masking, Content Moderation** - - ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](#content-moderation) - - ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai) + - ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](./proxy/enterprise#content-moderation) + - ✅ [Prompt Injection Detection (with LakeraAI API)](./proxy/enterprise#prompt-injection-detection---lakeraai) - ✅ Reject calls from Blocked User list - ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors) - **Custom Branding** - - ✅ [Custom Branding + Routes on Swagger Docs](#swagger-docs---custom-routes--branding) + - ✅ [Custom Branding + Routes on Swagger Docs](./proxy/enterprise#swagger-docs---custom-routes--branding) - ✅ [Public Model Hub](../docs/proxy/enterprise.md#public-model-hub) - ✅ [Custom Email Branding](../docs/proxy/email.md#customizing-email-branding) - ✅ **Feature Prioritization** diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index f380a6a50e..b72dac10bc 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -549,6 +549,10 @@ response = completion( This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues) + +Experimental - 2024-Jun-23: + `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` will be extracted from boto3.client and be passed into the httpx client + ::: Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth. diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index 3ccf8f383a..fe3a462508 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -117,6 +117,8 @@ That's IT. Now Verify your spend was tracked +Expect to see `x-litellm-response-cost` in the response headers with calculated cost + @@ -145,16 +147,16 @@ Navigate to the Usage Tab on the LiteLLM UI (found on https://your-proxy-endpoin -## API Endpoints to get Spend -#### Getting Spend Reports - To Charge Other Teams, Customers - -Use the `/global/spend/report` endpoint to get daily spend report per -- team -- customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm) - +## ✨ (Enterprise) API Endpoints to get Spend +#### Getting Spend Reports - To Charge Other Teams, Customers + +Use the `/global/spend/report` endpoint to get daily spend report per +- Team +- Customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm) +- [LiteLLM API key](virtual_keys.md) @@ -337,6 +339,61 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end ``` + + + + + +👉 Key Change: Specify `group_by=api_key` + + +```shell +curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=api_key' \ + -H 'Authorization: Bearer sk-1234' +``` + +##### Example Response + + +```shell +[ + { + "api_key": "ad64768847d05d978d62f623d872bff0f9616cc14b9c1e651c84d14fe3b9f539", + "total_cost": 0.0002157, + "total_input_tokens": 45.0, + "total_output_tokens": 1375.0, + "model_details": [ + { + "model": "gpt-3.5-turbo", + "total_cost": 0.0001095, + "total_input_tokens": 9, + "total_output_tokens": 70 + }, + { + "model": "llama3-8b-8192", + "total_cost": 0.0001062, + "total_input_tokens": 36, + "total_output_tokens": 1305 + } + ] + }, + { + "api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b", + "total_cost": 0.00012924, + "total_input_tokens": 36.0, + "total_output_tokens": 1593.0, + "model_details": [ + { + "model": "llama3-8b-8192", + "total_cost": 0.00012924, + "total_input_tokens": 36, + "total_output_tokens": 1593 + } + ] + } +] +``` + diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index d580f58b6b..e061a917e2 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -22,6 +22,7 @@ Features: - ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests) - **Spend Tracking** - ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags) + - ✅ [API Endpoints to get Spend Reports per Team, API Key, Customer](cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend) - **Guardrails, PII Masking, Content Moderation** - ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](#content-moderation) - ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai) diff --git a/docs/my-website/img/response_cost_img.png b/docs/my-website/img/response_cost_img.png index 9f466b3fbb..2fa9c20095 100644 Binary files a/docs/my-website/img/response_cost_img.png and b/docs/my-website/img/response_cost_img.png differ diff --git a/litellm/__init__.py b/litellm/__init__.py index 05497ee729..0fa822a98e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -800,7 +800,11 @@ from .llms.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.ollama_chat import OllamaChatConfig from .llms.maritalk import MaritTalkConfig -from .llms.bedrock_httpx import AmazonCohereChatConfig, AmazonConverseConfig +from .llms.bedrock_httpx import ( + AmazonCohereChatConfig, + AmazonConverseConfig, + BEDROCK_CONVERSE_MODELS, +) from .llms.bedrock import ( AmazonTitanConfig, AmazonAI21Config, diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index eae8b8e22a..983ec39428 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -311,22 +311,17 @@ class LangFuseLogger: try: tags = [] - try: - metadata = copy.deepcopy( - metadata - ) # Avoid modifying the original metadata - except: - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, dict) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = copy.deepcopy(value) - metadata = new_metadata + new_metadata = {} + for key, value in metadata.items(): + if ( + isinstance(value, list) + or isinstance(value, dict) + or isinstance(value, str) + or isinstance(value, int) + or isinstance(value, float) + ): + new_metadata[key] = copy.deepcopy(value) + metadata = new_metadata supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3") supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3") diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py new file mode 100644 index 0000000000..5ac26c7ae5 --- /dev/null +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -0,0 +1,40 @@ +import json +from typing import Optional + + +def get_error_message(error_obj) -> Optional[str]: + """ + OpenAI Returns Error message that is nested, this extract the message + + Example: + { + 'request': "", + 'message': "Error code: 400 - {\'error\': {\'message\': \"Invalid 'temperature': decimal above maximum value. Expected a value <= 2, but got 200 instead.\", 'type': 'invalid_request_error', 'param': 'temperature', 'code': 'decimal_above_max_value'}}", + 'body': { + 'message': "Invalid 'temperature': decimal above maximum value. Expected a value <= 2, but got 200 instead.", + 'type': 'invalid_request_error', + 'param': 'temperature', + 'code': 'decimal_above_max_value' + }, + 'code': 'decimal_above_max_value', + 'param': 'temperature', + 'type': 'invalid_request_error', + 'response': "", + 'status_code': 400, + 'request_id': 'req_f287898caa6364cd42bc01355f74dd2a' + } + """ + try: + # First, try to access the message directly from the 'body' key + if error_obj is None: + return None + + if hasattr(error_obj, "body"): + _error_obj_body = getattr(error_obj, "body") + if isinstance(_error_obj_body, dict): + return _error_obj_body.get("message") + + # If all else fails, return None + return None + except Exception as e: + return None diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py index 14abec784f..8aefc63cfa 100644 --- a/litellm/llms/bedrock_httpx.py +++ b/litellm/llms/bedrock_httpx.py @@ -60,6 +60,16 @@ from .prompt_templates.factory import ( prompt_factory, ) +BEDROCK_CONVERSE_MODELS = [ + "anthropic.claude-3-opus-20240229-v1:0", + "anthropic.claude-3-sonnet-20240229-v1:0", + "anthropic.claude-3-haiku-20240307-v1:0", + "anthropic.claude-v2", + "anthropic.claude-v2:1", + "anthropic.claude-v1", + "anthropic.claude-instant-v1", +] + iam_cache = DualCache() @@ -305,6 +315,7 @@ class BedrockLLM(BaseLLM): self, aws_access_key_id: Optional[str] = None, aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, aws_region_name: Optional[str] = None, aws_session_name: Optional[str] = None, aws_profile_name: Optional[str] = None, @@ -320,6 +331,7 @@ class BedrockLLM(BaseLLM): params_to_check: List[Optional[str]] = [ aws_access_key_id, aws_secret_access_key, + aws_session_token, aws_region_name, aws_session_name, aws_profile_name, @@ -337,6 +349,7 @@ class BedrockLLM(BaseLLM): ( aws_access_key_id, aws_secret_access_key, + aws_session_token, aws_region_name, aws_session_name, aws_profile_name, @@ -430,6 +443,19 @@ class BedrockLLM(BaseLLM): client = boto3.Session(profile_name=aws_profile_name) return client.get_credentials() + elif ( + aws_access_key_id is not None + and aws_secret_access_key is not None + and aws_session_token is not None + ): ### CHECK FOR AWS SESSION TOKEN ### + from botocore.credentials import Credentials + + credentials = Credentials( + access_key=aws_access_key_id, + secret_key=aws_secret_access_key, + token=aws_session_token, + ) + return credentials else: session = boto3.Session( aws_access_key_id=aws_access_key_id, @@ -734,9 +760,10 @@ class BedrockLLM(BaseLLM): provider = model.split(".")[0] ## CREDENTIALS ## - # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_session_token = optional_params.pop("aws_session_token", None) aws_region_name = optional_params.pop("aws_region_name", None) aws_role_name = optional_params.pop("aws_role_name", None) aws_session_name = optional_params.pop("aws_session_name", None) @@ -768,6 +795,7 @@ class BedrockLLM(BaseLLM): credentials: Credentials = self.get_credentials( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, aws_region_name=aws_region_name, aws_session_name=aws_session_name, aws_profile_name=aws_profile_name, @@ -1422,6 +1450,7 @@ class BedrockConverseLLM(BaseLLM): self, aws_access_key_id: Optional[str] = None, aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, aws_region_name: Optional[str] = None, aws_session_name: Optional[str] = None, aws_profile_name: Optional[str] = None, @@ -1437,6 +1466,7 @@ class BedrockConverseLLM(BaseLLM): params_to_check: List[Optional[str]] = [ aws_access_key_id, aws_secret_access_key, + aws_session_token, aws_region_name, aws_session_name, aws_profile_name, @@ -1454,6 +1484,7 @@ class BedrockConverseLLM(BaseLLM): ( aws_access_key_id, aws_secret_access_key, + aws_session_token, aws_region_name, aws_session_name, aws_profile_name, @@ -1547,6 +1578,19 @@ class BedrockConverseLLM(BaseLLM): client = boto3.Session(profile_name=aws_profile_name) return client.get_credentials() + elif ( + aws_access_key_id is not None + and aws_secret_access_key is not None + and aws_session_token is not None + ): ### CHECK FOR AWS SESSION TOKEN ### + from botocore.credentials import Credentials + + credentials = Credentials( + access_key=aws_access_key_id, + secret_key=aws_secret_access_key, + token=aws_session_token, + ) + return credentials else: session = boto3.Session( aws_access_key_id=aws_access_key_id, @@ -1682,6 +1726,7 @@ class BedrockConverseLLM(BaseLLM): # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_session_token = optional_params.pop("aws_session_token", None) aws_region_name = optional_params.pop("aws_region_name", None) aws_role_name = optional_params.pop("aws_role_name", None) aws_session_name = optional_params.pop("aws_session_name", None) @@ -1713,6 +1758,7 @@ class BedrockConverseLLM(BaseLLM): credentials: Credentials = self.get_credentials( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, aws_region_name=aws_region_name, aws_session_name=aws_session_name, aws_profile_name=aws_profile_name, diff --git a/litellm/llms/custom_httpx/azure_dall_e_2.py b/litellm/llms/custom_httpx/azure_dall_e_2.py index f361ede5bf..a6726eb98c 100644 --- a/litellm/llms/custom_httpx/azure_dall_e_2.py +++ b/litellm/llms/custom_httpx/azure_dall_e_2.py @@ -1,4 +1,8 @@ -import time, json, httpx, asyncio +import asyncio +import json +import time + +import httpx class AsyncCustomHTTPTransport(httpx.AsyncHTTPTransport): @@ -7,15 +11,18 @@ class AsyncCustomHTTPTransport(httpx.AsyncHTTPTransport): """ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - if "images/generations" in request.url.path and request.url.params[ - "api-version" - ] in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict - "2023-06-01-preview", - "2023-07-01-preview", - "2023-08-01-preview", - "2023-09-01-preview", - "2023-10-01-preview", - ]: + _api_version = request.url.params.get("api-version", "") + if ( + "images/generations" in request.url.path + and _api_version + in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict + "2023-06-01-preview", + "2023-07-01-preview", + "2023-08-01-preview", + "2023-09-01-preview", + "2023-10-01-preview", + ] + ): request.url = request.url.copy_with( path="/openai/images/generations:submit" ) @@ -77,15 +84,18 @@ class CustomHTTPTransport(httpx.HTTPTransport): self, request: httpx.Request, ) -> httpx.Response: - if "images/generations" in request.url.path and request.url.params[ - "api-version" - ] in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict - "2023-06-01-preview", - "2023-07-01-preview", - "2023-08-01-preview", - "2023-09-01-preview", - "2023-10-01-preview", - ]: + _api_version = request.url.params.get("api-version", "") + if ( + "images/generations" in request.url.path + and _api_version + in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict + "2023-06-01-preview", + "2023-07-01-preview", + "2023-08-01-preview", + "2023-09-01-preview", + "2023-10-01-preview", + ] + ): request.url = request.url.copy_with( path="/openai/images/generations:submit" ) diff --git a/litellm/main.py b/litellm/main.py index 9945e1b955..10bcbe9e3f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2200,13 +2200,22 @@ def completion( # boto3 reads keys from .env custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - if ( - "aws_bedrock_client" in optional_params - ): # use old bedrock flow for aws_bedrock_client users. - response = bedrock.completion( + if "aws_bedrock_client" in optional_params: + # Extract credentials for legacy boto3 client and pass thru to httpx + aws_bedrock_client = optional_params.pop("aws_bedrock_client") + creds = aws_bedrock_client._get_credentials().get_frozen_credentials() + if creds.access_key: + optional_params["aws_access_key_id"] = creds.access_key + if creds.secret_key: + optional_params["aws_secret_access_key"] = creds.secret_key + if creds.token: + optional_params["aws_session_token"] = creds.token + + if model in litellm.BEDROCK_CONVERSE_MODELS: + response = bedrock_converse_chat_completion.completion( model=model, messages=messages, - custom_prompt_dict=litellm.custom_prompt_dict, + custom_prompt_dict=custom_prompt_dict, model_response=model_response, print_verbose=print_verbose, optional_params=optional_params, @@ -2216,63 +2225,27 @@ def completion( logging_obj=logging, extra_headers=extra_headers, timeout=timeout, + acompletion=acompletion, + client=client, + ) + else: + response = bedrock_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + acompletion=acompletion, + client=client, ) - if ( - "stream" in optional_params - and optional_params["stream"] == True - and not isinstance(response, CustomStreamWrapper) - ): - # don't try to access stream object, - if "ai21" in model: - response = CustomStreamWrapper( - response, - model, - custom_llm_provider="bedrock", - logging_obj=logging, - ) - else: - response = CustomStreamWrapper( - iter(response), - model, - custom_llm_provider="bedrock", - logging_obj=logging, - ) - else: - if model.startswith("anthropic"): - response = bedrock_converse_chat_completion.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=encoding, - logging_obj=logging, - extra_headers=extra_headers, - timeout=timeout, - acompletion=acompletion, - client=client, - ) - else: - response = bedrock_chat_completion.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=encoding, - logging_obj=logging, - extra_headers=extra_headers, - timeout=timeout, - acompletion=acompletion, - client=client, - ) if optional_params.get("stream", False): ## LOGGING logging.post_call( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 640c7695a0..1f1aaf0eea 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1622,7 +1622,7 @@ class ProxyException(Exception): } -class CommonProxyErrors(enum.Enum): +class CommonProxyErrors(str, enum.Enum): db_not_connected_error = "DB not connected" no_llm_router = "No models configured on proxy" not_allowed_access = "Admin-only endpoint. Not allowed to access this." diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ff4b1e6633..0577ec0a04 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -593,7 +593,7 @@ async def _PROXY_failure_handler( _model_id = _metadata.get("model_info", {}).get("id", "") _model_group = _metadata.get("model_group", "") api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) - _exception_string = str(_exception)[:500] + _exception_string = str(_exception) error_log = LiteLLM_ErrorLogs( request_id=str(uuid.uuid4()), @@ -7511,7 +7511,9 @@ async def login(request: Request): # Non SSO -> If user is using UI_USERNAME and UI_PASSWORD they are Proxy admin user_role = LitellmUserRoles.PROXY_ADMIN user_id = username - key_user_id = user_id + + # we want the key created to have PROXY_ADMIN_PERMISSIONS + key_user_id = litellm_proxy_admin_name if ( os.getenv("PROXY_ADMIN_ID", None) is not None and os.environ["PROXY_ADMIN_ID"] == user_id @@ -7531,7 +7533,17 @@ async def login(request: Request): if os.getenv("DATABASE_URL") is not None: response = await generate_key_helper_fn( request_type="key", - **{"user_role": LitellmUserRoles.PROXY_ADMIN, "duration": "2hr", "key_max_budget": 5, "models": [], "aliases": {}, "config": {}, "spend": 0, "user_id": key_user_id, "team_id": "litellm-dashboard"}, # type: ignore + **{ + "user_role": LitellmUserRoles.PROXY_ADMIN, + "duration": "2hr", + "key_max_budget": 5, + "models": [], + "aliases": {}, + "config": {}, + "spend": 0, + "user_id": key_user_id, + "team_id": "litellm-dashboard", + }, # type: ignore ) else: raise ProxyException( diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 1fbd95b3cf..87bd85078c 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -817,9 +817,9 @@ async def get_global_spend_report( default=None, description="Time till which to view spend", ), - group_by: Optional[Literal["team", "customer"]] = fastapi.Query( + group_by: Optional[Literal["team", "customer", "api_key"]] = fastapi.Query( default="team", - description="Group spend by internal team or customer", + description="Group spend by internal team or customer or api_key", ), ): """ @@ -860,7 +860,7 @@ async def get_global_spend_report( start_date_obj = datetime.strptime(start_date, "%Y-%m-%d") end_date_obj = datetime.strptime(end_date, "%Y-%m-%d") - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import premium_user, prisma_client try: if prisma_client is None: @@ -868,6 +868,12 @@ async def get_global_spend_report( f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + if premium_user is not True: + verbose_proxy_logger.debug("accessing /spend/report but not a premium user") + raise ValueError( + "/spend/report endpoint " + CommonProxyErrors.not_premium_user.value + ) + if group_by == "team": # first get data from spend logs -> SpendByModelApiKey # then read data from "SpendByModelApiKey" to format the response obj @@ -992,6 +998,48 @@ async def get_global_spend_report( return [] return db_response + elif group_by == "api_key": + sql_query = """ + WITH SpendByModelApiKey AS ( + SELECT + sl.api_key, + sl.model, + SUM(sl.spend) AS model_cost, + SUM(sl.prompt_tokens) AS model_input_tokens, + SUM(sl.completion_tokens) AS model_output_tokens + FROM + "LiteLLM_SpendLogs" sl + WHERE + sl."startTime" BETWEEN $1::date AND $2::date + GROUP BY + sl.api_key, + sl.model + ) + SELECT + api_key, + SUM(model_cost) AS total_cost, + SUM(model_input_tokens) AS total_input_tokens, + SUM(model_output_tokens) AS total_output_tokens, + jsonb_agg(jsonb_build_object( + 'model', model, + 'total_cost', model_cost, + 'total_input_tokens', model_input_tokens, + 'total_output_tokens', model_output_tokens + )) AS model_details + FROM + SpendByModelApiKey + GROUP BY + api_key + ORDER BY + total_cost DESC; + """ + db_response = await prisma_client.db.query_raw( + sql_query, start_date_obj, end_date_obj + ) + if db_response is None: + return [] + + return db_response except Exception as e: raise HTTPException( diff --git a/litellm/tests/test_bedrock_completion.py b/litellm/tests/test_bedrock_completion.py index 24eefceeff..6e39c30b36 100644 --- a/litellm/tests/test_bedrock_completion.py +++ b/litellm/tests/test_bedrock_completion.py @@ -25,6 +25,7 @@ from litellm import ( completion_cost, embedding, ) +from litellm.llms.bedrock_httpx import BedrockLLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler # litellm.num_retries = 3 @@ -217,6 +218,234 @@ def test_completion_bedrock_claude_sts_client_auth(): pytest.fail(f"Error occurred: {e}") +@pytest.fixture() +def bedrock_session_token_creds(): + print("\ncalling oidc auto to get aws_session_token credentials") + import os + + aws_region_name = os.environ["AWS_REGION_NAME"] + aws_session_token = os.environ.get("AWS_SESSION_TOKEN") + + bllm = BedrockLLM() + if aws_session_token is not None: + # For local testing + creds = bllm.get_credentials( + aws_region_name=aws_region_name, + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + aws_session_token=aws_session_token, + ) + else: + # For circle-ci testing + # aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"] + # TODO: This is using ai.moda's IAM role, we should use LiteLLM's IAM role eventually + aws_role_name = ( + "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci" + ) + aws_web_identity_token = "oidc/circleci_v2/" + + creds = bllm.get_credentials( + aws_region_name=aws_region_name, + aws_web_identity_token=aws_web_identity_token, + aws_role_name=aws_role_name, + aws_session_name="my-test-session", + ) + return creds + + +def process_stream_response(res, messages): + import types + + if isinstance(res, litellm.utils.CustomStreamWrapper): + chunks = [] + for part in res: + chunks.append(part) + text = part.choices[0].delta.content or "" + print(text, end="") + res = litellm.stream_chunk_builder(chunks, messages=messages) + else: + raise ValueError("Response object is not a streaming response") + + return res + + +@pytest.mark.skipif( + os.environ.get("CIRCLE_OIDC_TOKEN_V2") is None, + reason="Cannot run without being in CircleCI Runner", +) +def test_completion_bedrock_claude_aws_session_token(bedrock_session_token_creds): + print("\ncalling bedrock claude with aws_session_token auth") + + import os + + aws_region_name = os.environ["AWS_REGION_NAME"] + aws_access_key_id = bedrock_session_token_creds.access_key + aws_secret_access_key = bedrock_session_token_creds.secret_key + aws_session_token = bedrock_session_token_creds.token + + try: + litellm.set_verbose = True + + response_1 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=10, + temperature=0.1, + aws_region_name=aws_region_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + ) + print(response_1) + assert len(response_1.choices) > 0 + assert len(response_1.choices[0].message.content) > 0 + + # This second call is to verify that the cache isn't breaking anything + response_2 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=5, + temperature=0.2, + aws_region_name=aws_region_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + ) + print(response_2) + assert len(response_2.choices) > 0 + assert len(response_2.choices[0].message.content) > 0 + + # This third call is to verify that the cache isn't used for a different region + response_3 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=6, + temperature=0.3, + aws_region_name="us-east-1", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + ) + print(response_3) + assert len(response_3.choices) > 0 + assert len(response_3.choices[0].message.content) > 0 + + # This fourth call is to verify streaming api works + response_4 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=6, + temperature=0.3, + aws_region_name="us-east-1", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + stream=True, + ) + response_4 = process_stream_response(response_4, messages) + print(response_4) + assert len(response_4.choices) > 0 + assert len(response_4.choices[0].message.content) > 0 + + except RateLimitError: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +@pytest.mark.skipif( + os.environ.get("CIRCLE_OIDC_TOKEN_V2") is None, + reason="Cannot run without being in CircleCI Runner", +) +def test_completion_bedrock_claude_aws_bedrock_client(bedrock_session_token_creds): + print("\ncalling bedrock claude with aws_session_token auth") + + import os + + import boto3 + from botocore.client import Config + + aws_region_name = os.environ["AWS_REGION_NAME"] + aws_access_key_id = bedrock_session_token_creds.access_key + aws_secret_access_key = bedrock_session_token_creds.secret_key + aws_session_token = bedrock_session_token_creds.token + + aws_bedrock_client_west = boto3.client( + service_name="bedrock-runtime", + region_name=aws_region_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + config=Config(read_timeout=600), + ) + + try: + litellm.set_verbose = True + + response_1 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=10, + temperature=0.1, + aws_bedrock_client=aws_bedrock_client_west, + ) + print(response_1) + assert len(response_1.choices) > 0 + assert len(response_1.choices[0].message.content) > 0 + + # This second call is to verify that the cache isn't breaking anything + response_2 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=5, + temperature=0.2, + aws_bedrock_client=aws_bedrock_client_west, + ) + print(response_2) + assert len(response_2.choices) > 0 + assert len(response_2.choices[0].message.content) > 0 + + # This third call is to verify that the cache isn't used for a different region + aws_bedrock_client_east = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + config=Config(read_timeout=600), + ) + + response_3 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=6, + temperature=0.3, + aws_bedrock_client=aws_bedrock_client_east, + ) + print(response_3) + assert len(response_3.choices) > 0 + assert len(response_3.choices[0].message.content) > 0 + + # This fourth call is to verify streaming api works + response_4 = completion( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + messages=messages, + max_tokens=6, + temperature=0.3, + aws_bedrock_client=aws_bedrock_client_east, + stream=True, + ) + response_4 = process_stream_response(response_4, messages) + print(response_4) + assert len(response_4.choices) > 0 + assert len(response_4.choices[0].message.content) > 0 + + except RateLimitError: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + # test_completion_bedrock_claude_sts_client_auth() @@ -489,61 +718,6 @@ def test_completion_claude_3_base64(): pytest.fail(f"An exception occurred - {str(e)}") -def test_provisioned_throughput(): - try: - litellm.set_verbose = True - import io - import json - - import botocore - import botocore.session - from botocore.stub import Stubber - - bedrock_client = botocore.session.get_session().create_client( - "bedrock-runtime", region_name="us-east-1" - ) - - expected_params = { - "accept": "application/json", - "body": '{"prompt": "\\n\\nHuman: Hello, how are you?\\n\\nAssistant: ", ' - '"max_tokens_to_sample": 256}', - "contentType": "application/json", - "modelId": "provisioned-model-arn", - } - response_from_bedrock = { - "body": io.StringIO( - json.dumps( - { - "completion": " Here is a short poem about the sky:", - "stop_reason": "max_tokens", - "stop": None, - } - ) - ), - "contentType": "contentType", - "ResponseMetadata": {"HTTPStatusCode": 200}, - } - - with Stubber(bedrock_client) as stubber: - stubber.add_response( - "invoke_model", - service_response=response_from_bedrock, - expected_params=expected_params, - ) - response = litellm.completion( - model="bedrock/anthropic.claude-instant-v1", - model_id="provisioned-model-arn", - messages=[{"content": "Hello, how are you?", "role": "user"}], - aws_bedrock_client=bedrock_client, - ) - print("response stubbed", response) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_provisioned_throughput() - - def test_completion_bedrock_mistral_completion_auth(): print("calling bedrock mistral completion params auth") import os diff --git a/litellm/tests/test_pass_through_endpoints.py b/litellm/tests/test_pass_through_endpoints.py new file mode 100644 index 0000000000..0f234dfa8b --- /dev/null +++ b/litellm/tests/test_pass_through_endpoints.py @@ -0,0 +1,85 @@ +import os +import sys + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds-the parent directory to the system path + +import asyncio + +import httpx + +from litellm.proxy.proxy_server import app, initialize_pass_through_endpoints + + +# Mock the async_client used in the pass_through_request function +async def mock_request(*args, **kwargs): + return httpx.Response(200, json={"message": "Mocked response"}) + + +@pytest.fixture +def client(): + return TestClient(app) + + +@pytest.mark.asyncio +async def test_pass_through_endpoint(client, monkeypatch): + # Mock the httpx.AsyncClient.request method + monkeypatch.setattr("httpx.AsyncClient.request", mock_request) + + # Define a pass-through endpoint + pass_through_endpoints = [ + { + "path": "/test-endpoint", + "target": "https://api.example.com/v1/chat/completions", + "headers": {"Authorization": "Bearer test-token"}, + } + ] + + # Initialize the pass-through endpoint + await initialize_pass_through_endpoints(pass_through_endpoints) + + # Make a request to the pass-through endpoint + response = client.post("/test-endpoint", json={"prompt": "Hello, world!"}) + + # Assert the response + assert response.status_code == 200 + assert response.json() == {"message": "Mocked response"} + + +@pytest.mark.asyncio +async def test_pass_through_endpoint_rerank(client): + _cohere_api_key = os.environ.get("COHERE_API_KEY") + + # Define a pass-through endpoint + pass_through_endpoints = [ + { + "path": "/v1/rerank", + "target": "https://api.cohere.com/v1/rerank", + "headers": {"Authorization": f"bearer {_cohere_api_key}"}, + } + ] + + # Initialize the pass-through endpoint + await initialize_pass_through_endpoints(pass_through_endpoints) + + _json_data = { + "model": "rerank-english-v3.0", + "query": "What is the capital of the United States?", + "top_n": 3, + "documents": [ + "Carson City is the capital city of the American state of Nevada." + ], + } + + # Make a request to the pass-through endpoint + response = client.post("/v1/rerank", json=_json_data) + + print("JSON response: ", _json_data) + + # Assert the response + assert response.status_code == 200 diff --git a/litellm/tests/test_proxy_exception_mapping.py b/litellm/tests/test_proxy_exception_mapping.py index 4988426616..4fb1e71349 100644 --- a/litellm/tests/test_proxy_exception_mapping.py +++ b/litellm/tests/test_proxy_exception_mapping.py @@ -1,25 +1,31 @@ # test that the proxy actually does exception mapping to the OpenAI format -import sys, os -from unittest import mock import json +import os +import sys +from unittest import mock + from dotenv import load_dotenv load_dotenv() -import os, io, asyncio +import asyncio +import io +import os sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path +import openai import pytest -import litellm, openai -from fastapi.testclient import TestClient from fastapi import Response -from litellm.proxy.proxy_server import ( +from fastapi.testclient import TestClient + +import litellm +from litellm.proxy.proxy_server import ( # Replace with the actual module where your FastAPI router is defined + initialize, router, save_worker_config, - initialize, -) # Replace with the actual module where your FastAPI router is defined +) invalid_authentication_error_response = Response( status_code=401, @@ -66,6 +72,12 @@ def test_chat_completion_exception(client): json_response = response.json() print("keys in json response", json_response.keys()) assert json_response.keys() == {"error"} + print("ERROR=", json_response["error"]) + assert isinstance(json_response["error"]["message"], str) + assert ( + json_response["error"]["message"] + == "litellm.AuthenticationError: AuthenticationError: OpenAIException - Incorrect API key provided: bad-key. You can find your API key at https://platform.openai.com/account/api-keys." + ) # make an openai client to call _make_status_error_from_response openai_client = openai.OpenAI(api_key="anything") diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py index 7c59611d73..5f22e88a17 100644 --- a/litellm/tests/test_router.py +++ b/litellm/tests/test_router.py @@ -1081,7 +1081,7 @@ async def test_aimg_gen_on_router(): { "model_name": "dall-e-3", "litellm_params": { - "model": "dall-e-3", + "model": "openai/dall-e-3", }, }, { @@ -1137,7 +1137,7 @@ def test_img_gen_on_router(): try: model_list = [ { - "model_name": "dall-e-3", + "model_name": "openai/dall-e-3", "litellm_params": { "model": "dall-e-3", }, diff --git a/litellm/tests/test_router_debug_logs.py b/litellm/tests/test_router_debug_logs.py index 09590e5ac6..20eccf5dea 100644 --- a/litellm/tests/test_router_debug_logs.py +++ b/litellm/tests/test_router_debug_logs.py @@ -1,16 +1,23 @@ -import sys, os, time -import traceback, asyncio +import asyncio +import os +import sys +import time +import traceback + import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm, asyncio, logging +import asyncio +import logging + +import litellm from litellm import Router # this tests debug logs from litellm router and litellm proxy server -from litellm._logging import verbose_router_logger, verbose_logger, verbose_proxy_logger +from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger # this tests debug logs from litellm router and litellm proxy server @@ -81,7 +88,7 @@ def test_async_fallbacks(caplog): # Define the expected log messages # - error request, falling back notice, success notice expected_logs = [ - "litellm.acompletion(model=gpt-3.5-turbo)\x1b[31m Exception litellm.AuthenticationError: AuthenticationError: OpenAIException - Error code: 401 - {'error': {'message': 'Incorrect API key provided: bad-key. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}\x1b[0m", + "litellm.acompletion(model=gpt-3.5-turbo)\x1b[31m Exception litellm.AuthenticationError: AuthenticationError: OpenAIException - Incorrect API key provided: bad-key. You can find your API key at https://platform.openai.com/account/api-keys.\x1b[0m", "Falling back to model_group = azure/gpt-3.5-turbo", "litellm.acompletion(model=azure/chatgpt-v-2)\x1b[32m 200 OK\x1b[0m", "Successful fallback b/w models.", diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index fa9e49f87c..1f1b253a06 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -1226,6 +1226,7 @@ async def test_completion_replicate_llama3_streaming(sync_mode): messages=messages, max_tokens=10, # type: ignore stream=True, + num_retries=3, ) complete_response = "" # Add any assertions here to check the response @@ -1247,6 +1248,7 @@ async def test_completion_replicate_llama3_streaming(sync_mode): messages=messages, max_tokens=100, # type: ignore stream=True, + num_retries=3, ) complete_response = "" # Add any assertions here to check the response diff --git a/litellm/utils.py b/litellm/utils.py index 2598aa46b1..4b80d203b4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -51,6 +51,7 @@ import litellm.litellm_core_utils import litellm.litellm_core_utils.json_validation_rule from litellm.caching import DualCache from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.exception_mapping_utils import get_error_message from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, @@ -2910,12 +2911,7 @@ def get_optional_params( optional_params=optional_params, ) ) - else: - optional_params = litellm.AmazonAnthropicConfig().map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - ) - else: # bedrock httpx route + elif model in litellm.BEDROCK_CONVERSE_MODELS: optional_params = litellm.AmazonConverseConfig().map_openai_params( model=model, non_default_params=non_default_params, @@ -2926,6 +2922,11 @@ def get_optional_params( else False ), ) + else: + optional_params = litellm.AmazonAnthropicConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + ) elif "amazon" in model: # amazon titan llms _check_valid_arg(supported_params=supported_params) # see https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-large @@ -5858,10 +5859,13 @@ def exception_type( or custom_llm_provider in litellm.openai_compatible_providers ): # custom_llm_provider is openai, make it OpenAI - if hasattr(original_exception, "message"): - message = original_exception.message - else: - message = str(original_exception) + message = get_error_message(error_obj=original_exception) + if message is None: + if hasattr(original_exception, "message"): + message = original_exception.message + else: + message = str(original_exception) + if message is not None and isinstance(message, str): message = message.replace("OPENAI", custom_llm_provider.upper()) message = message.replace("openai", custom_llm_provider) @@ -7314,10 +7318,17 @@ def exception_type( request=original_exception.request, ) elif custom_llm_provider == "azure": + message = get_error_message(error_obj=original_exception) + if message is None: + if hasattr(original_exception, "message"): + message = original_exception.message + else: + message = str(original_exception) + if "Internal server error" in error_str: exception_mapping_worked = True raise litellm.InternalServerError( - message=f"AzureException Internal server error - {original_exception.message}", + message=f"AzureException Internal server error - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7330,7 +7341,7 @@ def exception_type( elif "This model's maximum context length is" in error_str: exception_mapping_worked = True raise ContextWindowExceededError( - message=f"AzureException ContextWindowExceededError - {original_exception.message}", + message=f"AzureException ContextWindowExceededError - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7339,7 +7350,7 @@ def exception_type( elif "DeploymentNotFound" in error_str: exception_mapping_worked = True raise NotFoundError( - message=f"AzureException NotFoundError - {original_exception.message}", + message=f"AzureException NotFoundError - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7359,7 +7370,7 @@ def exception_type( ): exception_mapping_worked = True raise ContentPolicyViolationError( - message=f"litellm.ContentPolicyViolationError: AzureException - {original_exception.message}", + message=f"litellm.ContentPolicyViolationError: AzureException - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7368,7 +7379,7 @@ def exception_type( elif "invalid_request_error" in error_str: exception_mapping_worked = True raise BadRequestError( - message=f"AzureException BadRequestError - {original_exception.message}", + message=f"AzureException BadRequestError - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7380,7 +7391,7 @@ def exception_type( ): exception_mapping_worked = True raise AuthenticationError( - message=f"{exception_provider} AuthenticationError - {original_exception.message}", + message=f"{exception_provider} AuthenticationError - {message}", llm_provider=custom_llm_provider, model=model, litellm_debug_info=extra_information, @@ -7391,7 +7402,7 @@ def exception_type( if original_exception.status_code == 400: exception_mapping_worked = True raise BadRequestError( - message=f"AzureException - {original_exception.message}", + message=f"AzureException - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7400,7 +7411,7 @@ def exception_type( elif original_exception.status_code == 401: exception_mapping_worked = True raise AuthenticationError( - message=f"AzureException AuthenticationError - {original_exception.message}", + message=f"AzureException AuthenticationError - {message}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -7409,7 +7420,7 @@ def exception_type( elif original_exception.status_code == 408: exception_mapping_worked = True raise Timeout( - message=f"AzureException Timeout - {original_exception.message}", + message=f"AzureException Timeout - {message}", model=model, litellm_debug_info=extra_information, llm_provider="azure", @@ -7417,7 +7428,7 @@ def exception_type( elif original_exception.status_code == 422: exception_mapping_worked = True raise BadRequestError( - message=f"AzureException BadRequestError - {original_exception.message}", + message=f"AzureException BadRequestError - {message}", model=model, llm_provider="azure", litellm_debug_info=extra_information, @@ -7426,7 +7437,7 @@ def exception_type( elif original_exception.status_code == 429: exception_mapping_worked = True raise RateLimitError( - message=f"AzureException RateLimitError - {original_exception.message}", + message=f"AzureException RateLimitError - {message}", model=model, llm_provider="azure", litellm_debug_info=extra_information, @@ -7435,7 +7446,7 @@ def exception_type( elif original_exception.status_code == 503: exception_mapping_worked = True raise ServiceUnavailableError( - message=f"AzureException ServiceUnavailableError - {original_exception.message}", + message=f"AzureException ServiceUnavailableError - {message}", model=model, llm_provider="azure", litellm_debug_info=extra_information, @@ -7444,7 +7455,7 @@ def exception_type( elif original_exception.status_code == 504: # gateway timeout error exception_mapping_worked = True raise Timeout( - message=f"AzureException Timeout - {original_exception.message}", + message=f"AzureException Timeout - {message}", model=model, litellm_debug_info=extra_information, llm_provider="azure", @@ -7453,7 +7464,7 @@ def exception_type( exception_mapping_worked = True raise APIError( status_code=original_exception.status_code, - message=f"AzureException APIError - {original_exception.message}", + message=f"AzureException APIError - {message}", llm_provider="azure", litellm_debug_info=extra_information, model=model, diff --git a/pyproject.toml b/pyproject.toml index 578c50cc1d..2519c167f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.41.1" +version = "1.41.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -90,7 +90,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.41.1" +version = "1.41.2" version_files = [ "pyproject.toml:^version" ]