From 3105eaebc3949ad8c72006dfb248f22510d51718 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Sat, 22 Jun 2024 16:12:42 -0700
Subject: [PATCH 01/68] feat -  add debug_utils

---
 litellm/proxy/common_utils/debug_utils.py | 27 +++++++++++++++++++++++
 litellm/proxy/proxy_server.py             |  2 ++
 2 files changed, 29 insertions(+)
 create mode 100644 litellm/proxy/common_utils/debug_utils.py

diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py
new file mode 100644
index 0000000000..dc77958a62
--- /dev/null
+++ b/litellm/proxy/common_utils/debug_utils.py
@@ -0,0 +1,27 @@
+# Start tracing memory allocations
+import os
+import tracemalloc
+
+from fastapi import APIRouter
+
+from litellm._logging import verbose_proxy_logger
+
+router = APIRouter()
+
+if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
+    tracemalloc.start()
+
+    @router.get("/memory-usage", include_in_schema=False)
+    async def memory_usage():
+        # Take a snapshot of the current memory usage
+        snapshot = tracemalloc.take_snapshot()
+        top_stats = snapshot.statistics("lineno")
+        verbose_proxy_logger.debug("TOP STATS: %s", top_stats)
+
+        # Get the top 50 memory usage lines
+        top_50 = top_stats[:50]
+        result = []
+        for stat in top_50:
+            result.append(f"{stat.traceback.format()}: {stat.size / 1024} KiB")
+
+        return {"top_50_memory_usage": result}
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 630aa3f3e1..9b78ecf8ab 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -140,6 +140,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 
 ## Import All Misc routes here ##
 from litellm.proxy.caching_routes import router as caching_router
+from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
 from litellm.proxy.health_check import perform_health_check
 from litellm.proxy.health_endpoints._health_endpoints import router as health_router
@@ -9173,3 +9174,4 @@ app.include_router(team_router)
 app.include_router(spend_management_router)
 app.include_router(caching_router)
 app.include_router(analytics_router)
+app.include_router(debugging_endpoints_router)

From b2c1a3ad791df72c4bbf5160a5b621f219dc12e4 Mon Sep 17 00:00:00 2001
From: corrm <islamnofl.official@gmail.com>
Date: Mon, 24 Jun 2024 05:54:58 +0300
Subject: [PATCH 02/68] chore: Improved prompt generation in ollama_pt function

---
 litellm/llms/prompt_templates/factory.py | 25 +++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index 398e96af7e..02ed93fae3 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -172,14 +172,21 @@ def ollama_pt(
                             images.append(base64_image)
         return {"prompt": prompt, "images": images}
     else:
-        prompt = "".join(
-            (
-                m["content"]
-                if isinstance(m["content"], str) is str
-                else "".join(m["content"])
-            )
-            for m in messages
-        )
+        prompt = ""
+        for message in messages:
+            role = message["role"]
+            content = message.get("content", "")
+
+            if "tool_calls" in message:
+                for call in message["tool_calls"]:
+                    function_name = call["function"]["name"]
+                    arguments = json.loads(call["function"]["arguments"])
+                    prompt += f"### Tool Call ({call["id"]}):\nFunction: {function_name}\nArguments: {json.dumps(arguments)}\n\n"
+            elif "tool_call_id" in message:
+                prompt += f"### Tool Call Result ({message["tool_call_id"]}):\n{message["content"]}\n\n"
+            elif content:
+                prompt += f"### {role.capitalize()}:\n{content}\n\n"
+
     return prompt
 
 
@@ -710,7 +717,7 @@ def convert_to_anthropic_tool_result_xml(message: dict) -> str:
 
     """
     Anthropic tool_results look like:
-    
+
     [Successful results]
     <function_results>
     <result>

From 423a60c8bcfc1e7b4fd81ccbcba7757e772b85d8 Mon Sep 17 00:00:00 2001
From: corrm <islamnofl.official@gmail.com>
Date: Mon, 24 Jun 2024 05:55:22 +0300
Subject: [PATCH 03/68] chore: Improved OllamaConfig get_required_params and
 ollama_acompletion and ollama_async_streaming functions

---
 litellm/llms/ollama.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/ollama.py b/litellm/llms/ollama.py
index e7dd1d5f55..1939715b35 100644
--- a/litellm/llms/ollama.py
+++ b/litellm/llms/ollama.py
@@ -126,7 +126,7 @@ class OllamaConfig:
             )
             and v is not None
         }
-    
+
     def get_required_params(self) -> List[ProviderField]:
         """For a given provider, return it's required fields with a description"""
         return [
@@ -451,7 +451,7 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj):
                         {
                             "id": f"call_{str(uuid.uuid4())}",
                             "function": {
-                                "name": function_call["name"],
+                                "name": function_call.get("name", function_call.get("function", None)),
                                 "arguments": json.dumps(function_call["arguments"]),
                             },
                             "type": "function",

From b8a8b0847c9e68726e56674add4aaa2951482a99 Mon Sep 17 00:00:00 2001
From: corrm <islamnofl.official@gmail.com>
Date: Mon, 24 Jun 2024 05:56:56 +0300
Subject: [PATCH 04/68] Added improved function name handling in
 ollama_async_streaming

---
 litellm/llms/ollama_chat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/llms/ollama_chat.py b/litellm/llms/ollama_chat.py
index a7439bbcc0..af6fd5b806 100644
--- a/litellm/llms/ollama_chat.py
+++ b/litellm/llms/ollama_chat.py
@@ -434,7 +434,7 @@ async def ollama_async_streaming(
                         {
                             "id": f"call_{str(uuid.uuid4())}",
                             "function": {
-                                "name": function_call["name"],
+                                "name": function_call.get("name", function_call.get("function", None)),
                                 "arguments": json.dumps(function_call["arguments"]),
                             },
                             "type": "function",

From e2af13550a29db7ea0757a313c3590105c925a74 Mon Sep 17 00:00:00 2001
From: Islam Nofl <islamnofl.official@gmail.com>
Date: Mon, 24 Jun 2024 08:01:15 +0300
Subject: [PATCH 05/68] Rename ollama prompt 'Function' word to 'Name'

---
 litellm/llms/prompt_templates/factory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index 02ed93fae3..109c5b8d83 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -181,7 +181,7 @@ def ollama_pt(
                 for call in message["tool_calls"]:
                     function_name = call["function"]["name"]
                     arguments = json.loads(call["function"]["arguments"])
-                    prompt += f"### Tool Call ({call["id"]}):\nFunction: {function_name}\nArguments: {json.dumps(arguments)}\n\n"
+                    prompt += f"### Tool Call ({call["id"]}):\nName: {function_name}\nArguments: {json.dumps(arguments)}\n\n"
             elif "tool_call_id" in message:
                 prompt += f"### Tool Call Result ({message["tool_call_id"]}):\n{message["content"]}\n\n"
             elif content:

From 515dc9acc7f8bc133952585ab4bccd912bb0bd1a Mon Sep 17 00:00:00 2001
From: 7HR4IZ3 <90985774+7HR4IZ3@users.noreply.github.com>
Date: Mon, 24 Jun 2024 14:09:40 +0100
Subject: [PATCH 06/68] fix: Lunary integration

Fixes the bug of litellm not logging system messages to lunary
---
 litellm/integrations/lunary.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py
index f9b2f25e70..b0cc069c40 100644
--- a/litellm/integrations/lunary.py
+++ b/litellm/integrations/lunary.py
@@ -108,6 +108,7 @@ class LunaryLogger:
         try:
             print_verbose(f"Lunary Logging - Logging request for model {model}")
 
+            template_id = None
             litellm_params = kwargs.get("litellm_params", {})
             optional_params = kwargs.get("optional_params", {})
             metadata = litellm_params.get("metadata", {}) or {}

From acbcfef5af24663ac61b377ac860378019b00197 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 07:54:08 -0700
Subject: [PATCH 07/68] fix ui login bug

---
 litellm/proxy/proxy_server.py | 52 ++++++++++++++---------------------
 1 file changed, 20 insertions(+), 32 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 630aa3f3e1..1e987778c4 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -7502,12 +7502,6 @@ async def login(request: Request):
             litellm_dashboard_ui += "/ui/"
         import jwt
 
-        if litellm_master_key_hash is None:
-            raise HTTPException(
-                status_code=500,
-                detail={"error": "No master key set, please set LITELLM_MASTER_KEY"},
-            )
-
         jwt_token = jwt.encode(
             {
                 "user_id": user_id,
@@ -7517,7 +7511,7 @@ async def login(request: Request):
                 "login_method": "username_password",
                 "premium_user": premium_user,
             },
-            litellm_master_key_hash,
+            master_key,
             algorithm="HS256",
         )
         litellm_dashboard_ui += "?userID=" + user_id
@@ -7572,14 +7566,6 @@ async def login(request: Request):
                 litellm_dashboard_ui += "/ui/"
             import jwt
 
-            if litellm_master_key_hash is None:
-                raise HTTPException(
-                    status_code=500,
-                    detail={
-                        "error": "No master key set, please set LITELLM_MASTER_KEY"
-                    },
-                )
-
             jwt_token = jwt.encode(
                 {
                     "user_id": user_id,
@@ -7589,7 +7575,7 @@ async def login(request: Request):
                     "login_method": "username_password",
                     "premium_user": premium_user,
                 },
-                litellm_master_key_hash,
+                master_key,
                 algorithm="HS256",
             )
             litellm_dashboard_ui += "?userID=" + user_id
@@ -7636,7 +7622,14 @@ async def onboarding(invite_link: str):
     - Get user from db
     - Pass in user_email if set
     """
-    global prisma_client
+    global prisma_client, master_key
+    if master_key is None:
+        raise ProxyException(
+            message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml.  https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.",
+            type="auth_error",
+            param="master_key",
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
     ### VALIDATE INVITE LINK ###
     if prisma_client is None:
         raise HTTPException(
@@ -7708,12 +7701,6 @@ async def onboarding(invite_link: str):
         litellm_dashboard_ui += "/ui/onboarding"
     import jwt
 
-    if litellm_master_key_hash is None:
-        raise HTTPException(
-            status_code=500,
-            detail={"error": "No master key set, please set LITELLM_MASTER_KEY"},
-        )
-
     jwt_token = jwt.encode(
         {
             "user_id": user_obj.user_id,
@@ -7723,7 +7710,7 @@ async def onboarding(invite_link: str):
             "login_method": "username_password",
             "premium_user": premium_user,
         },
-        litellm_master_key_hash,
+        master_key,
         algorithm="HS256",
     )
 
@@ -7856,11 +7843,18 @@ def get_image():
 @app.get("/sso/callback", tags=["experimental"], include_in_schema=False)
 async def auth_callback(request: Request):
     """Verify login"""
-    global general_settings, ui_access_mode, premium_user
+    global general_settings, ui_access_mode, premium_user, master_key
     microsoft_client_id = os.getenv("MICROSOFT_CLIENT_ID", None)
     google_client_id = os.getenv("GOOGLE_CLIENT_ID", None)
     generic_client_id = os.getenv("GENERIC_CLIENT_ID", None)
     # get url from request
+    if master_key is None:
+        raise ProxyException(
+            message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml.  https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.",
+            type="auth_error",
+            param="master_key",
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
     redirect_url = os.getenv("PROXY_BASE_URL", str(request.base_url))
     if redirect_url.endswith("/"):
         redirect_url += "sso/callback"
@@ -8134,12 +8128,6 @@ async def auth_callback(request: Request):
 
     import jwt
 
-    if litellm_master_key_hash is None:
-        raise HTTPException(
-            status_code=500,
-            detail={"error": "No master key set, please set LITELLM_MASTER_KEY"},
-        )
-
     jwt_token = jwt.encode(
         {
             "user_id": user_id,
@@ -8149,7 +8137,7 @@ async def auth_callback(request: Request):
             "login_method": "sso",
             "premium_user": premium_user,
         },
-        litellm_master_key_hash,
+        master_key,
         algorithm="HS256",
     )
     litellm_dashboard_ui += "?userID=" + user_id

From 70a605b3cc927b55192c80e04fd4139eb28ca496 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 13:21:16 -0700
Subject: [PATCH 08/68] docs - update telemetry

---
 docs/my-website/docs/observability/telemetry.md | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/docs/my-website/docs/observability/telemetry.md b/docs/my-website/docs/observability/telemetry.md
index 78267b9c56..2322955662 100644
--- a/docs/my-website/docs/observability/telemetry.md
+++ b/docs/my-website/docs/observability/telemetry.md
@@ -1,13 +1,8 @@
 # Telemetry 
 
-LiteLLM contains a telemetry feature that tells us what models are used, and what errors are hit.
+There is no Telemetry on LiteLLM - no data is stored by us
 
 ## What is logged? 
 
-Only the model name and exception raised is logged. 
+NOTHING - no data is sent to LiteLLM Servers
 
-## Why?
-We use this information to help us understand how LiteLLM is used, and improve stability. 
-
-## Opting out
-If you prefer to opt out of telemetry, you can do this by setting `litellm.telemetry = False`. 
\ No newline at end of file

From 123477b55a62825b429ebe5cbaedd7b3bad4b900 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 16:55:19 -0700
Subject: [PATCH 09/68] fix(utils.py): fix exception_mapping check for errors

If exception already mapped - don't attach traceback to it
---
 litellm/exceptions.py | 16 +++++-----------
 litellm/utils.py      |  4 ++++
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/litellm/exceptions.py b/litellm/exceptions.py
index 9674d48b12..98b5192784 100644
--- a/litellm/exceptions.py
+++ b/litellm/exceptions.py
@@ -9,10 +9,11 @@
 
 ## LiteLLM versions of the OpenAI Exception Types
 
-import openai
-import httpx
 from typing import Optional
 
+import httpx
+import openai
+
 
 class AuthenticationError(openai.AuthenticationError):  # type: ignore
     def __init__(
@@ -658,15 +659,8 @@ class APIResponseValidationError(openai.APIResponseValidationError):  # type: ig
 
 
 class OpenAIError(openai.OpenAIError):  # type: ignore
-    def __init__(self, original_exception):
-        self.status_code = original_exception.http_status
-        super().__init__(
-            http_body=original_exception.http_body,
-            http_status=original_exception.http_status,
-            json_body=original_exception.json_body,
-            headers=original_exception.headers,
-            code=original_exception.code,
-        )
+    def __init__(self, original_exception=None):
+        super().__init__()
         self.llm_provider = "openai"
 
 
diff --git a/litellm/utils.py b/litellm/utils.py
index 0849ba3a26..ce66d0fbb0 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -5914,6 +5914,7 @@ def exception_type(
                         )
                 else:
                     # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
+                    # exception_mapping_worked = True
                     raise APIConnectionError(
                         message=f"APIConnectionError: {exception_provider} - {message}",
                         llm_provider=custom_llm_provider,
@@ -7460,6 +7461,9 @@ def exception_type(
         if exception_mapping_worked:
             raise e
         else:
+            for error_type in litellm.LITELLM_EXCEPTION_TYPES:
+                if isinstance(e, error_type):
+                    raise e  # it's already mapped
             raise APIConnectionError(
                 message="{}\n{}".format(original_exception, traceback.format_exc()),
                 llm_provider="",

From f5fbdf0feebcfe252e97dcf9be10732c45500dca Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 17:25:26 -0700
Subject: [PATCH 10/68] fix(router.py): use user-defined model_input_tokens for
 pre-call filter checks

---
 litellm/proxy/_new_secret_config.yaml | 16 ++++++++--
 litellm/router.py                     | 42 +++++++++++++++++++++++++--
 litellm/tests/test_router.py          |  5 ++++
 3 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml
index 640a3b2cf2..78d7dc70c3 100644
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@@ -4,7 +4,17 @@ model_list:
       model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
       api_key: my-fake-key
       aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
+      mock_response: "Hello world 1"
+    model_info: 
+      max_input_tokens: 0 # trigger context window fallback
+  - model_name: my-fake-model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+      api_key: my-fake-key
+      aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
+      mock_response: "Hello world 2"
+    model_info: 
+      max_input_tokens: 0
 
-litellm_settings:
-  success_callback: ["langfuse"]
-  failure_callback: ["langfuse"]
+router_settings:
+  enable_pre_call_checks: True
diff --git a/litellm/router.py b/litellm/router.py
index e9b0cc00a9..6163da487a 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -404,6 +404,7 @@ class Router:
             litellm.failure_callback = [self.deployment_callback_on_failure]
         print(  # noqa
             f"Intialized router with Routing strategy: {self.routing_strategy}\n\n"
+            f"Routing enable_pre_call_checks: {self.enable_pre_call_checks}\n\n"
             f"Routing fallbacks: {self.fallbacks}\n\n"
             f"Routing content fallbacks: {self.content_policy_fallbacks}\n\n"
             f"Routing context window fallbacks: {self.context_window_fallbacks}\n\n"
@@ -3915,9 +3916,38 @@ class Router:
                         raise Exception("Model invalid format - {}".format(type(model)))
         return None
 
+    def get_router_model_info(self, deployment: dict) -> ModelMapInfo:
+        """
+        For a given model id, return the model info (max tokens, input cost, output cost, etc.).
+
+        Augment litellm info with additional params set in `model_info`.
+
+        Returns
+        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.
+        """
+        ## SET MODEL NAME
+        base_model = deployment.get("model_info", {}).get("base_model", None)
+        if base_model is None:
+            base_model = deployment.get("litellm_params", {}).get("base_model", None)
+        model = base_model or deployment.get("litellm_params", {}).get("model", None)
+
+        ## GET LITELLM MODEL INFO
+        model_info = litellm.get_model_info(model=model)
+
+        ## CHECK USER SET MODEL INFO
+        user_model_info = deployment.get("model_info", {})
+
+        model_info.update(user_model_info)
+
+        return model_info
+
     def get_model_info(self, id: str) -> Optional[dict]:
         """
         For a given model id, return the model info
+
+        Returns
+        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
+        - None: could not find deployment in list
         """
         for model in self.model_list:
             if "model_info" in model and "id" in model["model_info"]:
@@ -4307,6 +4337,7 @@ class Router:
             return _returned_deployments
 
         _context_window_error = False
+        _potential_error_str = ""
         _rate_limit_error = False
 
         ## get model group RPM ##
@@ -4327,7 +4358,7 @@ class Router:
                 model = base_model or deployment.get("litellm_params", {}).get(
                     "model", None
                 )
-                model_info = litellm.get_model_info(model=model)
+                model_info = self.get_router_model_info(deployment=deployment)
 
                 if (
                     isinstance(model_info, dict)
@@ -4339,6 +4370,11 @@ class Router:
                     ):
                         invalid_model_indices.append(idx)
                         _context_window_error = True
+                        _potential_error_str += (
+                            "Model={}, Max Input Tokens={}, Got={}".format(
+                                model, model_info["max_input_tokens"], input_tokens
+                            )
+                        )
                         continue
             except Exception as e:
                 verbose_router_logger.debug("An error occurs - {}".format(str(e)))
@@ -4440,7 +4476,9 @@ class Router:
                 )
             elif _context_window_error == True:
                 raise litellm.ContextWindowExceededError(
-                    message="Context Window exceeded for given call",
+                    message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{}".format(
+                        _potential_error_str
+                    ),
                     model=model,
                     llm_provider="",
                     response=httpx.Response(
diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py
index 2e88143273..84ea9e1c9c 100644
--- a/litellm/tests/test_router.py
+++ b/litellm/tests/test_router.py
@@ -755,6 +755,7 @@ def test_router_context_window_check_pre_call_check_in_group():
                     "api_version": os.getenv("AZURE_API_VERSION"),
                     "api_base": os.getenv("AZURE_API_BASE"),
                     "base_model": "azure/gpt-35-turbo",
+                    "mock_response": "Hello world 1!",
                 },
             },
             {
@@ -762,6 +763,7 @@ def test_router_context_window_check_pre_call_check_in_group():
                 "litellm_params": {  # params for litellm completion/embedding call
                     "model": "gpt-3.5-turbo-1106",
                     "api_key": os.getenv("OPENAI_API_KEY"),
+                    "mock_response": "Hello world 2!",
                 },
             },
         ]
@@ -777,6 +779,9 @@ def test_router_context_window_check_pre_call_check_in_group():
         )
 
         print(f"response: {response}")
+
+        assert response.choices[0].message.content == "Hello world 2!"
+        assert False
     except Exception as e:
         pytest.fail(f"Got unexpected exception on router! - {str(e)}")
 

From 341c7857c10b60af87602eabdea7ea0948a80f49 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 17:28:12 -0700
Subject: [PATCH 11/68] test(test_router.py): add testing

---
 litellm/tests/test_router.py | 57 ++++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py
index 84ea9e1c9c..3237c8084a 100644
--- a/litellm/tests/test_router.py
+++ b/litellm/tests/test_router.py
@@ -732,7 +732,61 @@ def test_router_rpm_pre_call_check():
         pytest.fail(f"Got unexpected exception on router! - {str(e)}")
 
 
-def test_router_context_window_check_pre_call_check_in_group():
+def test_router_context_window_check_pre_call_check_in_group_custom_model_info():
+    """
+    - Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k)
+    - Send a 5k prompt
+    - Assert it works
+    """
+    import os
+
+    from large_text import text
+
+    litellm.set_verbose = False
+
+    print(f"len(text): {len(text)}")
+    try:
+        model_list = [
+            {
+                "model_name": "gpt-3.5-turbo",  # openai model name
+                "litellm_params": {  # params for litellm completion/embedding call
+                    "model": "azure/chatgpt-v-2",
+                    "api_key": os.getenv("AZURE_API_KEY"),
+                    "api_version": os.getenv("AZURE_API_VERSION"),
+                    "api_base": os.getenv("AZURE_API_BASE"),
+                    "base_model": "azure/gpt-35-turbo",
+                    "mock_response": "Hello world 1!",
+                },
+                "model_info": {"max_input_tokens": 100},
+            },
+            {
+                "model_name": "gpt-3.5-turbo",  # openai model name
+                "litellm_params": {  # params for litellm completion/embedding call
+                    "model": "gpt-3.5-turbo-1106",
+                    "api_key": os.getenv("OPENAI_API_KEY"),
+                    "mock_response": "Hello world 2!",
+                },
+                "model_info": {"max_input_tokens": 0},
+            },
+        ]
+
+        router = Router(model_list=model_list, set_verbose=True, enable_pre_call_checks=True, num_retries=0)  # type: ignore
+
+        response = router.completion(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "user", "content": "Who was Alexander?"},
+            ],
+        )
+
+        print(f"response: {response}")
+
+        assert response.choices[0].message.content == "Hello world 1!"
+    except Exception as e:
+        pytest.fail(f"Got unexpected exception on router! - {str(e)}")
+
+
+def test_router_context_window_check_pre_call_check():
     """
     - Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k)
     - Send a 5k prompt
@@ -781,7 +835,6 @@ def test_router_context_window_check_pre_call_check_in_group():
         print(f"response: {response}")
 
         assert response.choices[0].message.content == "Hello world 2!"
-        assert False
     except Exception as e:
         pytest.fail(f"Got unexpected exception on router! - {str(e)}")
 

From 2f90759db5ab4c3183695e11ae5a1822b5ec8206 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 17:45:13 -0700
Subject: [PATCH 12/68] feat - allow user to define public routes

---
 litellm/proxy/auth/user_api_key_auth.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 3d14f53000..f6e3a0dfeb 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -137,7 +137,9 @@ async def user_api_key_auth(
         """
         route: str = request.url.path
 
-        if route in LiteLLMRoutes.public_routes.value:
+        if route in LiteLLMRoutes.public_routes.value or route in general_settings.get(
+            "public_routes", []
+        ):
             # check if public endpoint
             return UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY)
 

From 5309be245652e2f64a7584c72a82919322ef6c5a Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 17:46:38 -0700
Subject: [PATCH 13/68] example config with public routes

---
 litellm/proxy/proxy_config.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index d5190455f1..8898dd8cb5 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -21,6 +21,9 @@ model_list:
 general_settings:
   master_key: sk-1234
   alerting: ["slack", "email"]
+  public_routes: [
+    "/spend/calculate",
+  ]
 
 litellm_settings:
   success_callback: ["prometheus"]

From a4bea47a2dad4b7d801e5aac8835d78f1d3794d8 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 17:52:01 -0700
Subject: [PATCH 14/68] fix(router.py): log rejected router requests to
 langfuse

Fixes issue where rejected requests weren't being logged
---
 .gitignore                            |   1 +
 litellm/integrations/langfuse.py      |  38 ++--
 litellm/proxy/_new_secret_config.yaml |   4 +
 litellm/router.py                     | 262 ++++++++++++++------------
 4 files changed, 167 insertions(+), 138 deletions(-)

diff --git a/.gitignore b/.gitignore
index b633e1d3d8..8a9095b840 100644
--- a/.gitignore
+++ b/.gitignore
@@ -61,3 +61,4 @@ litellm/proxy/_experimental/out/model_hub/index.html
 litellm/proxy/_experimental/out/onboarding/index.html
 litellm/tests/log.txt
 litellm/tests/langfuse.log
+litellm/tests/langfuse.log
diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py
index eae8b8e22a..794524684d 100644
--- a/litellm/integrations/langfuse.py
+++ b/litellm/integrations/langfuse.py
@@ -36,9 +36,9 @@ class LangFuseLogger:
         self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
 
         parameters = {
-            "public_key": self.public_key,
-            "secret_key": self.secret_key,
-            "host": self.langfuse_host,
+            "public_key": "pk-lf-a65841e9-5192-4397-a679-cfff029fd5b0",
+            "secret_key": "sk-lf-d58c2891-3717-4f98-89dd-df44826215fd",
+            "host": "https://us.cloud.langfuse.com",
             "release": self.langfuse_release,
             "debug": self.langfuse_debug,
             "flush_interval": flush_interval,  # flush interval in seconds
@@ -311,22 +311,22 @@ class LangFuseLogger:
 
         try:
             tags = []
-            try:
-                metadata = copy.deepcopy(
-                    metadata
-                )  # Avoid modifying the original metadata
-            except:
-                new_metadata = {}
-                for key, value in metadata.items():
-                    if (
-                        isinstance(value, list)
-                        or isinstance(value, dict)
-                        or isinstance(value, str)
-                        or isinstance(value, int)
-                        or isinstance(value, float)
-                    ):
-                        new_metadata[key] = copy.deepcopy(value)
-                metadata = new_metadata
+            # try:
+            #     metadata = copy.deepcopy(
+            #         metadata
+            #     )  # Avoid modifying the original metadata
+            # except:
+            new_metadata = {}
+            for key, value in metadata.items():
+                if (
+                    isinstance(value, list)
+                    or isinstance(value, dict)
+                    or isinstance(value, str)
+                    or isinstance(value, int)
+                    or isinstance(value, float)
+                ):
+                    new_metadata[key] = copy.deepcopy(value)
+            metadata = new_metadata
 
             supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3")
             supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3")
diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml
index 78d7dc70c3..16436c0ef9 100644
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@@ -18,3 +18,7 @@ model_list:
 
 router_settings:
   enable_pre_call_checks: True
+
+
+litellm_settings:
+  failure_callback: ["langfuse"]
\ No newline at end of file
diff --git a/litellm/router.py b/litellm/router.py
index 6163da487a..30bdbcba2d 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -4474,17 +4474,13 @@ class Router:
                 raise ValueError(
                     f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. Try again in {self.cooldown_time} seconds."
                 )
-            elif _context_window_error == True:
+            elif _context_window_error is True:
                 raise litellm.ContextWindowExceededError(
                     message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{}".format(
                         _potential_error_str
                     ),
                     model=model,
                     llm_provider="",
-                    response=httpx.Response(
-                        status_code=400,
-                        request=httpx.Request("GET", "https://example.com"),
-                    ),
                 )
         if len(invalid_model_indices) > 0:
             for idx in reversed(invalid_model_indices):
@@ -4596,127 +4592,155 @@ class Router:
                 specific_deployment=specific_deployment,
                 request_kwargs=request_kwargs,
             )
-
-        model, healthy_deployments = self._common_checks_available_deployment(
-            model=model,
-            messages=messages,
-            input=input,
-            specific_deployment=specific_deployment,
-        )  # type: ignore
-
-        if isinstance(healthy_deployments, dict):
-            return healthy_deployments
-
-        # filter out the deployments currently cooling down
-        deployments_to_remove = []
-        # cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]
-        cooldown_deployments = await self._async_get_cooldown_deployments()
-        verbose_router_logger.debug(
-            f"async cooldown deployments: {cooldown_deployments}"
-        )
-        # Find deployments in model_list whose model_id is cooling down
-        for deployment in healthy_deployments:
-            deployment_id = deployment["model_info"]["id"]
-            if deployment_id in cooldown_deployments:
-                deployments_to_remove.append(deployment)
-        # remove unhealthy deployments from healthy deployments
-        for deployment in deployments_to_remove:
-            healthy_deployments.remove(deployment)
-
-        # filter pre-call checks
-        _allowed_model_region = (
-            request_kwargs.get("allowed_model_region")
-            if request_kwargs is not None
-            else None
-        )
-
-        if self.enable_pre_call_checks and messages is not None:
-            healthy_deployments = self._pre_call_checks(
+        try:
+            model, healthy_deployments = self._common_checks_available_deployment(
                 model=model,
-                healthy_deployments=healthy_deployments,
-                messages=messages,
-                request_kwargs=request_kwargs,
-            )
-
-        if len(healthy_deployments) == 0:
-            if _allowed_model_region is None:
-                _allowed_model_region = "n/a"
-            raise ValueError(
-                f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. pre-call-checks={self.enable_pre_call_checks}, allowed_model_region={_allowed_model_region}"
-            )
-
-        if (
-            self.routing_strategy == "usage-based-routing-v2"
-            and self.lowesttpm_logger_v2 is not None
-        ):
-            deployment = await self.lowesttpm_logger_v2.async_get_available_deployments(
-                model_group=model,
-                healthy_deployments=healthy_deployments,  # type: ignore
                 messages=messages,
                 input=input,
-            )
-        if (
-            self.routing_strategy == "cost-based-routing"
-            and self.lowestcost_logger is not None
-        ):
-            deployment = await self.lowestcost_logger.async_get_available_deployments(
-                model_group=model,
-                healthy_deployments=healthy_deployments,  # type: ignore
-                messages=messages,
-                input=input,
-            )
-        elif self.routing_strategy == "simple-shuffle":
-            # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
-            ############## Check if we can do a RPM/TPM based weighted pick #################
-            rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
-            if rpm is not None:
-                # use weight-random pick if rpms provided
-                rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
-                verbose_router_logger.debug(f"\nrpms {rpms}")
-                total_rpm = sum(rpms)
-                weights = [rpm / total_rpm for rpm in rpms]
-                verbose_router_logger.debug(f"\n weights {weights}")
-                # Perform weighted random pick
-                selected_index = random.choices(range(len(rpms)), weights=weights)[0]
-                verbose_router_logger.debug(f"\n selected index, {selected_index}")
-                deployment = healthy_deployments[selected_index]
-                verbose_router_logger.info(
-                    f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
-                )
-                return deployment or deployment[0]
-            ############## Check if we can do a RPM/TPM based weighted pick #################
-            tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
-            if tpm is not None:
-                # use weight-random pick if rpms provided
-                tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
-                verbose_router_logger.debug(f"\ntpms {tpms}")
-                total_tpm = sum(tpms)
-                weights = [tpm / total_tpm for tpm in tpms]
-                verbose_router_logger.debug(f"\n weights {weights}")
-                # Perform weighted random pick
-                selected_index = random.choices(range(len(tpms)), weights=weights)[0]
-                verbose_router_logger.debug(f"\n selected index, {selected_index}")
-                deployment = healthy_deployments[selected_index]
-                verbose_router_logger.info(
-                    f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
-                )
-                return deployment or deployment[0]
+                specific_deployment=specific_deployment,
+            )  # type: ignore
 
-            ############## No RPM/TPM passed, we do a random pick #################
-            item = random.choice(healthy_deployments)
-            return item or item[0]
-        if deployment is None:
+            if isinstance(healthy_deployments, dict):
+                return healthy_deployments
+
+            # filter out the deployments currently cooling down
+            deployments_to_remove = []
+            # cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]
+            cooldown_deployments = await self._async_get_cooldown_deployments()
+            verbose_router_logger.debug(
+                f"async cooldown deployments: {cooldown_deployments}"
+            )
+            # Find deployments in model_list whose model_id is cooling down
+            for deployment in healthy_deployments:
+                deployment_id = deployment["model_info"]["id"]
+                if deployment_id in cooldown_deployments:
+                    deployments_to_remove.append(deployment)
+            # remove unhealthy deployments from healthy deployments
+            for deployment in deployments_to_remove:
+                healthy_deployments.remove(deployment)
+
+            # filter pre-call checks
+            _allowed_model_region = (
+                request_kwargs.get("allowed_model_region")
+                if request_kwargs is not None
+                else None
+            )
+
+            if self.enable_pre_call_checks and messages is not None:
+                healthy_deployments = self._pre_call_checks(
+                    model=model,
+                    healthy_deployments=healthy_deployments,
+                    messages=messages,
+                    request_kwargs=request_kwargs,
+                )
+
+            if len(healthy_deployments) == 0:
+                if _allowed_model_region is None:
+                    _allowed_model_region = "n/a"
+                raise ValueError(
+                    f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}. pre-call-checks={self.enable_pre_call_checks}, allowed_model_region={_allowed_model_region}"
+                )
+
+            if (
+                self.routing_strategy == "usage-based-routing-v2"
+                and self.lowesttpm_logger_v2 is not None
+            ):
+                deployment = (
+                    await self.lowesttpm_logger_v2.async_get_available_deployments(
+                        model_group=model,
+                        healthy_deployments=healthy_deployments,  # type: ignore
+                        messages=messages,
+                        input=input,
+                    )
+                )
+            if (
+                self.routing_strategy == "cost-based-routing"
+                and self.lowestcost_logger is not None
+            ):
+                deployment = (
+                    await self.lowestcost_logger.async_get_available_deployments(
+                        model_group=model,
+                        healthy_deployments=healthy_deployments,  # type: ignore
+                        messages=messages,
+                        input=input,
+                    )
+                )
+            elif self.routing_strategy == "simple-shuffle":
+                # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
+                ############## Check if we can do a RPM/TPM based weighted pick #################
+                rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
+                if rpm is not None:
+                    # use weight-random pick if rpms provided
+                    rpms = [
+                        m["litellm_params"].get("rpm", 0) for m in healthy_deployments
+                    ]
+                    verbose_router_logger.debug(f"\nrpms {rpms}")
+                    total_rpm = sum(rpms)
+                    weights = [rpm / total_rpm for rpm in rpms]
+                    verbose_router_logger.debug(f"\n weights {weights}")
+                    # Perform weighted random pick
+                    selected_index = random.choices(range(len(rpms)), weights=weights)[
+                        0
+                    ]
+                    verbose_router_logger.debug(f"\n selected index, {selected_index}")
+                    deployment = healthy_deployments[selected_index]
+                    verbose_router_logger.info(
+                        f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
+                    )
+                    return deployment or deployment[0]
+                ############## Check if we can do a RPM/TPM based weighted pick #################
+                tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
+                if tpm is not None:
+                    # use weight-random pick if rpms provided
+                    tpms = [
+                        m["litellm_params"].get("tpm", 0) for m in healthy_deployments
+                    ]
+                    verbose_router_logger.debug(f"\ntpms {tpms}")
+                    total_tpm = sum(tpms)
+                    weights = [tpm / total_tpm for tpm in tpms]
+                    verbose_router_logger.debug(f"\n weights {weights}")
+                    # Perform weighted random pick
+                    selected_index = random.choices(range(len(tpms)), weights=weights)[
+                        0
+                    ]
+                    verbose_router_logger.debug(f"\n selected index, {selected_index}")
+                    deployment = healthy_deployments[selected_index]
+                    verbose_router_logger.info(
+                        f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment) or deployment[0]} for model: {model}"
+                    )
+                    return deployment or deployment[0]
+
+                ############## No RPM/TPM passed, we do a random pick #################
+                item = random.choice(healthy_deployments)
+                return item or item[0]
+            if deployment is None:
+                verbose_router_logger.info(
+                    f"get_available_deployment for model: {model}, No deployment available"
+                )
+                raise ValueError(
+                    f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}"
+                )
             verbose_router_logger.info(
-                f"get_available_deployment for model: {model}, No deployment available"
+                f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}"
             )
-            raise ValueError(
-                f"{RouterErrors.no_deployments_available.value}, Try again in {self.cooldown_time} seconds. Passed model={model}"
-            )
-        verbose_router_logger.info(
-            f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}"
-        )
 
-        return deployment
+            return deployment
+        except Exception as e:
+            traceback_exception = traceback.format_exc()
+            # if router rejects call -> log to langfuse/otel/etc.
+            if request_kwargs is not None:
+                logging_obj = request_kwargs.get("litellm_logging_obj", None)
+                if logging_obj is not None:
+                    ## LOGGING
+                    threading.Thread(
+                        target=logging_obj.failure_handler,
+                        args=(e, traceback_exception),
+                    ).start()  # log response
+                    # Handle any exceptions that might occur during streaming
+                    asyncio.create_task(
+                        logging_obj.async_failure_handler(e, traceback_exception)  # type: ignore
+                    )
+            raise e
 
     def get_available_deployment(
         self,

From 02ffed25453f5dbe15f52811dcc937f531b81148 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 19:05:34 -0700
Subject: [PATCH 15/68] feat - refactor /spend/calculate

---
 litellm/proxy/_types.py                       |  6 ++
 .../spend_management_endpoints.py             | 75 +++++++++++++++++--
 2 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 0883763d1c..640c7695a0 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1627,3 +1627,9 @@ class CommonProxyErrors(enum.Enum):
     no_llm_router = "No models configured on proxy"
     not_allowed_access = "Admin-only endpoint. Not allowed to access this."
     not_premium_user = "You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+
+
+class SpendCalculateRequest(LiteLLMBase):
+    model: Optional[str] = None
+    messages: Optional[List] = None
+    completion_response: Optional[dict] = None
diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index 11edd18873..8089c7acbe 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -1199,7 +1199,7 @@ async def _get_spend_report_for_time_range(
         }
     },
 )
-async def calculate_spend(request: Request):
+async def calculate_spend(request: SpendCalculateRequest):
     """
     Accepts all the params of completion_cost.
 
@@ -1248,14 +1248,75 @@ async def calculate_spend(request: Request):
     }'
     ```
     """
-    from litellm import completion_cost
+    try:
+        from litellm import completion_cost
+        from litellm.cost_calculator import CostPerToken
+        from litellm.proxy.proxy_server import llm_router
 
-    data = await request.json()
-    if "completion_response" in data:
-        data["completion_response"] = litellm.ModelResponse(
-            **data["completion_response"]
+        _cost = None
+        if request.model is not None:
+            if request.messages is None:
+                raise HTTPException(
+                    status_code=400,
+                    detail="Bad Request - messages must be provided if 'model' is provided",
+                )
+
+            # check if model in llm_router
+            _model_in_llm_router = None
+            cost_per_token: Optional[CostPerToken] = None
+            if llm_router is not None:
+                for model in llm_router.model_list:
+                    if model.get("model_name") == request.model:
+                        _model_in_llm_router = model
+
+            """
+            3 cases for /spend/calculate
+
+            1. user passes model, and model is defined on litellm config.yaml or in DB. use info on config or in DB in this case
+            2. user passes model, and model is not defined on litellm config.yaml or in DB. Pass model as is to litellm.completion_cost
+            3. user passes completion_response
+            
+            """
+            if _model_in_llm_router is not None:
+                _litellm_params = _model_in_llm_router.get("litellm_params")
+                _litellm_model_name = _litellm_params.get("model")
+                input_cost_per_token = _litellm_params.get("input_cost_per_token")
+                output_cost_per_token = _litellm_params.get("output_cost_per_token")
+                if (
+                    input_cost_per_token is not None
+                    or output_cost_per_token is not None
+                ):
+                    cost_per_token = CostPerToken(
+                        input_cost_per_token=input_cost_per_token,
+                        output_cost_per_token=output_cost_per_token,
+                    )
+
+                _cost = completion_cost(
+                    model=_litellm_model_name,
+                    messages=request.messages,
+                    custom_cost_per_token=cost_per_token,
+                )
+            else:
+                _cost = completion_cost(model=request.model, messages=request.messages)
+        else:
+            _completion_response = litellm.ModelResponse(request.completion_response)
+            _cost = completion_cost(completion_response=_completion_response)
+        return {"cost": _cost}
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "detail", str(e)),
+                type=getattr(e, "type", "None"),
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
+            )
+        error_msg = f"{str(e)}"
+        raise ProxyException(
+            message=getattr(e, "message", error_msg),
+            type=getattr(e, "type", "None"),
+            param=getattr(e, "param", "None"),
+            code=getattr(e, "status_code", 500),
         )
-    return {"cost": completion_cost(**data)}
 
 
 @router.get(

From 1ff0129a94e8fa3b422e38b49d6ec24df6745791 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 19:13:56 -0700
Subject: [PATCH 16/68] fix(vertex_httpx.py): cover gemini content violation
 (on prompt)

---
 litellm/llms/vertex_httpx.py            | 87 +++++++++++++++++++++----
 litellm/proxy/_super_secret_config.yaml |  3 +
 litellm/types/llms/vertex_ai.py         |  6 +-
 3 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py
index 63bcd9f4f5..028c3f7217 100644
--- a/litellm/llms/vertex_httpx.py
+++ b/litellm/llms/vertex_httpx.py
@@ -563,6 +563,43 @@ class VertexLLM(BaseLLM):
             )
 
         ## CHECK IF RESPONSE FLAGGED
+        if "promptFeedback" in completion_response:
+            if "blockReason" in completion_response["promptFeedback"]:
+                # If set, the prompt was blocked and no candidates are returned. Rephrase your prompt
+                model_response.choices[0].finish_reason = "content_filter"
+
+                chat_completion_message: ChatCompletionResponseMessage = {
+                    "role": "assistant",
+                    "content": None,
+                }
+
+                choice = litellm.Choices(
+                    finish_reason="content_filter",
+                    index=0,
+                    message=chat_completion_message,  # type: ignore
+                    logprobs=None,
+                    enhancements=None,
+                )
+
+                model_response.choices = [choice]
+
+                ## GET USAGE ##
+                usage = litellm.Usage(
+                    prompt_tokens=completion_response["usageMetadata"][
+                        "promptTokenCount"
+                    ],
+                    completion_tokens=completion_response["usageMetadata"].get(
+                        "candidatesTokenCount", 0
+                    ),
+                    total_tokens=completion_response["usageMetadata"][
+                        "totalTokenCount"
+                    ],
+                )
+
+                setattr(model_response, "usage", usage)
+
+                return model_response
+
         if len(completion_response["candidates"]) > 0:
             content_policy_violations = (
                 VertexGeminiConfig().get_flagged_finish_reasons()
@@ -573,16 +610,40 @@ class VertexLLM(BaseLLM):
                 in content_policy_violations.keys()
             ):
                 ## CONTENT POLICY VIOLATION ERROR
-                raise VertexAIError(
-                    status_code=400,
-                    message="The response was blocked. Reason={}. Raw Response={}".format(
-                        content_policy_violations[
-                            completion_response["candidates"][0]["finishReason"]
-                        ],
-                        completion_response,
-                    ),
+                model_response.choices[0].finish_reason = "content_filter"
+
+                chat_completion_message = {
+                    "role": "assistant",
+                    "content": None,
+                }
+
+                choice = litellm.Choices(
+                    finish_reason="content_filter",
+                    index=0,
+                    message=chat_completion_message,  # type: ignore
+                    logprobs=None,
+                    enhancements=None,
                 )
 
+                model_response.choices = [choice]
+
+                ## GET USAGE ##
+                usage = litellm.Usage(
+                    prompt_tokens=completion_response["usageMetadata"][
+                        "promptTokenCount"
+                    ],
+                    completion_tokens=completion_response["usageMetadata"].get(
+                        "candidatesTokenCount", 0
+                    ),
+                    total_tokens=completion_response["usageMetadata"][
+                        "totalTokenCount"
+                    ],
+                )
+
+                setattr(model_response, "usage", usage)
+
+                return model_response
+
         model_response.choices = []  # type: ignore
 
         ## GET MODEL ##
@@ -590,9 +651,7 @@ class VertexLLM(BaseLLM):
 
         try:
             ## GET TEXT ##
-            chat_completion_message: ChatCompletionResponseMessage = {
-                "role": "assistant"
-            }
+            chat_completion_message = {"role": "assistant"}
             content_str = ""
             tools: List[ChatCompletionToolCallChunk] = []
             for idx, candidate in enumerate(completion_response["candidates"]):
@@ -632,9 +691,9 @@ class VertexLLM(BaseLLM):
             ## GET USAGE ##
             usage = litellm.Usage(
                 prompt_tokens=completion_response["usageMetadata"]["promptTokenCount"],
-                completion_tokens=completion_response["usageMetadata"][
-                    "candidatesTokenCount"
-                ],
+                completion_tokens=completion_response["usageMetadata"].get(
+                    "candidatesTokenCount", 0
+                ),
                 total_tokens=completion_response["usageMetadata"]["totalTokenCount"],
             )
 
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index 04a4806c12..c5f1b47687 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -1,4 +1,7 @@
 model_list:
+- model_name: gemini-1.5-flash-gemini
+  litellm_params:
+    model: gemini/gemini-1.5-flash
 - litellm_params:
     api_base: http://0.0.0.0:8080
     api_key: ''
diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py
index 1612f8761f..2dda57c2e9 100644
--- a/litellm/types/llms/vertex_ai.py
+++ b/litellm/types/llms/vertex_ai.py
@@ -227,9 +227,9 @@ class PromptFeedback(TypedDict):
     blockReasonMessage: str
 
 
-class UsageMetadata(TypedDict):
-    promptTokenCount: int
-    totalTokenCount: int
+class UsageMetadata(TypedDict, total=False):
+    promptTokenCount: Required[int]
+    totalTokenCount: Required[int]
     candidatesTokenCount: int
 
 

From 8e6e5a6d37cd30f17304ccf7bfa5b58aa76e74fc Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 19:22:20 -0700
Subject: [PATCH 17/68] fix(vertex_httpx.py): Return empty model response for
 content filter violations

---
 litellm/llms/vertex_httpx.py                  |  6 +--
 .../tests/test_amazing_vertex_completion.py   | 41 ++++++++++++++-----
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py
index 028c3f7217..856b05f61c 100644
--- a/litellm/llms/vertex_httpx.py
+++ b/litellm/llms/vertex_httpx.py
@@ -562,6 +562,9 @@ class VertexLLM(BaseLLM):
                 status_code=422,
             )
 
+        ## GET MODEL ##
+        model_response.model = model
+
         ## CHECK IF RESPONSE FLAGGED
         if "promptFeedback" in completion_response:
             if "blockReason" in completion_response["promptFeedback"]:
@@ -646,9 +649,6 @@ class VertexLLM(BaseLLM):
 
         model_response.choices = []  # type: ignore
 
-        ## GET MODEL ##
-        model_response.model = model
-
         try:
             ## GET TEXT ##
             chat_completion_message = {"role": "assistant"}
diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py
index fb28912493..c9e5501a8c 100644
--- a/litellm/tests/test_amazing_vertex_completion.py
+++ b/litellm/tests/test_amazing_vertex_completion.py
@@ -696,6 +696,18 @@ async def test_gemini_pro_function_calling_httpx(provider, sync_mode):
             pytest.fail("An unexpected exception occurred - {}".format(str(e)))
 
 
+def vertex_httpx_mock_reject_prompt_post(*args, **kwargs):
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.headers = {"Content-Type": "application/json"}
+    mock_response.json.return_value = {
+        "promptFeedback": {"blockReason": "OTHER"},
+        "usageMetadata": {"promptTokenCount": 6285, "totalTokenCount": 6285},
+    }
+
+    return mock_response
+
+
 # @pytest.mark.skip(reason="exhausted vertex quota. need to refactor to mock the call")
 def vertex_httpx_mock_post(url, data=None, json=None, headers=None):
     mock_response = MagicMock()
@@ -817,8 +829,11 @@ def vertex_httpx_mock_post(url, data=None, json=None, headers=None):
 
 
 @pytest.mark.parametrize("provider", ["vertex_ai_beta"])  # "vertex_ai",
+@pytest.mark.parametrize("content_filter_type", ["prompt", "response"])  # "vertex_ai",
 @pytest.mark.asyncio
-async def test_gemini_pro_json_schema_httpx_content_policy_error(provider):
+async def test_gemini_pro_json_schema_httpx_content_policy_error(
+    provider, content_filter_type
+):
     load_vertex_ai_credentials()
     litellm.set_verbose = True
     messages = [
@@ -839,16 +854,20 @@ Using this JSON schema:
 
     client = HTTPHandler()
 
-    with patch.object(client, "post", side_effect=vertex_httpx_mock_post) as mock_call:
-        try:
-            response = completion(
-                model="vertex_ai_beta/gemini-1.5-flash",
-                messages=messages,
-                response_format={"type": "json_object"},
-                client=client,
-            )
-        except litellm.ContentPolicyViolationError as e:
-            pass
+    if content_filter_type == "prompt":
+        _side_effect = vertex_httpx_mock_reject_prompt_post
+    else:
+        _side_effect = vertex_httpx_mock_post
+
+    with patch.object(client, "post", side_effect=_side_effect) as mock_call:
+        response = completion(
+            model="vertex_ai_beta/gemini-1.5-flash",
+            messages=messages,
+            response_format={"type": "json_object"},
+            client=client,
+        )
+
+        assert response.choices[0].finish_reason == "content_filter"
 
         mock_call.assert_called_once()
 

From 11117665e1ee9e9ac9207f68dc17c63dc6d77e4e Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 19:32:52 -0700
Subject: [PATCH 18/68] test - spend/calculate endpoints

---
 .../spend_management_endpoints.py             |   9 +-
 .../tests/test_spend_calculate_endpoint.py    | 103 ++++++++++++++++++
 2 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 litellm/tests/test_spend_calculate_endpoint.py

diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index 8089c7acbe..abbdc3419e 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -1298,9 +1298,14 @@ async def calculate_spend(request: SpendCalculateRequest):
                 )
             else:
                 _cost = completion_cost(model=request.model, messages=request.messages)
-        else:
-            _completion_response = litellm.ModelResponse(request.completion_response)
+        elif request.completion_response is not None:
+            _completion_response = litellm.ModelResponse(**request.completion_response)
             _cost = completion_cost(completion_response=_completion_response)
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail="Bad Request - Either 'model' or 'completion_response' must be provided",
+            )
         return {"cost": _cost}
     except Exception as e:
         if isinstance(e, HTTPException):
diff --git a/litellm/tests/test_spend_calculate_endpoint.py b/litellm/tests/test_spend_calculate_endpoint.py
new file mode 100644
index 0000000000..f8aff337ec
--- /dev/null
+++ b/litellm/tests/test_spend_calculate_endpoint.py
@@ -0,0 +1,103 @@
+import os
+import sys
+
+import pytest
+from dotenv import load_dotenv
+from fastapi import Request
+from fastapi.routing import APIRoute
+
+import litellm
+from litellm.proxy._types import SpendCalculateRequest
+from litellm.proxy.spend_tracking.spend_management_endpoints import calculate_spend
+from litellm.router import Router
+
+# this file is to test litellm/proxy
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+
+
+@pytest.mark.asyncio
+async def test_spend_calc_model_messages():
+    cost_obj = await calculate_spend(
+        request=SpendCalculateRequest(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "user", "content": "What is the capital of France?"},
+            ],
+        )
+    )
+
+    print("calculated cost", cost_obj)
+    cost = cost_obj["cost"]
+    assert cost > 0.0
+
+
+@pytest.mark.asyncio
+async def test_spend_calc_model_on_router_messages():
+    from litellm.proxy.proxy_server import llm_router as init_llm_router
+
+    temp_llm_router = Router(
+        model_list=[
+            {
+                "model_name": "special-llama-model",
+                "litellm_params": {
+                    "model": "groq/llama3-8b-8192",
+                },
+            }
+        ]
+    )
+
+    setattr(litellm.proxy.proxy_server, "llm_router", temp_llm_router)
+
+    cost_obj = await calculate_spend(
+        request=SpendCalculateRequest(
+            model="special-llama-model",
+            messages=[
+                {"role": "user", "content": "What is the capital of France?"},
+            ],
+        )
+    )
+
+    print("calculated cost", cost_obj)
+    _cost = cost_obj["cost"]
+
+    assert _cost > 0.0
+
+    # set router to init value
+    setattr(litellm.proxy.proxy_server, "llm_router", init_llm_router)
+
+
+@pytest.mark.asyncio
+async def test_spend_calc_using_response():
+    cost_obj = await calculate_spend(
+        request=SpendCalculateRequest(
+            completion_response={
+                "id": "chatcmpl-3bc7abcd-f70b-48ab-a16c-dfba0b286c86",
+                "choices": [
+                    {
+                        "finish_reason": "stop",
+                        "index": 0,
+                        "message": {
+                            "content": "Yooo! What's good?",
+                            "role": "assistant",
+                        },
+                    }
+                ],
+                "created": "1677652288",
+                "model": "groq/llama3-8b-8192",
+                "object": "chat.completion",
+                "system_fingerprint": "fp_873a560973",
+                "usage": {
+                    "completion_tokens": 8,
+                    "prompt_tokens": 12,
+                    "total_tokens": 20,
+                },
+            }
+        )
+    )
+
+    print("calculated cost", cost_obj)
+    cost = cost_obj["cost"]
+    assert cost > 0.0

From d182ea0f77668902e4a6a8fe5bb32c914b387188 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 19:41:29 -0700
Subject: [PATCH 19/68] fix(utils.py): catch 422-status errors

---
 litellm/llms/replicate.py | 27 ++++++++++++++++++++-------
 litellm/utils.py          |  8 ++++++++
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/litellm/llms/replicate.py b/litellm/llms/replicate.py
index ce62e51e90..56549cfd4a 100644
--- a/litellm/llms/replicate.py
+++ b/litellm/llms/replicate.py
@@ -1,13 +1,18 @@
-import os, types
+import asyncio
 import json
-import requests  # type: ignore
+import os
 import time
-from typing import Callable, Optional, Union, Tuple, Any
-from litellm.utils import ModelResponse, Usage, CustomStreamWrapper
-import litellm, asyncio
+import types
+from typing import Any, Callable, Optional, Tuple, Union
+
 import httpx  # type: ignore
-from .prompt_templates.factory import prompt_factory, custom_prompt
+import requests  # type: ignore
+
+import litellm
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
+
+from .prompt_templates.factory import custom_prompt, prompt_factory
 
 
 class ReplicateError(Exception):
@@ -329,7 +334,15 @@ async def async_handle_prediction_response_streaming(
             response_data = response.json()
             status = response_data["status"]
             if "output" in response_data:
-                output_string = "".join(response_data["output"])
+                try:
+                    output_string = "".join(response_data["output"])
+                except Exception as e:
+                    raise ReplicateError(
+                        status_code=422,
+                        message="Unable to parse response. Got={}".format(
+                            response_data["output"]
+                        ),
+                    )
                 new_output = output_string[len(previous_output) :]
                 print_verbose(f"New chunk: {new_output}")
                 yield {"output": new_output, "status": status}
diff --git a/litellm/utils.py b/litellm/utils.py
index ce66d0fbb0..1bc8bf771f 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -6068,6 +6068,14 @@ def exception_type(
                             model=model,
                             llm_provider="replicate",
                         )
+                    elif original_exception.status_code == 422:
+                        exception_mapping_worked = True
+                        raise UnprocessableEntityError(
+                            message=f"ReplicateException - {original_exception.message}",
+                            llm_provider="replicate",
+                            model=model,
+                            response=original_exception.response,
+                        )
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
                         raise RateLimitError(

From 5e1403cd0af8af747875662ce334cbb8489af678 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 19:50:35 -0700
Subject: [PATCH 20/68] add helper to check route_in_additonal_public_routes

---
 litellm/proxy/auth/auth_utils.py        | 42 +++++++++++++++++++++++++
 litellm/proxy/auth/user_api_key_auth.py |  6 ++--
 2 files changed, 46 insertions(+), 2 deletions(-)
 create mode 100644 litellm/proxy/auth/auth_utils.py

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
new file mode 100644
index 0000000000..60e59a5842
--- /dev/null
+++ b/litellm/proxy/auth/auth_utils.py
@@ -0,0 +1,42 @@
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import LiteLLMRoutes
+from litellm.proxy.proxy_server import general_settings, premium_user
+
+
+def route_in_additonal_public_routes(current_route: str):
+    """
+    Helper to check if the user defined public_routes on config.yaml
+
+    Parameters:
+    - current_route: str - the route the user is trying to call
+
+    Returns:
+    - bool - True if the route is defined in public_routes
+    - bool - False if the route is not defined in public_routes
+
+
+    In order to use this the litellm config.yaml should have the following in general_settings:
+
+    ```yaml
+    general_settings:
+        master_key: sk-1234
+        public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
+    ```
+    """
+
+    # check if user is premium_user - if not do nothing
+    try:
+        if premium_user is not True:
+            return False
+        # check if this is defined on the config
+        if general_settings is None:
+            return False
+
+        routes_defined = general_settings.get("public_routes", [])
+        if current_route in routes_defined:
+            return True
+
+        return False
+    except Exception as e:
+        verbose_proxy_logger.error(f"route_in_additonal_public_routes: {str(e)}")
+        return False
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index f6e3a0dfeb..d3e937734c 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -56,6 +56,7 @@ from litellm.proxy.auth.auth_checks import (
     get_user_object,
     log_to_opentelemetry,
 )
+from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
 from litellm.proxy.utils import _to_ns
 
@@ -137,8 +138,9 @@ async def user_api_key_auth(
         """
         route: str = request.url.path
 
-        if route in LiteLLMRoutes.public_routes.value or route in general_settings.get(
-            "public_routes", []
+        if (
+            route in LiteLLMRoutes.public_routes.value
+            or route_in_additonal_public_routes(current_route=route)
         ):
             # check if public endpoint
             return UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY)

From bd2298eba4598e338904f2eea1a9dd055af8a8fd Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 19:51:23 -0700
Subject: [PATCH 21/68] example cofnig with public routes

---
 litellm/proxy/proxy_config.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index 8898dd8cb5..caa6bc13b9 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -21,9 +21,8 @@ model_list:
 general_settings:
   master_key: sk-1234
   alerting: ["slack", "email"]
-  public_routes: [
-    "/spend/calculate",
-  ]
+  public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
+
 
 litellm_settings:
   success_callback: ["prometheus"]

From 2aa0ce50b5438b1c1f4ea0cb8ad2405bc3ef7031 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 19:58:53 -0700
Subject: [PATCH 22/68] fix importing litellm

---
 litellm/proxy/auth/auth_utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index 60e59a5842..cc09a9689b 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -1,6 +1,4 @@
 from litellm._logging import verbose_proxy_logger
-from litellm.proxy._types import LiteLLMRoutes
-from litellm.proxy.proxy_server import general_settings, premium_user
 
 
 def route_in_additonal_public_routes(current_route: str):
@@ -25,6 +23,9 @@ def route_in_additonal_public_routes(current_route: str):
     """
 
     # check if user is premium_user - if not do nothing
+    from litellm.proxy._types import LiteLLMRoutes
+    from litellm.proxy.proxy_server import general_settings, premium_user
+
     try:
         if premium_user is not True:
             return False

From 7f8c502a44ad5fb9601c29de1fcf23689157e064 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 20:54:43 -0700
Subject: [PATCH 23/68] docs control available public routes

---
 docs/my-website/docs/enterprise.md       |  1 +
 docs/my-website/docs/proxy/enterprise.md | 43 ++++++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md
index 0edf937ed3..2d45ea3ea7 100644
--- a/docs/my-website/docs/enterprise.md
+++ b/docs/my-website/docs/enterprise.md
@@ -12,6 +12,7 @@ This covers:
 - ✅ [**Secure UI access with Single Sign-On**](../docs/proxy/ui.md#setup-ssoauth-for-ui)
 - ✅ [**Audit Logs with retention policy**](../docs/proxy/enterprise.md#audit-logs)
 - ✅ [**JWT-Auth**](../docs/proxy/token_auth.md)
+- ✅ [**Control available public, private routes**](../docs/proxy/enterprise.md#control-available-public-private-routes)
 - ✅ [**Prompt Injection Detection**](#prompt-injection-detection-lakeraai)
 - ✅ [**Invite Team Members to access `/spend` Routes**](../docs/proxy/cost_tracking#allowing-non-proxy-admins-to-access-spend-endpoints)
 - ✅ **Feature Prioritization**
diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md
index e657d3b73e..40a5261cd5 100644
--- a/docs/my-website/docs/proxy/enterprise.md
+++ b/docs/my-website/docs/proxy/enterprise.md
@@ -14,6 +14,7 @@ Features:
 - ✅ [SSO for Admin UI](./ui.md#✨-enterprise-features)
 - ✅ [Audit Logs](#audit-logs)
 - ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags)
+- ✅ [Control available public, private routes](#control-available-public-private-routes)
 - ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests)
 - ✅ [Content Moderation with LLM Guard, LlamaGuard, Google Text Moderations](#content-moderation)
 - ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai)
@@ -448,6 +449,48 @@ Expected Response
 
 
 
+## Control available public, private routes
+
+:::info
+
+❓ Use this when you want to make an existing private route -> public
+
+Example - Make `/spend/calculate` a publicly available route (by default `/spend/calculate` on LiteLLM Proxy requires authentication)
+
+:::
+
+#### Usage - Define public routes
+
+**Step 1** - set allowed public routes on config.yaml 
+
+`LiteLLMRoutes.public_routes` is an ENUM corresponding to the default public routes on LiteLLM. [You can see this here](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/_types.py)
+
+```yaml
+general_settings:
+  master_key: sk-1234
+  public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
+```
+
+**Step 2** - start proxy 
+
+```shell
+litellm --config config.yaml
+```
+
+**Step 3** - Test it 
+
+```shell
+curl --request POST \
+  --url 'http://localhost:4000/spend/calculate' \
+  --header 'Content-Type: application/json' \
+  --data '{
+    "model": "gpt-4",
+    "messages": [{"role": "user", "content": "Hey, how'\''s it going?"}]
+  }'
+```
+
+🎉 Expect this endpoint to work without an `Authorization / Bearer Token`
+
 
 
 

From a64d1de80f988fda56304db2cc74abac1a82d500 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 21:13:01 -0700
Subject: [PATCH 24/68] test - aliases on /spend/calculate

---
 .../tests/test_spend_calculate_endpoint.py    | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/litellm/tests/test_spend_calculate_endpoint.py b/litellm/tests/test_spend_calculate_endpoint.py
index f8aff337ec..8bdd4a54d8 100644
--- a/litellm/tests/test_spend_calculate_endpoint.py
+++ b/litellm/tests/test_spend_calculate_endpoint.py
@@ -101,3 +101,41 @@ async def test_spend_calc_using_response():
     print("calculated cost", cost_obj)
     cost = cost_obj["cost"]
     assert cost > 0.0
+
+
+@pytest.mark.asyncio
+async def test_spend_calc_model_alias_on_router_messages():
+    from litellm.proxy.proxy_server import llm_router as init_llm_router
+
+    temp_llm_router = Router(
+        model_list=[
+            {
+                "model_name": "gpt-4o",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                },
+            }
+        ],
+        model_group_alias={
+            "gpt4o": "gpt-4o",
+        },
+    )
+
+    setattr(litellm.proxy.proxy_server, "llm_router", temp_llm_router)
+
+    cost_obj = await calculate_spend(
+        request=SpendCalculateRequest(
+            model="gpt4o",
+            messages=[
+                {"role": "user", "content": "What is the capital of France?"},
+            ],
+        )
+    )
+
+    print("calculated cost", cost_obj)
+    _cost = cost_obj["cost"]
+
+    assert _cost > 0.0
+
+    # set router to init value
+    setattr(litellm.proxy.proxy_server, "llm_router", init_llm_router)

From 9e7d8ba910cac16a61a88b109a41181ceb746746 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 21:14:00 -0700
Subject: [PATCH 25/68] /spend/calculate use model aliases on this endpoint

---
 .../spend_management_endpoints.py             | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index abbdc3419e..1fbd95b3cf 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -1265,9 +1265,22 @@ async def calculate_spend(request: SpendCalculateRequest):
             _model_in_llm_router = None
             cost_per_token: Optional[CostPerToken] = None
             if llm_router is not None:
-                for model in llm_router.model_list:
-                    if model.get("model_name") == request.model:
-                        _model_in_llm_router = model
+                if (
+                    llm_router.model_group_alias is not None
+                    and request.model in llm_router.model_group_alias
+                ):
+                    # lookup alias in llm_router
+                    _model_group_name = llm_router.model_group_alias[request.model]
+                    for model in llm_router.model_list:
+                        if model.get("model_name") == _model_group_name:
+                            _model_in_llm_router = model
+
+                else:
+                    # no model_group aliases set -> try finding model in llm_router
+                    # find model in llm_router
+                    for model in llm_router.model_list:
+                        if model.get("model_name") == request.model:
+                            _model_in_llm_router = model
 
             """
             3 cases for /spend/calculate

From 93dbdf6d928c25374df81aca8807f066dc1d0496 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 24 Jun 2024 21:15:36 -0700
Subject: [PATCH 26/68] =?UTF-8?q?bump:=20version=201.40.25=20=E2=86=92=201?=
 =?UTF-8?q?.40.26?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index fc3526dcc5..6b4884b5bb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.40.25"
+version = "1.40.26"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -90,7 +90,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.40.25"
+version = "1.40.26"
 version_files = [
     "pyproject.toml:^version"
 ]

From f8b390d421d6dbc7a4bd1d812ddf855bd4244841 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 21:43:40 -0700
Subject: [PATCH 27/68] fix(langfuse.py): cleanup

---
 litellm/integrations/langfuse.py | 38 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py
index 794524684d..eae8b8e22a 100644
--- a/litellm/integrations/langfuse.py
+++ b/litellm/integrations/langfuse.py
@@ -36,9 +36,9 @@ class LangFuseLogger:
         self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
 
         parameters = {
-            "public_key": "pk-lf-a65841e9-5192-4397-a679-cfff029fd5b0",
-            "secret_key": "sk-lf-d58c2891-3717-4f98-89dd-df44826215fd",
-            "host": "https://us.cloud.langfuse.com",
+            "public_key": self.public_key,
+            "secret_key": self.secret_key,
+            "host": self.langfuse_host,
             "release": self.langfuse_release,
             "debug": self.langfuse_debug,
             "flush_interval": flush_interval,  # flush interval in seconds
@@ -311,22 +311,22 @@ class LangFuseLogger:
 
         try:
             tags = []
-            # try:
-            #     metadata = copy.deepcopy(
-            #         metadata
-            #     )  # Avoid modifying the original metadata
-            # except:
-            new_metadata = {}
-            for key, value in metadata.items():
-                if (
-                    isinstance(value, list)
-                    or isinstance(value, dict)
-                    or isinstance(value, str)
-                    or isinstance(value, int)
-                    or isinstance(value, float)
-                ):
-                    new_metadata[key] = copy.deepcopy(value)
-            metadata = new_metadata
+            try:
+                metadata = copy.deepcopy(
+                    metadata
+                )  # Avoid modifying the original metadata
+            except:
+                new_metadata = {}
+                for key, value in metadata.items():
+                    if (
+                        isinstance(value, list)
+                        or isinstance(value, dict)
+                        or isinstance(value, str)
+                        or isinstance(value, int)
+                        or isinstance(value, float)
+                    ):
+                        new_metadata[key] = copy.deepcopy(value)
+                metadata = new_metadata
 
             supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3")
             supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3")

From 86cb5aa031478e825b6b222a48d020a96de8c3e6 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 24 Jun 2024 22:25:39 -0700
Subject: [PATCH 28/68] docs(routing.md): add quickstart

---
 docs/my-website/docs/routing.md | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md
index fd4fb86588..de0a4a7965 100644
--- a/docs/my-website/docs/routing.md
+++ b/docs/my-website/docs/routing.md
@@ -901,6 +901,39 @@ response = await router.acompletion(
 
 If a call fails after num_retries, fall back to another model group. 
 
+### Quick Start 
+
+```python
+from litellm import Router 
+router = Router(
+	model_list=[
+		{ # bad model
+			"model_name": "bad-model",
+			"litellm_params": {
+				"model": "openai/my-bad-model",
+				"api_key": "my-bad-api-key",
+				"mock_response": "Bad call"
+			},
+		},
+		{ # good model
+			"model_name": "my-good-model",
+			"litellm_params": {
+				"model": "gpt-4o",
+				"api_key": os.getenv("OPENAI_API_KEY"),
+				"mock_response": "Good call"
+			},
+		},
+	],
+	fallbacks=[{"bad-model": ["my-good-model"]}] # 👈 KEY CHANGE
+)
+
+response = router.completion(
+	model="bad-model",
+	messages=[{"role": "user", "content": "Hey, how's it going?"}],
+	mock_testing_fallbacks=True,
+)
+```
+
 If the error is a context window exceeded error, fall back to a larger model group (if given). 
 
 Fallbacks are done in-order - ["gpt-3.5-turbo, "gpt-4", "gpt-4-32k"], will do 'gpt-3.5-turbo' first, then 'gpt-4', etc.

From b69a092f56c713ac9dcaf9dceb59c29faec2d315 Mon Sep 17 00:00:00 2001
From: corrm <islamnofl.official@gmail.com>
Date: Tue, 25 Jun 2024 12:40:07 +0300
Subject: [PATCH 29/68] Rename ollama prompt: - 'Function' word to
 'FunctionName' - 'Tool Call' to `FunctionCall` - 'Tool Call Result' to
 'FunctionCall Result'

_I found that changes make some models better_
---
 litellm/llms/prompt_templates/factory.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index 109c5b8d83..7864d5ebc8 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -181,9 +181,9 @@ def ollama_pt(
                 for call in message["tool_calls"]:
                     function_name = call["function"]["name"]
                     arguments = json.loads(call["function"]["arguments"])
-                    prompt += f"### Tool Call ({call["id"]}):\nName: {function_name}\nArguments: {json.dumps(arguments)}\n\n"
+                    prompt += f"### FunctionCall ({call["id"]}):\nFunctionName: {function_name}\nArguments: {json.dumps(arguments)}\n\n"
             elif "tool_call_id" in message:
-                prompt += f"### Tool Call Result ({message["tool_call_id"]}):\n{message["content"]}\n\n"
+                prompt += f"### FunctionCall Result ({message["tool_call_id"]}):\n{message["content"]}\n\n"
             elif content:
                 prompt += f"### {role.capitalize()}:\n{content}\n\n"
 

From 5b720137fdb2e65c9584bc1dbdb7a7e50120eb7e Mon Sep 17 00:00:00 2001
From: corrm <islamnofl.official@gmail.com>
Date: Tue, 25 Jun 2024 13:53:27 +0300
Subject: [PATCH 30/68] Improve ollama prompt: this formula give good result
 with AutoGen

---
 litellm/llms/prompt_templates/factory.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index 7864d5ebc8..e359d36f4c 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -135,7 +135,7 @@ def convert_to_ollama_image(openai_image_url: str):
 
 
 def ollama_pt(
-    model, messages
+        model, messages
 ):  # https://github.com/ollama/ollama/blob/af4cf55884ac54b9e637cd71dadfe9b7a5685877/docs/modelfile.md#template
     if "instruct" in model:
         prompt = custom_prompt(
@@ -178,12 +178,27 @@ def ollama_pt(
             content = message.get("content", "")
 
             if "tool_calls" in message:
+                tool_calls = []
+
                 for call in message["tool_calls"]:
-                    function_name = call["function"]["name"]
+                    call_id: str = call["id"]
+                    function_name: str = call["function"]["name"]
                     arguments = json.loads(call["function"]["arguments"])
-                    prompt += f"### FunctionCall ({call["id"]}):\nFunctionName: {function_name}\nArguments: {json.dumps(arguments)}\n\n"
+
+                    tool_calls.append({
+                        "id": call_id,
+                        "type": "function",
+                        "function": {
+                            "name": function_name,
+                            "arguments": arguments
+                        }
+                    })
+
+                prompt += f"### Assistant:\nTool Calls: {json.dumps(tool_calls, indent=2)}\n\n"
+
             elif "tool_call_id" in message:
-                prompt += f"### FunctionCall Result ({message["tool_call_id"]}):\n{message["content"]}\n\n"
+                prompt += f"### User:\n{message["content"]}\n\n"
+
             elif content:
                 prompt += f"### {role.capitalize()}:\n{content}\n\n"
 

From a45e6064fa48fe6b4c574a52a851685c32e97749 Mon Sep 17 00:00:00 2001
From: Paul Gauthier <paul@paulg.com>
Date: Tue, 25 Jun 2024 07:35:49 -0700
Subject: [PATCH 31/68] Added openrouter/anthropic/claude-3.5-sonnet to model
 json

---
 model_prices_and_context_window.json | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index ef07d87ccb..45db237709 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -2073,6 +2073,18 @@
         "supports_function_calling": true,
         "supports_vision": true
     },
+    "openrouter/anthropic/claude-3.5-sonnet": {
+        "max_tokens": 4096,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000015,
+        "litellm_provider": "openrouter",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "tool_use_system_prompt_tokens": 159
+    },
     "openrouter/anthropic/claude-3-sonnet": {
         "max_tokens": 200000,
         "input_cost_per_token": 0.000003,

From 34493fb22584f897469d1a87a8f1d068bcce0236 Mon Sep 17 00:00:00 2001
From: Paul Gauthier <paul@paulg.com>
Date: Tue, 25 Jun 2024 07:43:58 -0700
Subject: [PATCH 32/68] Added openrouter/anthropic/claude-3-haiku-20240307

---
 model_prices_and_context_window.json | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 45db237709..c4bccfb08e 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -2073,6 +2073,18 @@
         "supports_function_calling": true,
         "supports_vision": true
     },
+    "openrouter/anthropic/claude-3-haiku-20240307": {
+        "max_tokens": 4096,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "input_cost_per_token": 0.00000025,
+        "output_cost_per_token": 0.00000125,
+        "litellm_provider": "openrouter",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "tool_use_system_prompt_tokens": 264
+    },
     "openrouter/anthropic/claude-3.5-sonnet": {
         "max_tokens": 4096,
         "max_input_tokens": 200000,

From 8765e120f07c1720e8b98dd74006951d352e673d Mon Sep 17 00:00:00 2001
From: Kyrylo Yefimenko <kyrylo@cast.ai>
Date: Tue, 25 Jun 2024 16:36:40 +0100
Subject: [PATCH 33/68] Fix Groq prices

---
 model_prices_and_context_window.json | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index ef07d87ccb..415d220f21 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -887,7 +887,7 @@
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "input_cost_per_token": 0.00000005,
-        "output_cost_per_token": 0.00000010,
+        "output_cost_per_token": 0.00000008,
         "litellm_provider": "groq",
         "mode": "chat",
         "supports_function_calling": true
@@ -906,8 +906,8 @@
         "max_tokens": 32768,
         "max_input_tokens": 32768,
         "max_output_tokens": 32768,
-        "input_cost_per_token": 0.00000027,
-        "output_cost_per_token": 0.00000027,
+        "input_cost_per_token": 0.00000024,
+        "output_cost_per_token": 0.00000024,
         "litellm_provider": "groq",
         "mode": "chat",
         "supports_function_calling": true
@@ -916,8 +916,8 @@
         "max_tokens": 8192,
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
-        "input_cost_per_token": 0.00000010,
-        "output_cost_per_token": 0.00000010,
+        "input_cost_per_token": 0.00000007,
+        "output_cost_per_token": 0.00000007,
         "litellm_provider": "groq",
         "mode": "chat",
         "supports_function_calling": true

From abf919eef244118ebf5b12bd98a3b8810a9aa59b Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 08:53:06 -0700
Subject: [PATCH 34/68] add nvidia nim to __init__

---
 litellm/__init__.py                      |  3 +++
 litellm/llms/prompt_templates/factory.py | 17 ++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index f07ce88092..d23247d531 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -401,6 +401,7 @@ openai_compatible_endpoints: List = [
     "codestral.mistral.ai/v1/chat/completions",
     "codestral.mistral.ai/v1/fim/completions",
     "api.groq.com/openai/v1",
+    "https://integrate.api.nvidia.com/v1",
     "api.deepseek.com/v1",
     "api.together.xyz/v1",
     "inference.friendli.ai/v1",
@@ -411,6 +412,7 @@ openai_compatible_providers: List = [
     "anyscale",
     "mistral",
     "groq",
+    "nvidia_nim",
     "codestral",
     "deepseek",
     "deepinfra",
@@ -640,6 +642,7 @@ provider_list: List = [
     "anyscale",
     "mistral",
     "groq",
+    "nvidia_nim",
     "codestral",
     "text-completion-codestral",
     "deepseek",
diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index e359d36f4c..a97d6812c8 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -135,7 +135,7 @@ def convert_to_ollama_image(openai_image_url: str):
 
 
 def ollama_pt(
-        model, messages
+    model, messages
 ):  # https://github.com/ollama/ollama/blob/af4cf55884ac54b9e637cd71dadfe9b7a5685877/docs/modelfile.md#template
     if "instruct" in model:
         prompt = custom_prompt(
@@ -185,19 +185,18 @@ def ollama_pt(
                     function_name: str = call["function"]["name"]
                     arguments = json.loads(call["function"]["arguments"])
 
-                    tool_calls.append({
-                        "id": call_id,
-                        "type": "function",
-                        "function": {
-                            "name": function_name,
-                            "arguments": arguments
+                    tool_calls.append(
+                        {
+                            "id": call_id,
+                            "type": "function",
+                            "function": {"name": function_name, "arguments": arguments},
                         }
-                    })
+                    )
 
                 prompt += f"### Assistant:\nTool Calls: {json.dumps(tool_calls, indent=2)}\n\n"
 
             elif "tool_call_id" in message:
-                prompt += f"### User:\n{message["content"]}\n\n"
+                prompt += f"### User:\n{message['content']}\n\n"
 
             elif content:
                 prompt += f"### {role.capitalize()}:\n{content}\n\n"

From d829d6393d46d17cab0ee192b7ca9b4da084184b Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 08:57:11 -0700
Subject: [PATCH 35/68] feat - add nvidia nim to main.py

---
 litellm/main.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/litellm/main.py b/litellm/main.py
index 307659c8a2..8c531643b8 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -348,6 +348,7 @@ async def acompletion(
             or custom_llm_provider == "deepinfra"
             or custom_llm_provider == "perplexity"
             or custom_llm_provider == "groq"
+            or custom_llm_provider == "nvidia_nim"
             or custom_llm_provider == "codestral"
             or custom_llm_provider == "text-completion-codestral"
             or custom_llm_provider == "deepseek"
@@ -1171,6 +1172,7 @@ def completion(
             or custom_llm_provider == "deepinfra"
             or custom_llm_provider == "perplexity"
             or custom_llm_provider == "groq"
+            or custom_llm_provider == "nvidia_nim"
             or custom_llm_provider == "codestral"
             or custom_llm_provider == "deepseek"
             or custom_llm_provider == "anyscale"
@@ -2932,6 +2934,7 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
             or custom_llm_provider == "deepinfra"
             or custom_llm_provider == "perplexity"
             or custom_llm_provider == "groq"
+            or custom_llm_provider == "nvidia_nim"
             or custom_llm_provider == "deepseek"
             or custom_llm_provider == "fireworks_ai"
             or custom_llm_provider == "ollama"
@@ -3507,6 +3510,7 @@ async def atext_completion(
             or custom_llm_provider == "deepinfra"
             or custom_llm_provider == "perplexity"
             or custom_llm_provider == "groq"
+            or custom_llm_provider == "nvidia_nim"
             or custom_llm_provider == "text-completion-codestral"
             or custom_llm_provider == "deepseek"
             or custom_llm_provider == "fireworks_ai"

From f3016250147304cb5a52fe1e4e402a2140d7540b Mon Sep 17 00:00:00 2001
From: Steven Osborn <steven@lolsborn.com>
Date: Tue, 25 Jun 2024 09:03:05 -0700
Subject: [PATCH 36/68] create litellm user to fix issue in k8s where prisma
 fails due to user nobody without home directory

---
 Dockerfile.database | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/Dockerfile.database b/Dockerfile.database
index 22084bab89..1901200d52 100644
--- a/Dockerfile.database
+++ b/Dockerfile.database
@@ -9,6 +9,27 @@ FROM $LITELLM_BUILD_IMAGE as builder
 # Set the working directory to /app
 WORKDIR /app
 
+ARG LITELLM_USER=litellm LITELLM_UID=1729
+ARG LITELLM_GROUP=litellm LITELLM_GID=1729
+
+RUN groupadd \
+	--gid ${LITELLM_GID} \
+	${LITELLM_GROUP} \
+	&& useradd \
+	--create-home \
+	--shell /bin/sh \
+	--gid ${LITELLM_GID} \
+	--uid ${LITELLM_UID} \
+	${LITELLM_USER}
+
+# Allows user to update python install.
+# This is necessary for prisma.
+RUN chown -R ${LITELLM_USER}:${LITELLM_GROUP} /usr/local/lib/python3.11
+
+# Set the HOME var forcefully because of prisma.
+ENV HOME=/home/${LITELLM_USER}
+USER ${LITELLM_USER}
+
 # Install build dependencies
 RUN apt-get clean && apt-get update && \
     apt-get install -y gcc python3-dev && \

From 07829514d11df6543d85c38e92ba477e1691cff8 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 09:13:08 -0700
Subject: [PATCH 37/68] feat - add param mapping for nvidia nim

---
 litellm/__init__.py        |  1 +
 litellm/llms/nvidia_nim.py | 79 ++++++++++++++++++++++++++++++++++++++
 litellm/utils.py           | 23 +++++++++++
 3 files changed, 103 insertions(+)
 create mode 100644 litellm/llms/nvidia_nim.py

diff --git a/litellm/__init__.py b/litellm/__init__.py
index d23247d531..08ee84aaad 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -816,6 +816,7 @@ from .llms.openai import (
     DeepInfraConfig,
     AzureAIStudioConfig,
 )
+from .llms.nvidia_nim import NvidiaNimConfig
 from .llms.text_completion_codestral import MistralTextCompletionConfig
 from .llms.azure import (
     AzureOpenAIConfig,
diff --git a/litellm/llms/nvidia_nim.py b/litellm/llms/nvidia_nim.py
new file mode 100644
index 0000000000..ebcc84c13e
--- /dev/null
+++ b/litellm/llms/nvidia_nim.py
@@ -0,0 +1,79 @@
+"""
+Nvidia NIM endpoint: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer 
+
+This is OpenAI compatible 
+
+This file only contains param mapping logic
+
+API calling is done using the OpenAI SDK with an api_base
+"""
+
+import types
+from typing import Optional, Union
+
+
+class NvidiaNimConfig:
+    """
+    Reference: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer
+
+    The class `NvidiaNimConfig` provides configuration for the Nvidia NIM's Chat Completions API interface. Below are the parameters:
+    """
+
+    temperature: Optional[int] = None
+    top_p: Optional[int] = None
+    frequency_penalty: Optional[int] = None
+    presence_penalty: Optional[int] = None
+    max_tokens: Optional[int] = None
+    stop: Optional[Union[str, list]] = None
+
+    def __init__(
+        self,
+        temperature: Optional[int] = None,
+        top_p: Optional[int] = None,
+        frequency_penalty: Optional[int] = None,
+        presence_penalty: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        stop: Optional[Union[str, list]] = None,
+    ) -> None:
+        locals_ = locals().copy()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+    def get_supported_openai_params(self):
+        return [
+            "stream",
+            "temperature",
+            "top_p",
+            "frequency_penalty",
+            "presence_penalty",
+            "max_tokens",
+            "stop",
+        ]
+
+    def map_openai_params(
+        self, non_default_params: dict, optional_params: dict
+    ) -> dict:
+        supported_openai_params = self.get_supported_openai_params()
+        for param, value in non_default_params.items():
+            if param in supported_openai_params:
+                optional_params[param] = value
+        return optional_params
diff --git a/litellm/utils.py b/litellm/utils.py
index 1bc8bf771f..7709e88210 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2410,6 +2410,7 @@ def get_optional_params(
             and custom_llm_provider != "anyscale"
             and custom_llm_provider != "together_ai"
             and custom_llm_provider != "groq"
+            and custom_llm_provider != "nvidia_nim"
             and custom_llm_provider != "deepseek"
             and custom_llm_provider != "codestral"
             and custom_llm_provider != "mistral"
@@ -3060,6 +3061,14 @@ def get_optional_params(
         optional_params = litellm.DatabricksConfig().map_openai_params(
             non_default_params=non_default_params, optional_params=optional_params
         )
+    elif custom_llm_provider == "nvidia_nim":
+        supported_params = get_supported_openai_params(
+            model=model, custom_llm_provider=custom_llm_provider
+        )
+        _check_valid_arg(supported_params=supported_params)
+        optional_params = litellm.NvidiaNimConfig().map_openai_params(
+            non_default_params=non_default_params, optional_params=optional_params
+        )
     elif custom_llm_provider == "groq":
         supported_params = get_supported_openai_params(
             model=model, custom_llm_provider=custom_llm_provider
@@ -3626,6 +3635,8 @@ def get_supported_openai_params(
         return litellm.OllamaChatConfig().get_supported_openai_params()
     elif custom_llm_provider == "anthropic":
         return litellm.AnthropicConfig().get_supported_openai_params()
+    elif custom_llm_provider == "nvidia_nim":
+        return litellm.NvidiaNimConfig().get_supported_openai_params()
     elif custom_llm_provider == "groq":
         return [
             "temperature",
@@ -3986,6 +3997,10 @@ def get_llm_provider(
                 # groq is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.groq.com/openai/v1
                 api_base = "https://api.groq.com/openai/v1"
                 dynamic_api_key = get_secret("GROQ_API_KEY")
+            elif custom_llm_provider == "nvidia_nim":
+                # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
+                api_base = "https://integrate.api.nvidia.com/v1"
+                dynamic_api_key = get_secret("NVIDIA_NIM_API_KEY")
             elif custom_llm_provider == "codestral":
                 # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1
                 api_base = "https://codestral.mistral.ai/v1"
@@ -4087,6 +4102,9 @@ def get_llm_provider(
                     elif endpoint == "api.groq.com/openai/v1":
                         custom_llm_provider = "groq"
                         dynamic_api_key = get_secret("GROQ_API_KEY")
+                    elif endpoint == "https://integrate.api.nvidia.com/v1":
+                        custom_llm_provider = "nvidia_nim"
+                        dynamic_api_key = get_secret("NVIDIA_NIM_API_KEY")
                     elif endpoint == "https://codestral.mistral.ai/v1":
                         custom_llm_provider = "codestral"
                         dynamic_api_key = get_secret("CODESTRAL_API_KEY")
@@ -4900,6 +4918,11 @@ def validate_environment(model: Optional[str] = None) -> dict:
                 keys_in_environment = True
             else:
                 missing_keys.append("GROQ_API_KEY")
+        elif custom_llm_provider == "nvidia_nim":
+            if "NVIDIA_NIM_API_KEY" in os.environ:
+                keys_in_environment = True
+            else:
+                missing_keys.append("NVIDIA_NIM_API_KEY")
         elif (
             custom_llm_provider == "codestral"
             or custom_llm_provider == "text-completion-codestral"

From e5f281074f1da6f2b6e9495c4078902c64227f59 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 09:16:31 -0700
Subject: [PATCH 38/68] test - nvidia nim

---
 litellm/tests/test_completion.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 830b3acd38..0c6da360bb 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -3470,6 +3470,28 @@ def test_completion_deep_infra_mistral():
 # test_completion_deep_infra_mistral()
 
 
+def test_completion_nvidia_nim():
+    model_name = "nvidia_nim/databricks/dbrx-instruct"
+    try:
+        response = completion(
+            model=model_name,
+            messages=[
+                {
+                    "role": "user",
+                    "content": "What's the weather like in Boston today in Fahrenheit?",
+                }
+            ],
+        )
+        # Add any assertions here to check the response
+        print(response)
+        assert response.choices[0].message.content is not None
+        assert len(response.choices[0].message.content) > 0
+    except litellm.exceptions.Timeout as e:
+        pass
+    except Exception as e:
+        pytest.fail(f"Error occurred: {e}")
+
+
 # Gemini tests
 @pytest.mark.parametrize(
     "model",

From e354840f90d95124a2eb7fbe59097fa44139c5a3 Mon Sep 17 00:00:00 2001
From: Krish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 09:23:19 -0700
Subject: [PATCH 39/68] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 91b709442b..ae354d1e3e 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 
 > [!IMPORTANT]
 > LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
+> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
 
 <a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
   <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>

From e2a50b66f2730db46c68ca2e9e4f8598e1effe0c Mon Sep 17 00:00:00 2001
From: Krish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 09:24:00 -0700
Subject: [PATCH 40/68] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ae354d1e3e..6d26e92c2b 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 # Usage ([**Docs**](https://docs.litellm.ai/docs/))
 
 > [!IMPORTANT]
-> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
+> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)  
 > LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
 
 <a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">

From bac5dfb30b4128b0f75d6cf823d371bddb7ef834 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 09:38:34 -0700
Subject: [PATCH 41/68] docs - add nvidia nim

---
 docs/my-website/docs/providers/nvidia_nim.md | 103 +++++++++++++++++++
 docs/my-website/sidebars.js                  |   5 +-
 2 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 docs/my-website/docs/providers/nvidia_nim.md

diff --git a/docs/my-website/docs/providers/nvidia_nim.md b/docs/my-website/docs/providers/nvidia_nim.md
new file mode 100644
index 0000000000..f90450768b
--- /dev/null
+++ b/docs/my-website/docs/providers/nvidia_nim.md
@@ -0,0 +1,103 @@
+# Nvidia NIM
+https://docs.api.nvidia.com/nim/reference/
+
+:::tip
+
+**We support ALL Nvidia NIM models, just set `model=nvidia_nim/<any-model-on-nvidia_nim>` as a prefix when sending litellm requests**
+
+:::
+
+## API Key
+```python
+# env variable
+os.environ['NVIDIA_NIM_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['NVIDIA_NIM_API_KEY'] = ""
+response = completion(
+    model=model_name,
+    messages=[
+        {
+            "role": "user",
+            "content": "What's the weather like in Boston today in Fahrenheit?",
+        }
+    ],
+    temperature=0.2,        # optional
+    top_p=0.9,              # optional
+    frequency_penalty=0.1,  # optional
+    presence_penalty=0.1,   # optional
+    max_tokens=10,          # optional
+    stop=["\n\n"],          # optional
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['NVIDIA_NIM_API_KEY'] = ""
+response = completion(
+    model=model_name,
+    messages=[
+        {
+            "role": "user",
+            "content": "What's the weather like in Boston today in Fahrenheit?",
+        }
+    ],
+    stream=True,
+    temperature=0.2,        # optional
+    top_p=0.9,              # optional
+    frequency_penalty=0.1,  # optional
+    presence_penalty=0.1,   # optional
+    max_tokens=10,          # optional
+    stop=["\n\n"],          # optional
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+
+## Supported Models - 💥 ALL Nvidia NIM Models Supported!
+We support ALL `nvidia_nim` models, just set `nvidia_nim/` as a prefix when sending completion requests
+
+| Model Name | Function Call |
+|------------|---------------|
+| nvidia/nemotron-4-340b-reward | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-reward", messages)` |
+| 01-ai/yi-large | `completion(model="nvidia_nim/01-ai/yi-large", messages)` |
+| aisingapore/sea-lion-7b-instruct | `completion(model="nvidia_nim/aisingapore/sea-lion-7b-instruct", messages)` |
+| databricks/dbrx-instruct | `completion(model="nvidia_nim/databricks/dbrx-instruct", messages)` |
+| google/gemma-7b | `completion(model="nvidia_nim/google/gemma-7b", messages)` |
+| google/gemma-2b | `completion(model="nvidia_nim/google/gemma-2b", messages)` |
+| google/codegemma-1.1-7b | `completion(model="nvidia_nim/google/codegemma-1.1-7b", messages)` |
+| google/codegemma-7b | `completion(model="nvidia_nim/google/codegemma-7b", messages)` |
+| google/recurrentgemma-2b | `completion(model="nvidia_nim/google/recurrentgemma-2b", messages)` |
+| ibm/granite-34b-code-instruct | `completion(model="nvidia_nim/ibm/granite-34b-code-instruct", messages)` |
+| ibm/granite-8b-code-instruct | `completion(model="nvidia_nim/ibm/granite-8b-code-instruct", messages)` |
+| mediatek/breeze-7b-instruct | `completion(model="nvidia_nim/mediatek/breeze-7b-instruct", messages)` |
+| meta/codellama-70b | `completion(model="nvidia_nim/meta/codellama-70b", messages)` |
+| meta/llama2-70b | `completion(model="nvidia_nim/meta/llama2-70b", messages)` |
+| meta/llama3-8b | `completion(model="nvidia_nim/meta/llama3-8b", messages)` |
+| meta/llama3-70b | `completion(model="nvidia_nim/meta/llama3-70b", messages)` |
+| microsoft/phi-3-medium-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-medium-4k-instruct", messages)` |
+| microsoft/phi-3-mini-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-128k-instruct", messages)` |
+| microsoft/phi-3-mini-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-4k-instruct", messages)` |
+| microsoft/phi-3-small-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-128k-instruct", messages)` |
+| microsoft/phi-3-small-8k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-8k-instruct", messages)` |
+| mistralai/codestral-22b-instruct-v0.1 | `completion(model="nvidia_nim/mistralai/codestral-22b-instruct-v0.1", messages)` |
+| mistralai/mistral-7b-instruct | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct", messages)` |
+| mistralai/mistral-7b-instruct-v0.3 | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct-v0.3", messages)` |
+| mistralai/mixtral-8x7b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x7b-instruct", messages)` |
+| mistralai/mixtral-8x22b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x22b-instruct", messages)` |
+| mistralai/mistral-large | `completion(model="nvidia_nim/mistralai/mistral-large", messages)` |
+| nvidia/nemotron-4-340b-instruct | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-instruct", messages)` |
+| seallms/seallm-7b-v2.5 | `completion(model="nvidia_nim/seallms/seallm-7b-v2.5", messages)` |
+| snowflake/arctic | `completion(model="nvidia_nim/snowflake/arctic", messages)` |
+| upstage/solar-10.7b-instruct | `completion(model="nvidia_nim/upstage/solar-10.7b-instruct", messages)` |
\ No newline at end of file
diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js
index 2673933f4c..9835a260b3 100644
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@@ -146,13 +146,14 @@ const sidebars = {
         "providers/databricks",
         "providers/watsonx",
         "providers/predibase",
-        "providers/clarifai",
+        "providers/nvidia_nim", 
         "providers/triton-inference-server",
         "providers/ollama", 
         "providers/perplexity", 
         "providers/groq", 
         "providers/deepseek", 
-        "providers/fireworks_ai", 
+        "providers/fireworks_ai",
+        "providers/clarifai", 
         "providers/vllm", 
         "providers/xinference", 
         "providers/cloudflare_workers", 

From 100f245fefa93cc3ae950c7fec7fdd52130a5cf3 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 09:46:42 -0700
Subject: [PATCH 42/68] docs nvidia_nim

---
 docs/my-website/docs/providers/nvidia_nim.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/my-website/docs/providers/nvidia_nim.md b/docs/my-website/docs/providers/nvidia_nim.md
index f90450768b..7f895aa337 100644
--- a/docs/my-website/docs/providers/nvidia_nim.md
+++ b/docs/my-website/docs/providers/nvidia_nim.md
@@ -20,7 +20,7 @@ import os
 
 os.environ['NVIDIA_NIM_API_KEY'] = ""
 response = completion(
-    model=model_name,
+    model="nvidia_nim/meta/llama3-70b-instruct",
     messages=[
         {
             "role": "user",
@@ -44,7 +44,7 @@ import os
 
 os.environ['NVIDIA_NIM_API_KEY'] = ""
 response = completion(
-    model=model_name,
+    model="nvidia_nim/meta/llama3-70b-instruct",
     messages=[
         {
             "role": "user",

From b4497acfc2acb5e3d869350d220bce8985490c70 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 09:48:49 -0700
Subject: [PATCH 43/68] ci/cd run again

---
 litellm/tests/test_completion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 0c6da360bb..30ae1d0ab1 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.prompt_templates.factory import anthropic_messages_pt
 
-# litellm.num_retries=3
+# litellm.num_retries = 3
 litellm.cache = None
 litellm.success_callback = []
 user_message = "Write a short poem about the sky"

From a230f5f6c53f0215fb66656d0b5d888bbc3f14ec Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 10:50:47 -0700
Subject: [PATCH 44/68] feat - use n in mock completion

---
 litellm/llms/prompt_templates/factory.py | 17 ++++++++---------
 litellm/main.py                          | 17 +++++++++++++++--
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py
index e359d36f4c..a97d6812c8 100644
--- a/litellm/llms/prompt_templates/factory.py
+++ b/litellm/llms/prompt_templates/factory.py
@@ -135,7 +135,7 @@ def convert_to_ollama_image(openai_image_url: str):
 
 
 def ollama_pt(
-        model, messages
+    model, messages
 ):  # https://github.com/ollama/ollama/blob/af4cf55884ac54b9e637cd71dadfe9b7a5685877/docs/modelfile.md#template
     if "instruct" in model:
         prompt = custom_prompt(
@@ -185,19 +185,18 @@ def ollama_pt(
                     function_name: str = call["function"]["name"]
                     arguments = json.loads(call["function"]["arguments"])
 
-                    tool_calls.append({
-                        "id": call_id,
-                        "type": "function",
-                        "function": {
-                            "name": function_name,
-                            "arguments": arguments
+                    tool_calls.append(
+                        {
+                            "id": call_id,
+                            "type": "function",
+                            "function": {"name": function_name, "arguments": arguments},
                         }
-                    })
+                    )
 
                 prompt += f"### Assistant:\nTool Calls: {json.dumps(tool_calls, indent=2)}\n\n"
 
             elif "tool_call_id" in message:
-                prompt += f"### User:\n{message["content"]}\n\n"
+                prompt += f"### User:\n{message['content']}\n\n"
 
             elif content:
                 prompt += f"### {role.capitalize()}:\n{content}\n\n"
diff --git a/litellm/main.py b/litellm/main.py
index 307659c8a2..07d7be2bad 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -428,6 +428,7 @@ def mock_completion(
     model: str,
     messages: List,
     stream: Optional[bool] = False,
+    n: Optional[int] = None,
     mock_response: Union[str, Exception, dict] = "This is a mock request",
     mock_tool_calls: Optional[List] = None,
     logging=None,
@@ -496,8 +497,19 @@ def mock_completion(
                 model_response, mock_response=mock_response, model=model
             )
             return response
-
-        model_response["choices"][0]["message"]["content"] = mock_response
+        if n is None:
+            model_response["choices"][0]["message"]["content"] = mock_response
+        else:
+            _all_choices = []
+            for i in range(n):
+                _choice = litellm.utils.Choices(
+                    index=i,
+                    message=litellm.utils.Message(
+                        content=mock_response, role="assistant"
+                    ),
+                )
+                _all_choices.append(_choice)
+            model_response["choices"] = _all_choices
         model_response["created"] = int(time.time())
         model_response["model"] = model
 
@@ -944,6 +956,7 @@ def completion(
                 model,
                 messages,
                 stream=stream,
+                n=n,
                 mock_response=mock_response,
                 mock_tool_calls=mock_tool_calls,
                 logging=logging,

From 343e3f3e909c02f38533604ff998b3dbef79153e Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 10:54:18 -0700
Subject: [PATCH 45/68] test - test_mock_request_n_greater_than_1

---
 litellm/tests/test_mock_request.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/litellm/tests/test_mock_request.py b/litellm/tests/test_mock_request.py
index 7d670feb5b..6b58c94b2b 100644
--- a/litellm/tests/test_mock_request.py
+++ b/litellm/tests/test_mock_request.py
@@ -58,3 +58,18 @@ async def test_async_mock_streaming_request():
     assert (
         complete_response == "LiteLLM is awesome"
     ), f"Unexpected response got {complete_response}"
+
+
+def test_mock_request_n_greater_than_1():
+    try:
+        model = "gpt-3.5-turbo"
+        messages = [{"role": "user", "content": "Hey, I'm a mock request"}]
+        response = litellm.mock_completion(model=model, messages=messages, n=5)
+        print("response: ", response)
+
+        assert len(response.choices) == 5
+        for choice in response.choices:
+            assert choice.message.content == "This is a mock request"
+
+    except:
+        traceback.print_exc()

From 0396d484fbce1b635519e8144352f707167205bd Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 10:57:32 -0700
Subject: [PATCH 46/68] feat(router.py): support mock testing content policy +
 context window fallbacks

---
 litellm/proxy/_new_secret_config.yaml | 70 +++++++++++++++++++--------
 litellm/router.py                     | 26 ++++++++++
 2 files changed, 76 insertions(+), 20 deletions(-)

diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml
index 16436c0ef9..75545bb604 100644
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@@ -1,24 +1,54 @@
-model_list: 
-  - model_name: my-fake-model
-    litellm_params:
-      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
-      api_key: my-fake-key
-      aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
-      mock_response: "Hello world 1"
-    model_info: 
-      max_input_tokens: 0 # trigger context window fallback
-  - model_name: my-fake-model
-    litellm_params:
-      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
-      api_key: my-fake-key
-      aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
-      mock_response: "Hello world 2"
-    model_info: 
-      max_input_tokens: 0
+# model_list: 
+#   - model_name: my-fake-model
+#     litellm_params:
+#       model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+#       api_key: my-fake-key
+#       aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
+#       mock_response: "Hello world 1"
+#     model_info: 
+#       max_input_tokens: 0 # trigger context window fallback
+#   - model_name: my-fake-model
+#     litellm_params:
+#       model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+#       api_key: my-fake-key
+#       aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
+#       mock_response: "Hello world 2"
+#     model_info: 
+#       max_input_tokens: 0
 
-router_settings:
-  enable_pre_call_checks: True
+# router_settings:
+#   enable_pre_call_checks: True
 
 
+# litellm_settings:
+#   failure_callback: ["langfuse"]
+
+model_list:
+  - model_name: summarize
+    litellm_params:
+        model: openai/gpt-4o
+        rpm: 10000      
+        tpm: 12000000
+        api_key: os.environ/OPENAI_API_KEY
+        mock_response: Hello world 1
+
+  - model_name: summarize-l
+    litellm_params:
+        model: claude-3-5-sonnet-20240620
+        rpm: 4000
+        tpm: 400000
+        api_key: os.environ/ANTHROPIC_API_KEY
+        mock_response: Hello world 2
+
 litellm_settings:
-  failure_callback: ["langfuse"]
\ No newline at end of file
+  num_retries: 3
+  request_timeout: 120
+  allowed_fails: 3
+  # fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
+  context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
+
+
+
+router_settings:
+  routing_strategy: simple-shuffle
+  enable_pre_call_checks: true.
diff --git a/litellm/router.py b/litellm/router.py
index 30bdbcba2d..8256a67528 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -2117,6 +2117,12 @@ class Router:
         If it fails after num_retries, fall back to another model group
         """
         mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None)
+        mock_testing_context_fallbacks = kwargs.pop(
+            "mock_testing_context_fallbacks", None
+        )
+        mock_testing_content_policy_fallbacks = kwargs.pop(
+            "mock_testing_content_policy_fallbacks", None
+        )
         model_group = kwargs.get("model")
         fallbacks = kwargs.get("fallbacks", self.fallbacks)
         context_window_fallbacks = kwargs.get(
@@ -2130,6 +2136,26 @@ class Router:
                 raise Exception(
                     f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}"
                 )
+            elif (
+                mock_testing_context_fallbacks is not None
+                and mock_testing_context_fallbacks is True
+            ):
+                raise litellm.ContextWindowExceededError(
+                    model=model_group,
+                    llm_provider="",
+                    message=f"This is a mock exception for model={model_group}, to trigger a fallback. \
+                        Context_Window_Fallbacks={context_window_fallbacks}",
+                )
+            elif (
+                mock_testing_content_policy_fallbacks is not None
+                and mock_testing_content_policy_fallbacks is True
+            ):
+                raise litellm.ContentPolicyViolationError(
+                    model=model_group,
+                    llm_provider="",
+                    message=f"This is a mock exception for model={model_group}, to trigger a fallback. \
+                        Context_Policy_Fallbacks={content_policy_fallbacks}",
+                )
 
             response = await self.async_function_with_retries(*args, **kwargs)
             verbose_router_logger.debug(f"Async Response: {response}")

From 71b7c2886b1559d23ad0901c65d9adfe113956aa Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 11:07:07 -0700
Subject: [PATCH 47/68] docs(reliability.md): add doc on mock testing fallbacks

---
 docs/my-website/docs/proxy/reliability.md     | 61 +++++++++++++++++++
 ...odel_prices_and_context_window_backup.json | 10 +--
 2 files changed, 66 insertions(+), 5 deletions(-)

diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md
index a2d24da69b..c07fc3c26a 100644
--- a/docs/my-website/docs/proxy/reliability.md
+++ b/docs/my-website/docs/proxy/reliability.md
@@ -431,6 +431,67 @@ litellm_settings:
   content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}]
 ```
 
+
+
+### Test Fallbacks! 
+
+Check if your fallbacks are working as expected. 
+
+#### **Regular Fallbacks**
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-D '{
+  "model": "my-bad-model",
+  "messages": [
+    {
+      "role": "user",
+      "content": "ping"
+    }
+  ],
+  "mock_testing_fallbacks": true # 👈 KEY CHANGE
+}
+'
+```
+
+#### **Content Policy Fallbacks**
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-D '{
+  "model": "my-bad-model",
+  "messages": [
+    {
+      "role": "user",
+      "content": "ping"
+    }
+  ],
+  "mock_testing_content_policy_fallbacks": true # 👈 KEY CHANGE
+}
+'
+```
+
+#### **Context Window Fallbacks**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-D '{
+  "model": "my-bad-model",
+  "messages": [
+    {
+      "role": "user",
+      "content": "ping"
+    }
+  ],
+  "mock_testing_context_window_fallbacks": true # 👈 KEY CHANGE
+}
+'
+```
+
 ### EU-Region Filtering (Pre-Call Checks)
 
 **Before call is made** check if a call is within model context window with  **`enable_pre_call_checks: true`**.
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index ef07d87ccb..415d220f21 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -887,7 +887,7 @@
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "input_cost_per_token": 0.00000005,
-        "output_cost_per_token": 0.00000010,
+        "output_cost_per_token": 0.00000008,
         "litellm_provider": "groq",
         "mode": "chat",
         "supports_function_calling": true
@@ -906,8 +906,8 @@
         "max_tokens": 32768,
         "max_input_tokens": 32768,
         "max_output_tokens": 32768,
-        "input_cost_per_token": 0.00000027,
-        "output_cost_per_token": 0.00000027,
+        "input_cost_per_token": 0.00000024,
+        "output_cost_per_token": 0.00000024,
         "litellm_provider": "groq",
         "mode": "chat",
         "supports_function_calling": true
@@ -916,8 +916,8 @@
         "max_tokens": 8192,
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
-        "input_cost_per_token": 0.00000010,
-        "output_cost_per_token": 0.00000010,
+        "input_cost_per_token": 0.00000007,
+        "output_cost_per_token": 0.00000007,
         "litellm_provider": "groq",
         "mode": "chat",
         "supports_function_calling": true

From ccf1bbc5d75c310fc45816d4bc55a9aa5be925d1 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 11:14:40 -0700
Subject: [PATCH 48/68] fix using mock completion

---
 litellm/main.py                    |  7 ++++--
 litellm/tests/test_mock_request.py | 19 +++++++++++++++
 litellm/utils.py                   | 39 +++++++++++++++++++++++++-----
 3 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/litellm/main.py b/litellm/main.py
index 07d7be2bad..ecb6edd0d6 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -487,14 +487,17 @@ def mock_completion(
             if kwargs.get("acompletion", False) == True:
                 return CustomStreamWrapper(
                     completion_stream=async_mock_completion_streaming_obj(
-                        model_response, mock_response=mock_response, model=model
+                        model_response, mock_response=mock_response, model=model, n=n
                     ),
                     model=model,
                     custom_llm_provider="openai",
                     logging_obj=logging,
                 )
             response = mock_completion_streaming_obj(
-                model_response, mock_response=mock_response, model=model
+                model_response,
+                mock_response=mock_response,
+                model=model,
+                n=n,
             )
             return response
         if n is None:
diff --git a/litellm/tests/test_mock_request.py b/litellm/tests/test_mock_request.py
index 6b58c94b2b..48b054371f 100644
--- a/litellm/tests/test_mock_request.py
+++ b/litellm/tests/test_mock_request.py
@@ -73,3 +73,22 @@ def test_mock_request_n_greater_than_1():
 
     except:
         traceback.print_exc()
+
+
+@pytest.mark.asyncio()
+async def test_async_mock_streaming_request_n_greater_than_1():
+    generator = await litellm.acompletion(
+        messages=[{"role": "user", "content": "Why is LiteLLM amazing?"}],
+        mock_response="LiteLLM is awesome",
+        stream=True,
+        model="gpt-3.5-turbo",
+        n=5,
+    )
+    complete_response = ""
+    async for chunk in generator:
+        print(chunk)
+        # complete_response += chunk["choices"][0]["delta"]["content"] or ""
+
+    # assert (
+    #     complete_response == "LiteLLM is awesome"
+    # ), f"Unexpected response got {complete_response}"
diff --git a/litellm/utils.py b/litellm/utils.py
index 1bc8bf771f..cae86c6f84 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -9708,18 +9708,45 @@ class TextCompletionStreamWrapper:
             raise StopAsyncIteration
 
 
-def mock_completion_streaming_obj(model_response, mock_response, model):
+def mock_completion_streaming_obj(
+    model_response, mock_response, model, n: Optional[int] = None
+):
     for i in range(0, len(mock_response), 3):
-        completion_obj = {"role": "assistant", "content": mock_response[i : i + 3]}
-        model_response.choices[0].delta = completion_obj
+        completion_obj = Delta(role="assistant", content=mock_response[i : i + 3])
+        if n is None:
+            model_response.choices[0].delta = completion_obj
+        else:
+            _all_choices = []
+            for j in range(n):
+                _streaming_choice = litellm.utils.StreamingChoices(
+                    index=j,
+                    delta=litellm.utils.Delta(
+                        role="assistant", content=mock_response[i : i + 3]
+                    ),
+                )
+                _all_choices.append(_streaming_choice)
+            model_response.choices = _all_choices
         yield model_response
 
 
-async def async_mock_completion_streaming_obj(model_response, mock_response, model):
+async def async_mock_completion_streaming_obj(
+    model_response, mock_response, model, n: Optional[int] = None
+):
     for i in range(0, len(mock_response), 3):
         completion_obj = Delta(role="assistant", content=mock_response[i : i + 3])
-        model_response.choices[0].delta = completion_obj
-        model_response.choices[0].finish_reason = "stop"
+        if n is None:
+            model_response.choices[0].delta = completion_obj
+        else:
+            _all_choices = []
+            for j in range(n):
+                _streaming_choice = litellm.utils.StreamingChoices(
+                    index=j,
+                    delta=litellm.utils.Delta(
+                        role="assistant", content=mock_response[i : i + 3]
+                    ),
+                )
+                _all_choices.append(_streaming_choice)
+            model_response.choices = _all_choices
         yield model_response
 
 

From cccc55213b1d904f4d596f87f2670c31fef55109 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 11:26:56 -0700
Subject: [PATCH 49/68] fix(router.py): improve error message returned for
 fallbacks

---
 litellm/proxy/_new_secret_config.yaml  |   2 +-
 litellm/router.py                      | 142 ++++++++++++++-----------
 litellm/tests/test_router_fallbacks.py |   4 +-
 3 files changed, 85 insertions(+), 63 deletions(-)

diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml
index 75545bb604..938e74b5e7 100644
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@@ -45,7 +45,7 @@ litellm_settings:
   request_timeout: 120
   allowed_fails: 3
   # fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
-  context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
+  # context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
 
 
 
diff --git a/litellm/router.py b/litellm/router.py
index 8256a67528..840df5b54e 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -2175,73 +2175,93 @@ class Router:
                     )
                 ):  # don't retry a malformed request
                     raise e
-                if (
-                    isinstance(e, litellm.ContextWindowExceededError)
-                    and context_window_fallbacks is not None
-                ):
-                    fallback_model_group = None
-                    for (
-                        item
-                    ) in context_window_fallbacks:  # [{"gpt-3.5-turbo": ["gpt-4"]}]
-                        if list(item.keys())[0] == model_group:
-                            fallback_model_group = item[model_group]
-                            break
+                if isinstance(e, litellm.ContextWindowExceededError):
+                    if context_window_fallbacks is not None:
+                        fallback_model_group = None
+                        for (
+                            item
+                        ) in context_window_fallbacks:  # [{"gpt-3.5-turbo": ["gpt-4"]}]
+                            if list(item.keys())[0] == model_group:
+                                fallback_model_group = item[model_group]
+                                break
 
-                    if fallback_model_group is None:
-                        raise original_exception
+                        if fallback_model_group is None:
+                            raise original_exception
 
-                    for mg in fallback_model_group:
-                        """
-                        Iterate through the model groups and try calling that deployment
-                        """
-                        try:
-                            kwargs["model"] = mg
-                            kwargs.setdefault("metadata", {}).update(
-                                {"model_group": mg}
-                            )  # update model_group used, if fallbacks are done
-                            response = await self.async_function_with_retries(
-                                *args, **kwargs
+                        for mg in fallback_model_group:
+                            """
+                            Iterate through the model groups and try calling that deployment
+                            """
+                            try:
+                                kwargs["model"] = mg
+                                kwargs.setdefault("metadata", {}).update(
+                                    {"model_group": mg}
+                                )  # update model_group used, if fallbacks are done
+                                response = await self.async_function_with_retries(
+                                    *args, **kwargs
+                                )
+                                verbose_router_logger.info(
+                                    "Successful fallback b/w models."
+                                )
+                                return response
+                            except Exception as e:
+                                pass
+                    else:
+                        error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
+                            model_group, context_window_fallbacks, fallbacks
+                        )
+                        verbose_router_logger.info(
+                            msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \
+                            to fallbacks, if available.{}".format(
+                                error_message
                             )
-                            verbose_router_logger.info(
-                                "Successful fallback b/w models."
-                            )
-                            return response
-                        except Exception as e:
-                            pass
-                elif (
-                    isinstance(e, litellm.ContentPolicyViolationError)
-                    and content_policy_fallbacks is not None
-                ):
-                    fallback_model_group = None
-                    for (
-                        item
-                    ) in content_policy_fallbacks:  # [{"gpt-3.5-turbo": ["gpt-4"]}]
-                        if list(item.keys())[0] == model_group:
-                            fallback_model_group = item[model_group]
-                            break
+                        )
 
-                    if fallback_model_group is None:
-                        raise original_exception
+                        e.message += "\n{}".format(error_message)
+                elif isinstance(e, litellm.ContentPolicyViolationError):
+                    if content_policy_fallbacks is not None:
+                        fallback_model_group = None
+                        for (
+                            item
+                        ) in content_policy_fallbacks:  # [{"gpt-3.5-turbo": ["gpt-4"]}]
+                            if list(item.keys())[0] == model_group:
+                                fallback_model_group = item[model_group]
+                                break
 
-                    for mg in fallback_model_group:
-                        """
-                        Iterate through the model groups and try calling that deployment
-                        """
-                        try:
-                            kwargs["model"] = mg
-                            kwargs.setdefault("metadata", {}).update(
-                                {"model_group": mg}
-                            )  # update model_group used, if fallbacks are done
-                            response = await self.async_function_with_retries(
-                                *args, **kwargs
+                        if fallback_model_group is None:
+                            raise original_exception
+
+                        for mg in fallback_model_group:
+                            """
+                            Iterate through the model groups and try calling that deployment
+                            """
+                            try:
+                                kwargs["model"] = mg
+                                kwargs.setdefault("metadata", {}).update(
+                                    {"model_group": mg}
+                                )  # update model_group used, if fallbacks are done
+                                response = await self.async_function_with_retries(
+                                    *args, **kwargs
+                                )
+                                verbose_router_logger.info(
+                                    "Successful fallback b/w models."
+                                )
+                                return response
+                            except Exception as e:
+                                pass
+                    else:
+                        error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
+                            model_group, content_policy_fallbacks, fallbacks
+                        )
+                        verbose_router_logger.info(
+                            msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
+                            to fallbacks, if available.{}".format(
+                                error_message
                             )
-                            verbose_router_logger.info(
-                                "Successful fallback b/w models."
-                            )
-                            return response
-                        except Exception as e:
-                            pass
-                elif fallbacks is not None:
+                        )
+
+                        e.message += "\n{}".format(error_message)
+                if fallbacks is not None:
                     verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
                     generic_fallback_idx: Optional[int] = None
                     ## check for specific model group-specific fallbacks
diff --git a/litellm/tests/test_router_fallbacks.py b/litellm/tests/test_router_fallbacks.py
index 99d2a600c8..2c552a64bf 100644
--- a/litellm/tests/test_router_fallbacks.py
+++ b/litellm/tests/test_router_fallbacks.py
@@ -1129,7 +1129,9 @@ async def test_router_content_policy_fallbacks(
         mock_response = Exception("content filtering policy")
     else:
         mock_response = litellm.ModelResponse(
-            choices=[litellm.Choices(finish_reason="content_filter")]
+            choices=[litellm.Choices(finish_reason="content_filter")],
+            model="gpt-3.5-turbo",
+            usage=litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10),
         )
     router = Router(
         model_list=[

From 6e02ac00569d927648742d2b5739232e2e33dea6 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 11:47:39 -0700
Subject: [PATCH 50/68] fix(utils.py): add coverage for anthropic content
 policy error - vertex ai

---
 litellm/utils.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index 8549989010..9f6ebaff0c 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -6470,7 +6470,11 @@ def exception_type(
                         ),
                         litellm_debug_info=extra_information,
                     )
-                elif "The response was blocked." in error_str:
+                elif (
+                    "The response was blocked." in error_str
+                    or "Output blocked by content filtering policy"
+                    in error_str  # anthropic on vertex ai
+                ):
                     exception_mapping_worked = True
                     raise ContentPolicyViolationError(
                         message=f"VertexAIException ContentPolicyViolationError - {error_str}",

From 6889a4c0dd2275fc112dbe0badda42bd68f1adf0 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 13:47:38 -0700
Subject: [PATCH 51/68] fix(utils.py): predibase exception mapping - map 424 as
 a badrequest error

---
 litellm/llms/predibase.py               | 39 +++++++++++++------------
 litellm/proxy/_super_secret_config.yaml |  5 +++-
 litellm/utils.py                        | 12 +++-----
 3 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py
index 8ad294457e..7a137da703 100644
--- a/litellm/llms/predibase.py
+++ b/litellm/llms/predibase.py
@@ -1,27 +1,26 @@
 # What is this?
 ## Controller file for Predibase Integration - https://predibase.com/
 
-from functools import partial
-import os, types
-import traceback
+import copy
 import json
-from enum import Enum
-import requests, copy  # type: ignore
+import os
 import time
-from typing import Callable, Optional, List, Literal, Union
-from litellm.utils import (
-    ModelResponse,
-    Usage,
-    CustomStreamWrapper,
-    Message,
-    Choices,
-)
-from litellm.litellm_core_utils.core_helpers import map_finish_reason
-import litellm
-from .prompt_templates.factory import prompt_factory, custom_prompt
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-from .base import BaseLLM
+import traceback
+import types
+from enum import Enum
+from functools import partial
+from typing import Callable, List, Literal, Optional, Union
+
 import httpx  # type: ignore
+import requests  # type: ignore
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage
+
+from .base import BaseLLM
+from .prompt_templates.factory import custom_prompt, prompt_factory
 
 
 class PredibaseError(Exception):
@@ -496,7 +495,9 @@ class PredibaseChatCompletion(BaseLLM):
         except httpx.HTTPStatusError as e:
             raise PredibaseError(
                 status_code=e.response.status_code,
-                message="HTTPStatusError - {}".format(e.response.text),
+                message="HTTPStatusError - received status_code={}, error_message={}".format(
+                    e.response.status_code, e.response.text
+                ),
             )
         except Exception as e:
             raise PredibaseError(
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index c5f1b47687..94df97c54b 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -14,9 +14,10 @@ model_list:
 - model_name: fake-openai-endpoint
   litellm_params:
     model: predibase/llama-3-8b-instruct
-    api_base: "http://0.0.0.0:8000"
+    # api_base: "http://0.0.0.0:8081"
     api_key: os.environ/PREDIBASE_API_KEY
     tenant_id: os.environ/PREDIBASE_TENANT_ID
+    adapter_id: qwoiqjdoqin
     max_retries: 0
     temperature: 0.1
     max_new_tokens: 256
@@ -73,6 +74,8 @@ model_list:
 
 litellm_settings:
   callbacks: ["dynamic_rate_limiter"]
+  # success_callback: ["langfuse"]
+  # failure_callback: ["langfuse"]
   # default_team_settings: 
   #   - team_id: proj1
   #     success_callback: ["langfuse"]
diff --git a/litellm/utils.py b/litellm/utils.py
index 9f6ebaff0c..00833003ba 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -6157,13 +6157,6 @@ def exception_type(
                         response=original_exception.response,
                         litellm_debug_info=extra_information,
                     )
-                if "Request failed during generation" in error_str:
-                    # this is an internal server error from predibase
-                    raise litellm.InternalServerError(
-                        message=f"PredibaseException - {error_str}",
-                        llm_provider="predibase",
-                        model=model,
-                    )
                 elif hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -6201,7 +6194,10 @@ def exception_type(
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
                         )
-                    elif original_exception.status_code == 422:
+                    elif (
+                        original_exception.status_code == 422
+                        or original_exception.status_code == 424
+                    ):
                         exception_mapping_worked = True
                         raise BadRequestError(
                             message=f"PredibaseException - {original_exception.message}",

From 4c99010eeea1d21e1370326fd1dcd16133b4b99a Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 13:55:54 -0700
Subject: [PATCH 52/68] fix - verify license without api request

---
 litellm/proxy/auth/litellm_license.py | 65 +++++++++++++++++++++++++++
 litellm/proxy/auth/public_key.pem     |  9 ++++
 requirements.txt                      |  1 +
 3 files changed, 75 insertions(+)
 create mode 100644 litellm/proxy/auth/public_key.pem

diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py
index ffd9f5273e..ec51f904c6 100644
--- a/litellm/proxy/auth/litellm_license.py
+++ b/litellm/proxy/auth/litellm_license.py
@@ -1,6 +1,14 @@
 # What is this?
 ## If litellm license in env, checks if it's valid
+import base64
+import json
 import os
+from datetime import datetime
+
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import padding, rsa
+
+from litellm._logging import verbose_proxy_logger
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
@@ -15,6 +23,20 @@ class LicenseCheck:
     def __init__(self) -> None:
         self.license_str = os.getenv("LITELLM_LICENSE", None)
         self.http_handler = HTTPHandler()
+        self.public_key = None
+        self.read_public_key()
+
+    def read_public_key(self):
+        # current dir
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+
+        # check if public_key.pem exists
+        _path_to_public_key = os.path.join(current_dir, "public_key.pem")
+        if os.path.exists(_path_to_public_key):
+            with open(_path_to_public_key, "rb") as key_file:
+                self.public_key = serialization.load_pem_public_key(key_file.read())
+        else:
+            self.public_key = None
 
     def _verify(self, license_str: str) -> bool:
         url = "{}/verify_license/{}".format(self.base_url, license_str)
@@ -35,11 +57,54 @@ class LicenseCheck:
             return False
 
     def is_premium(self) -> bool:
+        """
+        1. verify_license_without_api_request: checks if license was generate using private / public key pair
+        2. _verify: checks if license is valid calling litellm API. This is the old way we were generating/validating license
+        """
         try:
             if self.license_str is None:
                 return False
+            elif self.verify_license_without_api_request(
+                public_key=self.public_key, license_key=self.license_str
+            ):
+                return True
             elif self._verify(license_str=self.license_str):
                 return True
             return False
         except Exception as e:
             return False
+
+    def verify_license_without_api_request(self, public_key, license_key):
+        try:
+            # Decode the license key
+            decoded = base64.b64decode(license_key)
+            message, signature = decoded.split(b".", 1)
+
+            # Verify the signature
+            public_key.verify(
+                signature,
+                message,
+                padding.PSS(
+                    mgf=padding.MGF1(hashes.SHA256()),
+                    salt_length=padding.PSS.MAX_LENGTH,
+                ),
+                hashes.SHA256(),
+            )
+
+            # Decode and parse the data
+            license_data = json.loads(message.decode())
+
+            # debug information provided in license data
+            verbose_proxy_logger.debug("License data: %s", license_data)
+
+            # Check expiration date
+            expiration_date = datetime.strptime(
+                license_data["expiration_date"], "%Y-%m-%d"
+            )
+            if expiration_date < datetime.now():
+                return False, "License has expired"
+
+            return True
+
+        except Exception as e:
+            return False
diff --git a/litellm/proxy/auth/public_key.pem b/litellm/proxy/auth/public_key.pem
new file mode 100644
index 0000000000..12a69dde27
--- /dev/null
+++ b/litellm/proxy/auth/public_key.pem
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmfBuNiNzDkNWyce23koQ
+w0vq3bSVHkq7fd9Sw/U1q7FwRwL221daLTyGWssd8xAoQSFXAJKoBwzJQ9wd+o44
+lfL54E3a61nfjZuF+D9ntpXZFfEAxLVtIahDeQjUz4b/EpgciWIJyUfjCJrQo6LY
+eyAZPTGSO8V3zHyaU+CFywq5XCuCnfZqCZeCw051St59A2v8W32mXSCJ+A+x0hYP
+yXJyRRFcefSFG5IBuRHr4Y24Vx7NUIAoco5cnxJho9g2z3J/Hb0GKW+oBNvRVumk
+nuA2Ljmjh4yI0OoTIW8ZWxemvCCJHSjdfKlMyb+QI4fmeiIUZzP5Au+F561Styqq
+YQIDAQAB
+-----END PUBLIC KEY-----
diff --git a/requirements.txt b/requirements.txt
index fbf2bfc1d1..8c5e4ab3b4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -31,6 +31,7 @@ azure-identity==1.16.1 # for azure content safety
 opentelemetry-api==1.25.0
 opentelemetry-sdk==1.25.0
 opentelemetry-exporter-otlp==1.25.0
+cryptography==42.0.7
 
 ### LITELLM PACKAGE DEPENDENCIES
 python-dotenv==1.0.0 # for env 

From e813e984f74ea09ea92646c44c5a5ab7a30bbff0 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 16:03:47 -0700
Subject: [PATCH 53/68] fix(predibase.py): support json schema on predibase

---
 litellm/llms/predibase.py               | 59 ++++++++++++++++++++++---
 litellm/proxy/_super_secret_config.yaml | 16 +++----
 litellm/utils.py                        | 10 ++++-
 3 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py
index 7a137da703..534f8e26f2 100644
--- a/litellm/llms/predibase.py
+++ b/litellm/llms/predibase.py
@@ -15,6 +15,8 @@ import httpx  # type: ignore
 import requests  # type: ignore
 
 import litellm
+import litellm.litellm_core_utils
+import litellm.litellm_core_utils.litellm_logging
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage
@@ -145,7 +147,49 @@ class PredibaseConfig:
         }
 
     def get_supported_openai_params(self):
-        return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"]
+        return [
+            "stream",
+            "temperature",
+            "max_tokens",
+            "top_p",
+            "stop",
+            "n",
+            "response_format",
+        ]
+
+    def map_openai_params(self, non_default_params: dict, optional_params: dict):
+        for param, value in non_default_params.items():
+            # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None
+            if param == "temperature":
+                if value == 0.0 or value == 0:
+                    # hugging face exception raised when temp==0
+                    # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive
+                    value = 0.01
+                optional_params["temperature"] = value
+            if param == "top_p":
+                optional_params["top_p"] = value
+            if param == "n":
+                optional_params["best_of"] = value
+                optional_params["do_sample"] = (
+                    True  # Need to sample if you want best of for hf inference endpoints
+                )
+            if param == "stream":
+                optional_params["stream"] = value
+            if param == "stop":
+                optional_params["stop"] = value
+            if param == "max_tokens":
+                # HF TGI raises the following exception when max_new_tokens==0
+                # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive
+                if value == 0:
+                    value = 1
+                optional_params["max_new_tokens"] = value
+            if param == "echo":
+                # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details
+                #  Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False
+                optional_params["decoder_input_details"] = True
+            if param == "response_format":
+                optional_params["response_format"] = value
+        return optional_params
 
 
 class PredibaseChatCompletion(BaseLLM):
@@ -224,15 +268,16 @@ class PredibaseChatCompletion(BaseLLM):
                 status_code=response.status_code,
             )
         else:
-            if (
-                not isinstance(completion_response, dict)
-                or "generated_text" not in completion_response
-            ):
+            if not isinstance(completion_response, dict):
                 raise PredibaseError(
                     status_code=422,
-                    message=f"response is not in expected format - {completion_response}",
+                    message=f"'completion_response' is not a dictionary - {completion_response}",
+                )
+            elif "generated_text" not in completion_response:
+                raise PredibaseError(
+                    status_code=422,
+                    message=f"'generated_text' is not a key response dictionary - {completion_response}",
                 )
-
             if len(completion_response["generated_text"]) > 0:
                 model_response["choices"][0]["message"]["content"] = self.output_parser(
                     completion_response["generated_text"]
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index 94df97c54b..2060f61ca4 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -14,14 +14,10 @@ model_list:
 - model_name: fake-openai-endpoint
   litellm_params:
     model: predibase/llama-3-8b-instruct
-    # api_base: "http://0.0.0.0:8081"
+    api_base: "http://0.0.0.0:8081"
     api_key: os.environ/PREDIBASE_API_KEY
     tenant_id: os.environ/PREDIBASE_TENANT_ID
-    adapter_id: qwoiqjdoqin
-    max_retries: 0
-    temperature: 0.1
     max_new_tokens: 256
-    return_full_text: false
 
 # - litellm_params:
 #     api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
@@ -97,8 +93,8 @@ assistant_settings:
 router_settings:
   enable_pre_call_checks: true
 
-general_settings:
-  alerting: ["slack"]
-  enable_jwt_auth: True
-  litellm_jwtauth:
-    team_id_jwt_field: "client_id" 
\ No newline at end of file
+# general_settings:
+#   # alerting: ["slack"]
+#   enable_jwt_auth: True
+#   litellm_jwtauth:
+#     team_id_jwt_field: "client_id" 
\ No newline at end of file
diff --git a/litellm/utils.py b/litellm/utils.py
index 00833003ba..4465c5b0a4 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2609,7 +2609,15 @@ def get_optional_params(
             optional_params["top_p"] = top_p
         if stop is not None:
             optional_params["stop_sequences"] = stop
-    elif custom_llm_provider == "huggingface" or custom_llm_provider == "predibase":
+    elif custom_llm_provider == "predibase":
+        supported_params = get_supported_openai_params(
+            model=model, custom_llm_provider=custom_llm_provider
+        )
+        _check_valid_arg(supported_params=supported_params)
+        optional_params = litellm.PredibaseConfig().map_openai_params(
+            non_default_params=non_default_params, optional_params=optional_params
+        )
+    elif custom_llm_provider == "huggingface":
         ## check if unsupported param passed in
         supported_params = get_supported_openai_params(
             model=model, custom_llm_provider=custom_llm_provider

From 4aa686bb0569937a1504e7952b556ee82a603ef9 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 16:23:56 -0700
Subject: [PATCH 54/68] feat - add secret detection

---
 .../enterprise_hooks/secret_detection.py      | 164 ++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 enterprise/enterprise_hooks/secret_detection.py

diff --git a/enterprise/enterprise_hooks/secret_detection.py b/enterprise/enterprise_hooks/secret_detection.py
new file mode 100644
index 0000000000..75a578b2cc
--- /dev/null
+++ b/enterprise/enterprise_hooks/secret_detection.py
@@ -0,0 +1,164 @@
+# +-------------------------------------------------------------+
+#
+#           Use SecretDetection /moderations for your LLM calls
+#
+# +-------------------------------------------------------------+
+#  Thank you users! We ❤️ you! - Krrish & Ishaan
+
+import sys, os
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+from typing import Optional, Literal, Union
+import litellm, traceback, sys, uuid
+from litellm.caching import DualCache
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.integrations.custom_logger import CustomLogger
+from fastapi import HTTPException
+from litellm._logging import verbose_proxy_logger
+from litellm.utils import (
+    ModelResponse,
+    EmbeddingResponse,
+    ImageResponse,
+    StreamingChoices,
+)
+from datetime import datetime
+import aiohttp, asyncio
+from litellm._logging import verbose_proxy_logger
+import tempfile
+from litellm._logging import verbose_proxy_logger
+
+
+litellm.set_verbose = True
+
+
+class _ENTERPRISE_SecretDetection(CustomLogger):
+    def __init__(self):
+        pass
+
+    def scan_message_for_secrets(self, message_content: str):
+        from detect_secrets import SecretsCollection
+        from detect_secrets.settings import default_settings
+
+        temp_file = tempfile.NamedTemporaryFile(delete=False)
+        temp_file.write(message_content.encode("utf-8"))
+        temp_file.close()
+
+        secrets = SecretsCollection()
+        with default_settings():
+            secrets.scan_file(temp_file.name)
+
+        os.remove(temp_file.name)
+
+        detected_secrets = []
+        for file in secrets.files:
+            for found_secret in secrets[file]:
+                if found_secret.secret_value is None:
+                    continue
+                detected_secrets.append(
+                    {"type": found_secret.type, "value": found_secret.secret_value}
+                )
+
+        return detected_secrets
+
+    #### CALL HOOKS - proxy only ####
+    def async_pre_call_hook(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        cache: DualCache,
+        data: dict,
+        call_type: str,  # "completion", "embeddings", "image_generation", "moderation"
+    ):
+        from detect_secrets import SecretsCollection
+        from detect_secrets.settings import default_settings
+
+        if "messages" in data and isinstance(data["messages"], list):
+            for message in data["messages"]:
+                if "content" in message and isinstance(message["content"], str):
+                    detected_secrets = self.scan_message_for_secrets(message["content"])
+
+                    for secret in detected_secrets:
+                        message["content"] = message["content"].replace(
+                            secret["value"], "[REDACTED]"
+                        )
+
+                    if len(detected_secrets) > 0:
+                        secret_types = [secret["type"] for secret in detected_secrets]
+                        verbose_proxy_logger.warning(
+                            f"Detected and redacted secrets in message: {secret_types}"
+                        )
+
+        if "prompt" in data:
+            if isinstance(data["prompt"], str):
+                detected_secrets = self.scan_message_for_secrets(data["prompt"])
+                for secret in detected_secrets:
+                    data["prompt"] = data["prompt"].replace(
+                        secret["value"], "[REDACTED]"
+                    )
+                if len(detected_secrets) > 0:
+                    secret_types = [secret["type"] for secret in detected_secrets]
+                    verbose_proxy_logger.warning(
+                        f"Detected and redacted secrets in prompt: {secret_types}"
+                    )
+            elif isinstance(data["prompt"], list):
+                for item in data["prompt"]:
+                    if isinstance(item, str):
+                        detected_secrets = self.scan_message_for_secrets(item)
+                        for secret in detected_secrets:
+                            item = item.replace(secret["value"], "[REDACTED]")
+                        if len(detected_secrets) > 0:
+                            secret_types = [
+                                secret["type"] for secret in detected_secrets
+                            ]
+                            verbose_proxy_logger.warning(
+                                f"Detected and redacted secrets in prompt: {secret_types}"
+                            )
+
+        if "input" in data:
+            if isinstance(data["input"], str):
+                detected_secrets = self.scan_message_for_secrets(data["input"])
+                for secret in detected_secrets:
+                    data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
+                if len(detected_secrets) > 0:
+                    secret_types = [secret["type"] for secret in detected_secrets]
+                    verbose_proxy_logger.warning(
+                        f"Detected and redacted secrets in input: {secret_types}"
+                    )
+            elif isinstance(data["input"], list):
+                for item in data["input"]:
+                    if isinstance(item, str):
+                        detected_secrets = self.scan_message_for_secrets(item)
+                        for secret in detected_secrets:
+                            item = item.replace(secret["value"], "[REDACTED]")
+                        if len(detected_secrets) > 0:
+                            secret_types = [
+                                secret["type"] for secret in detected_secrets
+                            ]
+                            verbose_proxy_logger.warning(
+                                f"Detected and redacted secrets in input: {secret_types}"
+                            )
+
+
+# secretDetect = _ENTERPRISE_SecretDetection()
+
+# from litellm.caching import DualCache
+# print("running hook to detect a secret")
+# test_data = {
+#     "messages": [
+#         {"role": "user", "content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'"},
+#         {"role": "assistant", "content": "Hello! I'm doing well. How can I assist you today?"},
+#         {"role": "user", "content": "this is my OPENAI_API_KEY = 'sk_1234567890abcdef'"},
+#          {"role": "user", "content": "i think it is sk-1234567890abcdef"},
+#     ],
+#     "model": "gpt-3.5-turbo",
+# }
+# secretDetect.async_pre_call_hook(
+#     data=test_data,
+#     user_api_key_dict=UserAPIKeyAuth(token="your_api_key"),
+#     cache=DualCache(),
+#     call_type="completion",
+# )
+
+
+# print("finished hook to detect a secret - test data=", test_data)

From e22c9ba0cc0e231b88343f28b1c782fd6515d500 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 16:25:14 -0700
Subject: [PATCH 55/68] init secret detection callback

---
 litellm/proxy/proxy_server.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 59ad7ba922..c3b855c5f5 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -1479,6 +1479,21 @@ class ProxyConfig:
 
                                 llama_guard_object = _ENTERPRISE_LlamaGuard()
                                 imported_list.append(llama_guard_object)
+                            elif (
+                                isinstance(callback, str) and callback == "hide_secrets"
+                            ):
+                                from enterprise.enterprise_hooks.secret_detection import (
+                                    _ENTERPRISE_SecretDetection,
+                                )
+
+                                if premium_user != True:
+                                    raise Exception(
+                                        "Trying to use secret hiding"
+                                        + CommonProxyErrors.not_premium_user.value
+                                    )
+
+                                _secret_detection_object = _ENTERPRISE_SecretDetection()
+                                imported_list.append(_secret_detection_object)
                             elif (
                                 isinstance(callback, str)
                                 and callback == "openai_moderations"

From 4abb83b12dff6b9cada26d9ce86fa29ce3860c9c Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 16:28:47 -0700
Subject: [PATCH 56/68] fix only use crypto imports when needed

---
 litellm/proxy/auth/litellm_license.py | 31 ++++++++++++++++-----------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py
index ec51f904c6..0310dcaf58 100644
--- a/litellm/proxy/auth/litellm_license.py
+++ b/litellm/proxy/auth/litellm_license.py
@@ -5,9 +5,6 @@ import json
 import os
 from datetime import datetime
 
-from cryptography.hazmat.primitives import hashes, serialization
-from cryptography.hazmat.primitives.asymmetric import padding, rsa
-
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
@@ -27,16 +24,22 @@ class LicenseCheck:
         self.read_public_key()
 
     def read_public_key(self):
-        # current dir
-        current_dir = os.path.dirname(os.path.realpath(__file__))
+        try:
+            from cryptography.hazmat.primitives import hashes, serialization
+            from cryptography.hazmat.primitives.asymmetric import padding, rsa
 
-        # check if public_key.pem exists
-        _path_to_public_key = os.path.join(current_dir, "public_key.pem")
-        if os.path.exists(_path_to_public_key):
-            with open(_path_to_public_key, "rb") as key_file:
-                self.public_key = serialization.load_pem_public_key(key_file.read())
-        else:
-            self.public_key = None
+            # current dir
+            current_dir = os.path.dirname(os.path.realpath(__file__))
+
+            # check if public_key.pem exists
+            _path_to_public_key = os.path.join(current_dir, "public_key.pem")
+            if os.path.exists(_path_to_public_key):
+                with open(_path_to_public_key, "rb") as key_file:
+                    self.public_key = serialization.load_pem_public_key(key_file.read())
+            else:
+                self.public_key = None
+        except Exception as e:
+            verbose_proxy_logger.error(f"Error reading public key: {str(e)}")
 
     def _verify(self, license_str: str) -> bool:
         url = "{}/verify_license/{}".format(self.base_url, license_str)
@@ -76,6 +79,9 @@ class LicenseCheck:
 
     def verify_license_without_api_request(self, public_key, license_key):
         try:
+            from cryptography.hazmat.primitives import hashes, serialization
+            from cryptography.hazmat.primitives.asymmetric import padding, rsa
+
             # Decode the license key
             decoded = base64.b64decode(license_key)
             message, signature = decoded.split(b".", 1)
@@ -107,4 +113,5 @@ class LicenseCheck:
             return True
 
         except Exception as e:
+            verbose_proxy_logger.error(str(e))
             return False

From e4653e6032319f781c479d60ebb4c5998bd7d0db Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 16:38:47 -0700
Subject: [PATCH 57/68] fix async_pre_call_hook

---
 enterprise/enterprise_hooks/secret_detection.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/enterprise/enterprise_hooks/secret_detection.py b/enterprise/enterprise_hooks/secret_detection.py
index 75a578b2cc..ade8b71727 100644
--- a/enterprise/enterprise_hooks/secret_detection.py
+++ b/enterprise/enterprise_hooks/secret_detection.py
@@ -63,7 +63,7 @@ class _ENTERPRISE_SecretDetection(CustomLogger):
         return detected_secrets
 
     #### CALL HOOKS - proxy only ####
-    def async_pre_call_hook(
+    async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
         cache: DualCache,

From d98e00d1e02dbcd31023a45219487a1af0a340dd Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 16:51:55 -0700
Subject: [PATCH 58/68] fix(router.py): set `cooldown_time:` per model

---
 litellm/integrations/custom_logger.py         | 12 ++--
 litellm/litellm_core_utils/litellm_logging.py |  3 +-
 litellm/main.py                               |  6 ++
 litellm/router.py                             |  4 +-
 litellm/tests/test_router_cooldowns.py        | 56 ++++++++++++++++++-
 litellm/utils.py                              |  2 +
 6 files changed, 72 insertions(+), 11 deletions(-)

diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py
index 5a6282994c..da9826b9b5 100644
--- a/litellm/integrations/custom_logger.py
+++ b/litellm/integrations/custom_logger.py
@@ -1,11 +1,13 @@
 #### What this does ####
 #    On success, logs events to Promptlayer
-import dotenv, os
-
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.caching import DualCache
-from typing import Literal, Union, Optional
+import os
 import traceback
+from typing import Literal, Optional, Union
+
+import dotenv
+
+from litellm.caching import DualCache
+from litellm.proxy._types import UserAPIKeyAuth
 
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index aa22b51534..add281e43f 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -19,8 +19,7 @@ from litellm import (
     turn_off_message_logging,
     verbose_logger,
 )
-
-from litellm.caching import InMemoryCache, S3Cache, DualCache
+from litellm.caching import DualCache, InMemoryCache, S3Cache
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.redact_messages import (
     redact_message_input_output_from_logging,
diff --git a/litellm/main.py b/litellm/main.py
index 573b2c19fe..b7aa47ab74 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -650,6 +650,7 @@ def completion(
     headers = kwargs.get("headers", None) or extra_headers
     num_retries = kwargs.get("num_retries", None)  ## deprecated
     max_retries = kwargs.get("max_retries", None)
+    cooldown_time = kwargs.get("cooldown_time", None)
     context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None)
     organization = kwargs.get("organization", None)
     ### CUSTOM MODEL COST ###
@@ -763,6 +764,7 @@ def completion(
         "allowed_model_region",
         "model_config",
         "fastest_response",
+        "cooldown_time",
     ]
 
     default_params = openai_params + litellm_params
@@ -947,6 +949,7 @@ def completion(
             input_cost_per_token=input_cost_per_token,
             output_cost_per_second=output_cost_per_second,
             output_cost_per_token=output_cost_per_token,
+            cooldown_time=cooldown_time,
         )
         logging.update_environment_variables(
             model=model,
@@ -3030,6 +3033,7 @@ def embedding(
     client = kwargs.pop("client", None)
     rpm = kwargs.pop("rpm", None)
     tpm = kwargs.pop("tpm", None)
+    cooldown_time = kwargs.get("cooldown_time", None)
     max_parallel_requests = kwargs.pop("max_parallel_requests", None)
     model_info = kwargs.get("model_info", None)
     metadata = kwargs.get("metadata", None)
@@ -3105,6 +3109,7 @@ def embedding(
         "region_name",
         "allowed_model_region",
         "model_config",
+        "cooldown_time",
     ]
     default_params = openai_params + litellm_params
     non_default_params = {
@@ -3165,6 +3170,7 @@ def embedding(
                 "aembedding": aembedding,
                 "preset_cache_key": None,
                 "stream_response": {},
+                "cooldown_time": cooldown_time,
             },
         )
         if azure == True or custom_llm_provider == "azure":
diff --git a/litellm/router.py b/litellm/router.py
index 840df5b54e..e2f7ce8b21 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -2816,7 +2816,9 @@ class Router:
 
             exception_response = getattr(exception, "response", {})
             exception_headers = getattr(exception_response, "headers", None)
-            _time_to_cooldown = self.cooldown_time
+            _time_to_cooldown = kwargs.get("litellm_params", {}).get(
+                "cooldown_time", self.cooldown_time
+            )
 
             if exception_headers is not None:
 
diff --git a/litellm/tests/test_router_cooldowns.py b/litellm/tests/test_router_cooldowns.py
index 35095bb2cf..3eef6e5423 100644
--- a/litellm/tests/test_router_cooldowns.py
+++ b/litellm/tests/test_router_cooldowns.py
@@ -1,18 +1,26 @@
 #### What this tests ####
 #    This tests calling router with fallback models
 
-import sys, os, time
-import traceback, asyncio
+import asyncio
+import os
+import sys
+import time
+import traceback
+
 import pytest
 
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
 
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+import openai
+
 import litellm
 from litellm import Router
 from litellm.integrations.custom_logger import CustomLogger
-import openai, httpx
 
 
 @pytest.mark.asyncio
@@ -62,3 +70,45 @@ async def test_cooldown_badrequest_error():
     assert response is not None
 
     print(response)
+
+
+@pytest.mark.asyncio
+async def test_dynamic_cooldowns():
+    """
+    Assert kwargs for completion/embedding have 'cooldown_time' as a litellm_param
+    """
+    # litellm.set_verbose = True
+    tmp_mock = MagicMock()
+
+    litellm.failure_callback = [tmp_mock]
+
+    router = Router(
+        model_list=[
+            {
+                "model_name": "my-fake-model",
+                "litellm_params": {
+                    "model": "openai/gpt-1",
+                    "api_key": "my-key",
+                    "mock_response": Exception("this is an error"),
+                },
+            }
+        ],
+        cooldown_time=60,
+    )
+
+    try:
+        _ = router.completion(
+            model="my-fake-model",
+            messages=[{"role": "user", "content": "Hey, how's it going?"}],
+            cooldown_time=0,
+            num_retries=0,
+        )
+    except Exception:
+        pass
+
+    tmp_mock.assert_called_once()
+
+    print(tmp_mock.call_count)
+
+    assert "cooldown_time" in tmp_mock.call_args[0][0]["litellm_params"]
+    assert tmp_mock.call_args[0][0]["litellm_params"]["cooldown_time"] == 0
diff --git a/litellm/utils.py b/litellm/utils.py
index 4465c5b0a4..beae7ba4ab 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2017,6 +2017,7 @@ def get_litellm_params(
     input_cost_per_token=None,
     output_cost_per_token=None,
     output_cost_per_second=None,
+    cooldown_time=None,
 ):
     litellm_params = {
         "acompletion": acompletion,
@@ -2039,6 +2040,7 @@ def get_litellm_params(
         "input_cost_per_second": input_cost_per_second,
         "output_cost_per_token": output_cost_per_token,
         "output_cost_per_second": output_cost_per_second,
+        "cooldown_time": cooldown_time,
     }
 
     return litellm_params

From 8fbc34e7e9e1e2f8d05eafb278f39eadfd0cf866 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 17:01:58 -0700
Subject: [PATCH 59/68] docs(routing.md): add dynamic cooldowns to docs

---
 docs/my-website/docs/proxy/reliability.md |  1 +
 docs/my-website/docs/routing.md           | 35 ++++++++++++++++++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md
index c07fc3c26a..9228071b0d 100644
--- a/docs/my-website/docs/proxy/reliability.md
+++ b/docs/my-website/docs/proxy/reliability.md
@@ -272,6 +272,7 @@ litellm_settings:
   fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo"]}] # fallback to gpt-3.5-turbo if call fails num_retries 
   context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error
   allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. 
+  cooldown_time: 30 # how long to cooldown model if fails/min > allowed_fails
 ```
 ### Context Window Fallbacks (Pre-Call Checks + Fallbacks)
 
diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md
index de0a4a7965..240e6c8e04 100644
--- a/docs/my-website/docs/routing.md
+++ b/docs/my-website/docs/routing.md
@@ -762,6 +762,9 @@ asyncio.run(router_acompletion())
 
 Set the limit for how many calls a model is allowed to fail in a minute, before being cooled down for a minute. 
 
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
 ```python
 from litellm import Router
 
@@ -779,9 +782,39 @@ messages = [{"content": user_message, "role": "user"}]
 response = router.completion(model="gpt-3.5-turbo", messages=messages)
 
 print(f"response: {response}")
-
 ```
 
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+**Set Global Value**
+
+```yaml
+router_settings:
+	allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. 
+  	cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
+```
+
+Defaults:
+- allowed_fails: 0
+- cooldown_time: 60s 
+
+**Set Per Model**
+
+```yaml
+model_list:
+- model_name: fake-openai-endpoint
+  litellm_params:
+    model: predibase/llama-3-8b-instruct
+    api_key: os.environ/PREDIBASE_API_KEY
+    tenant_id: os.environ/PREDIBASE_TENANT_ID
+    max_new_tokens: 256
+    cooldown_time: 0 # 👈 KEY CHANGE
+```
+
+</TabItem>
+</Tabs>
+
 ### Retries
 
 For both async + sync functions, we support retrying failed requests. 

From 304e84f87c299fe77c3aff4bac089b2454ab0b2b Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 17:25:53 -0700
Subject: [PATCH 60/68] clean up secret detection

---
 .../enterprise_hooks/secret_detection.py      | 33 ++++---------------
 requirements.txt                              |  1 +
 2 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/enterprise/enterprise_hooks/secret_detection.py b/enterprise/enterprise_hooks/secret_detection.py
index ade8b71727..ded9f27c17 100644
--- a/enterprise/enterprise_hooks/secret_detection.py
+++ b/enterprise/enterprise_hooks/secret_detection.py
@@ -126,11 +126,14 @@ class _ENTERPRISE_SecretDetection(CustomLogger):
                         f"Detected and redacted secrets in input: {secret_types}"
                     )
             elif isinstance(data["input"], list):
-                for item in data["input"]:
+                _input_in_request = data["input"]
+                for idx, item in enumerate(_input_in_request):
                     if isinstance(item, str):
                         detected_secrets = self.scan_message_for_secrets(item)
                         for secret in detected_secrets:
-                            item = item.replace(secret["value"], "[REDACTED]")
+                            _input_in_request[idx] = item.replace(
+                                secret["value"], "[REDACTED]"
+                            )
                         if len(detected_secrets) > 0:
                             secret_types = [
                                 secret["type"] for secret in detected_secrets
@@ -138,27 +141,5 @@ class _ENTERPRISE_SecretDetection(CustomLogger):
                             verbose_proxy_logger.warning(
                                 f"Detected and redacted secrets in input: {secret_types}"
                             )
-
-
-# secretDetect = _ENTERPRISE_SecretDetection()
-
-# from litellm.caching import DualCache
-# print("running hook to detect a secret")
-# test_data = {
-#     "messages": [
-#         {"role": "user", "content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'"},
-#         {"role": "assistant", "content": "Hello! I'm doing well. How can I assist you today?"},
-#         {"role": "user", "content": "this is my OPENAI_API_KEY = 'sk_1234567890abcdef'"},
-#          {"role": "user", "content": "i think it is sk-1234567890abcdef"},
-#     ],
-#     "model": "gpt-3.5-turbo",
-# }
-# secretDetect.async_pre_call_hook(
-#     data=test_data,
-#     user_api_key_dict=UserAPIKeyAuth(token="your_api_key"),
-#     cache=DualCache(),
-#     call_type="completion",
-# )
-
-
-# print("finished hook to detect a secret - test data=", test_data)
+                verbose_proxy_logger.debug("Data after redacting input %s", data)
+        return
diff --git a/requirements.txt b/requirements.txt
index fbf2bfc1d1..e40c44e4d0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -31,6 +31,7 @@ azure-identity==1.16.1 # for azure content safety
 opentelemetry-api==1.25.0
 opentelemetry-sdk==1.25.0
 opentelemetry-exporter-otlp==1.25.0
+detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
 
 ### LITELLM PACKAGE DEPENDENCIES
 python-dotenv==1.0.0 # for env 

From 66f0d6ae8f611bb289508f7ba0da4fc403686f27 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 17:27:02 -0700
Subject: [PATCH 61/68] test secret detection

---
 litellm/proxy/proxy_config.yaml          |   2 +-
 litellm/tests/test_secret_detect_hook.py | 216 +++++++++++++++++++++++
 2 files changed, 217 insertions(+), 1 deletion(-)
 create mode 100644 litellm/tests/test_secret_detect_hook.py

diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index caa6bc13b9..0c0365f43d 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -26,7 +26,7 @@ general_settings:
 
 litellm_settings:
   success_callback: ["prometheus"]
-  callbacks: ["otel"]
+  callbacks: ["otel", "hide_secrets"]
   failure_callback: ["prometheus"]
   store_audit_logs: true
   redact_messages_in_exceptions: True
diff --git a/litellm/tests/test_secret_detect_hook.py b/litellm/tests/test_secret_detect_hook.py
new file mode 100644
index 0000000000..a1bf10ebad
--- /dev/null
+++ b/litellm/tests/test_secret_detect_hook.py
@@ -0,0 +1,216 @@
+# What is this?
+## This tests the llm guard integration
+
+import asyncio
+import os
+import random
+
+# What is this?
+## Unit test for presidio pii masking
+import sys
+import time
+import traceback
+from datetime import datetime
+
+from dotenv import load_dotenv
+
+load_dotenv()
+import os
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import pytest
+
+import litellm
+from litellm import Router, mock_completion
+from litellm.caching import DualCache
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.enterprise.enterprise_hooks.secret_detection import (
+    _ENTERPRISE_SecretDetection,
+)
+from litellm.proxy.utils import ProxyLogging, hash_token
+
+### UNIT TESTS FOR OpenAI Moderation ###
+
+
+@pytest.mark.asyncio
+async def test_basic_secret_detection_chat():
+    """
+    Tests to see if secret detection hook will mask api keys
+
+
+    It should mask the following API_KEY = 'sk_1234567890abcdef' and  OPENAI_API_KEY = 'sk_1234567890abcdef'
+    """
+    secret_instance = _ENTERPRISE_SecretDetection()
+    _api_key = "sk-12345"
+    _api_key = hash_token("sk-12345")
+    user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
+    local_cache = DualCache()
+
+    from litellm.proxy.proxy_server import llm_router
+
+    test_data = {
+        "messages": [
+            {
+                "role": "user",
+                "content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'",
+            },
+            {
+                "role": "assistant",
+                "content": "Hello! I'm doing well. How can I assist you today?",
+            },
+            {
+                "role": "user",
+                "content": "this is my OPENAI_API_KEY = 'sk_1234567890abcdef'",
+            },
+            {"role": "user", "content": "i think it is +1 412-555-5555"},
+        ],
+        "model": "gpt-3.5-turbo",
+    }
+
+    await secret_instance.async_pre_call_hook(
+        cache=local_cache,
+        data=test_data,
+        user_api_key_dict=user_api_key_dict,
+        call_type="completion",
+    )
+    print(
+        "test data after running pre_call_hook: Expect all API Keys to be masked",
+        test_data,
+    )
+
+    assert test_data == {
+        "messages": [
+            {"role": "user", "content": "Hey, how's it going, API_KEY = '[REDACTED]'"},
+            {
+                "role": "assistant",
+                "content": "Hello! I'm doing well. How can I assist you today?",
+            },
+            {"role": "user", "content": "this is my OPENAI_API_KEY = '[REDACTED]'"},
+            {"role": "user", "content": "i think it is +1 412-555-5555"},
+        ],
+        "model": "gpt-3.5-turbo",
+    }, "Expect all API Keys to be masked"
+
+
+@pytest.mark.asyncio
+async def test_basic_secret_detection_text_completion():
+    """
+    Tests to see if secret detection hook will mask api keys
+
+
+    It should mask the following API_KEY = 'sk_1234567890abcdef' and  OPENAI_API_KEY = 'sk_1234567890abcdef'
+    """
+    secret_instance = _ENTERPRISE_SecretDetection()
+    _api_key = "sk-12345"
+    _api_key = hash_token("sk-12345")
+    user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
+    local_cache = DualCache()
+
+    from litellm.proxy.proxy_server import llm_router
+
+    test_data = {
+        "prompt": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef', my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather",
+        "model": "gpt-3.5-turbo",
+    }
+
+    await secret_instance.async_pre_call_hook(
+        cache=local_cache,
+        data=test_data,
+        user_api_key_dict=user_api_key_dict,
+        call_type="completion",
+    )
+
+    test_data == {
+        "prompt": "Hey, how's it going, API_KEY = '[REDACTED]', my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather",
+        "model": "gpt-3.5-turbo",
+    }
+    print(
+        "test data after running pre_call_hook: Expect all API Keys to be masked",
+        test_data,
+    )
+
+
+@pytest.mark.asyncio
+async def test_basic_secret_detection_embeddings():
+    """
+    Tests to see if secret detection hook will mask api keys
+
+
+    It should mask the following API_KEY = 'sk_1234567890abcdef' and  OPENAI_API_KEY = 'sk_1234567890abcdef'
+    """
+    secret_instance = _ENTERPRISE_SecretDetection()
+    _api_key = "sk-12345"
+    _api_key = hash_token("sk-12345")
+    user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
+    local_cache = DualCache()
+
+    from litellm.proxy.proxy_server import llm_router
+
+    test_data = {
+        "input": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef', my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather",
+        "model": "gpt-3.5-turbo",
+    }
+
+    await secret_instance.async_pre_call_hook(
+        cache=local_cache,
+        data=test_data,
+        user_api_key_dict=user_api_key_dict,
+        call_type="embedding",
+    )
+
+    assert test_data == {
+        "input": "Hey, how's it going, API_KEY = '[REDACTED]', my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather",
+        "model": "gpt-3.5-turbo",
+    }
+    print(
+        "test data after running pre_call_hook: Expect all API Keys to be masked",
+        test_data,
+    )
+
+
+@pytest.mark.asyncio
+async def test_basic_secret_detection_embeddings_list():
+    """
+    Tests to see if secret detection hook will mask api keys
+
+
+    It should mask the following API_KEY = 'sk_1234567890abcdef' and  OPENAI_API_KEY = 'sk_1234567890abcdef'
+    """
+    secret_instance = _ENTERPRISE_SecretDetection()
+    _api_key = "sk-12345"
+    _api_key = hash_token("sk-12345")
+    user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
+    local_cache = DualCache()
+
+    from litellm.proxy.proxy_server import llm_router
+
+    test_data = {
+        "input": [
+            "hey",
+            "how's it going, API_KEY = 'sk_1234567890abcdef'",
+            "my OPENAI_API_KEY = 'sk_1234567890abcdef' and i want to know what is the weather",
+        ],
+        "model": "gpt-3.5-turbo",
+    }
+
+    await secret_instance.async_pre_call_hook(
+        cache=local_cache,
+        data=test_data,
+        user_api_key_dict=user_api_key_dict,
+        call_type="embedding",
+    )
+
+    print(
+        "test data after running pre_call_hook: Expect all API Keys to be masked",
+        test_data,
+    )
+    assert test_data == {
+        "input": [
+            "hey",
+            "how's it going, API_KEY = '[REDACTED]'",
+            "my OPENAI_API_KEY = '[REDACTED]' and i want to know what is the weather",
+        ],
+        "model": "gpt-3.5-turbo",
+    }

From 734ed571e9f91d5f060116afcd5a78e35baff019 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 17:42:54 -0700
Subject: [PATCH 62/68] docs - secret detection

---
 docs/my-website/docs/enterprise.md       |   3 +-
 docs/my-website/docs/proxy/enterprise.md | 101 +++++++++++++++++++++--
 2 files changed, 98 insertions(+), 6 deletions(-)

diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md
index 2d45ea3ea7..875aec57f0 100644
--- a/docs/my-website/docs/enterprise.md
+++ b/docs/my-website/docs/enterprise.md
@@ -13,7 +13,8 @@ This covers:
 - ✅ [**Audit Logs with retention policy**](../docs/proxy/enterprise.md#audit-logs)
 - ✅ [**JWT-Auth**](../docs/proxy/token_auth.md)
 - ✅ [**Control available public, private routes**](../docs/proxy/enterprise.md#control-available-public-private-routes)
-- ✅ [**Prompt Injection Detection**](#prompt-injection-detection-lakeraai)
+- ✅ [**Guardrails, Content Moderation, PII Masking, Secret/API Key Masking**](../docs/proxy/enterprise.md#prompt-injection-detection---lakeraai)
+- ✅ [**Prompt Injection Detection**](../docs/proxy/enterprise.md#prompt-injection-detection---lakeraai)
 - ✅ [**Invite Team Members to access `/spend` Routes**](../docs/proxy/cost_tracking#allowing-non-proxy-admins-to-access-spend-endpoints)
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md
index 40a5261cd5..9fff879e54 100644
--- a/docs/my-website/docs/proxy/enterprise.md
+++ b/docs/my-website/docs/proxy/enterprise.md
@@ -15,10 +15,10 @@ Features:
 - ✅ [Audit Logs](#audit-logs)
 - ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags)
 - ✅ [Control available public, private routes](#control-available-public-private-routes)
-- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests)
-- ✅ [Content Moderation with LLM Guard, LlamaGuard, Google Text Moderations](#content-moderation)
+- ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](#content-moderation)
 - ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai)
 - ✅ [Custom Branding + Routes on Swagger Docs](#swagger-docs---custom-routes--branding)
+- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests)
 - ✅ Reject calls from Blocked User list 
 - ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors)
 
@@ -495,7 +495,98 @@ curl --request POST \
 
 
 ## Content Moderation
-#### Content Moderation with LLM Guard
+### Content Moderation - Secret Detection
+❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM. 
+
+Example if you want to redact the value of `OPENAI_API_KEY` in the following request
+
+#### Incoming Request 
+
+```json
+{
+    "messages": [
+        {
+            "role": "user",
+            "content": "Hey, how's it going, API_KEY = 'sk_1234567890abcdef'",
+        }
+    ]
+}
+```
+
+#### Request after Moderation
+
+```json
+{
+    "messages": [
+        {
+            "role": "user",
+            "content": "Hey, how's it going, API_KEY = '[REDACTED]'",
+        }
+    ]
+}
+```
+
+**Usage**
+
+**Step 1** Add this to your config.yaml 
+
+```yaml
+litellm_settings:
+  callbacks: ["hide_secrets"]
+```
+
+**Step 2** Run litellm proxy with `--detailed_debug` to see the server logs
+
+```
+litellm --config config.yaml --detailed_debug
+```
+
+**Step 3** Test it with request
+
+Send this request
+```shell
+curl --location 'http://localhost:4000/chat/completions' \
+    --header 'Authorization: Bearer sk-1234' \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "model": "llama3",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what is the value of my open ai key? openai_api_key=sk-1234998222"
+        }
+    ]
+}'
+```
+
+
+Expect to see the following warning on your litellm server logs
+
+```shell
+LiteLLM Proxy:WARNING: secret_detection.py:88 - Detected and redacted secrets in message: ['Secret Keyword']
+```
+
+
+You can also see the raw request sent from litellm to the API Provider
+```json
+POST Request Sent from LiteLLM:
+curl -X POST \
+https://api.groq.com/openai/v1/ \
+-H 'Authorization: Bearer gsk_mySVchjY********************************************' \
+-d {
+  "model": "llama3-8b-8192",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what is the time today, openai_api_key=[REDACTED]"
+    }
+  ],
+  "stream": false,
+  "extra_body": {}
+}
+```
+
+### Content Moderation with LLM Guard
 
 Set the LLM Guard API Base in your environment 
 
@@ -630,7 +721,7 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \
 </TabItem>
 </Tabs>
 
-#### Content Moderation with LlamaGuard 
+### Content Moderation with LlamaGuard 
 
 Currently works with Sagemaker's LlamaGuard endpoint. 
 
@@ -664,7 +755,7 @@ callbacks: ["llamaguard_moderations"]
 
 
 
-#### Content Moderation with Google Text Moderation 
+### Content Moderation with Google Text Moderation 
 
 Requires your GOOGLE_APPLICATION_CREDENTIALS to be set in your .env (same as VertexAI).
 

From 33c11c94f87d4deb931bb34a228885ebe8867d16 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 17:44:36 -0700
Subject: [PATCH 63/68] fix detect secrets test

---
 .circleci/config.yml             | 3 ++-
 litellm/tests/test_completion.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fd1b48a9c6..5dfeedcaa2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -48,7 +48,8 @@ jobs:
             pip install opentelemetry-sdk==1.25.0
             pip install opentelemetry-exporter-otlp==1.25.0
             pip install openai
-            pip install prisma            
+            pip install prisma   
+            pip install "detect_secrets==1.5.0"         
             pip install "httpx==0.24.1"
             pip install fastapi
             pip install "gunicorn==21.2.0"
diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 30ae1d0ab1..0c6da360bb 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.prompt_templates.factory import anthropic_messages_pt
 
-# litellm.num_retries = 3
+# litellm.num_retries=3
 litellm.cache = None
 litellm.success_callback = []
 user_message = "Write a short poem about the sky"

From 45c748475bb42c3354cf01485e647510542a7952 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 18:13:31 -0700
Subject: [PATCH 64/68] =?UTF-8?q?bump:=20version=201.40.26=20=E2=86=92=201?=
 =?UTF-8?q?.40.27?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 6b4884b5bb..321f44b23b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.40.26"
+version = "1.40.27"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -90,7 +90,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.40.26"
+version = "1.40.27"
 version_files = [
     "pyproject.toml:^version"
 ]

From ac1f343487878dfb1aa02dc8b79314fb281451a9 Mon Sep 17 00:00:00 2001
From: Krish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 18:19:24 -0700
Subject: [PATCH 65/68] Revert "Create litellm user to fix issue with prisma in
 k8s "

---
 Dockerfile.database | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/Dockerfile.database b/Dockerfile.database
index 1901200d52..22084bab89 100644
--- a/Dockerfile.database
+++ b/Dockerfile.database
@@ -9,27 +9,6 @@ FROM $LITELLM_BUILD_IMAGE as builder
 # Set the working directory to /app
 WORKDIR /app
 
-ARG LITELLM_USER=litellm LITELLM_UID=1729
-ARG LITELLM_GROUP=litellm LITELLM_GID=1729
-
-RUN groupadd \
-	--gid ${LITELLM_GID} \
-	${LITELLM_GROUP} \
-	&& useradd \
-	--create-home \
-	--shell /bin/sh \
-	--gid ${LITELLM_GID} \
-	--uid ${LITELLM_UID} \
-	${LITELLM_USER}
-
-# Allows user to update python install.
-# This is necessary for prisma.
-RUN chown -R ${LITELLM_USER}:${LITELLM_GROUP} /usr/local/lib/python3.11
-
-# Set the HOME var forcefully because of prisma.
-ENV HOME=/home/${LITELLM_USER}
-USER ${LITELLM_USER}
-
 # Install build dependencies
 RUN apt-get clean && apt-get update && \
     apt-get install -y gcc python3-dev && \

From eca4876bbf1cad01042c2f44aa7c018249055a73 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 25 Jun 2024 18:21:57 -0700
Subject: [PATCH 66/68] run again

---
 litellm/tests/test_completion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 0c6da360bb..30ae1d0ab1 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.prompt_templates.factory import anthropic_messages_pt
 
-# litellm.num_retries=3
+# litellm.num_retries = 3
 litellm.cache = None
 litellm.success_callback = []
 user_message = "Write a short poem about the sky"

From d6ed8c10b24386f0fe642022319b684cfe64ea57 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 25 Jun 2024 18:26:16 -0700
Subject: [PATCH 67/68] docs(function_call.md): cleanup

---
 docs/my-website/docs/completion/function_call.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/my-website/docs/completion/function_call.md b/docs/my-website/docs/completion/function_call.md
index 5daccf7232..514e8cda1a 100644
--- a/docs/my-website/docs/completion/function_call.md
+++ b/docs/my-website/docs/completion/function_call.md
@@ -502,10 +502,10 @@ response = completion(model="gpt-3.5-turbo-0613", messages=messages, functions=f
 print(response)
 ```
 
-## Function calling for Non-OpenAI LLMs
+## Function calling for Models w/out function-calling support
 
 ### Adding Function to prompt
-For Non OpenAI LLMs LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True`
+For Models/providers without function calling support, LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True`
 
 #### Usage
 ```python

From 09f4eb7617e05bd057154a2519598381b3ef265b Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Wed, 26 Jun 2024 08:09:14 -0700
Subject: [PATCH 68/68] docs(reliable_completions.md): improve headers for
 easier searching

---
 .../docs/completion/reliable_completions.md        | 14 ++++++++++----
 litellm/llms/azure.py                              |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/docs/my-website/docs/completion/reliable_completions.md b/docs/my-website/docs/completion/reliable_completions.md
index 2656f9a4fb..94102e1944 100644
--- a/docs/my-website/docs/completion/reliable_completions.md
+++ b/docs/my-website/docs/completion/reliable_completions.md
@@ -31,9 +31,15 @@ response = completion(
         )
 ```
 
-## Fallbacks 
+## Fallbacks (SDK)
 
-### Context Window Fallbacks
+:::info
+
+[See how to do on PROXY](../proxy/reliability.md)
+
+:::
+
+### Context Window Fallbacks (SDK)
 ```python 
 from litellm import completion
 
@@ -43,7 +49,7 @@ messages = [{"content": "how does a court case get to the Supreme Court?" * 500,
 completion(model="gpt-3.5-turbo", messages=messages, context_window_fallback_dict=ctx_window_fallback_dict)
 ```
 
-### Fallbacks - Switch Models/API Keys/API Bases
+### Fallbacks - Switch Models/API Keys/API Bases (SDK)
 
 LLM APIs can be unstable, completion() with fallbacks ensures you'll always get a response from your calls
 
@@ -69,7 +75,7 @@ response = completion(model="azure/gpt-4", messages=messages, api_key=api_key,
 
 [Check out this section for implementation details](#fallbacks-1)
 
-## Implementation Details 
+## Implementation Details (SDK)
 
 ### Fallbacks
 #### Output from calls
diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py
index c292c3423f..b763a7c955 100644
--- a/litellm/llms/azure.py
+++ b/litellm/llms/azure.py
@@ -902,7 +902,7 @@ class AzureChatCompletion(BaseLLM):
                 },
             )
 
-            if aembedding == True:
+            if aembedding is True:
                 response = self.aembedding(
                     data=data,
                     input=input,