From f04604910bde07823ee00cf9c6ff840a2d7e3c51 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 20:57:22 -0700 Subject: [PATCH 1/3] fix - show api base on hanging requests --- litellm/proxy/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 22e95d7717..071b882d22 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -327,6 +327,16 @@ class ProxyLogging: _api_base = "" request_info += f"\nAPI Base: {_api_base}" + elif request_data.get("metadata", None) is not None and isinstance( + request_data["metadata"], dict + ): + # In hanging requests sometime it has not made it to the point where the deployment has been set + # in that case we fallback to the api base set in the request metadata + _metadata = request_data["metadata"] + _api_base = _metadata.get("api_base", "") + if _api_base is None: + _api_base = "" + request_info += f"\nAPI Base: {_api_base}" # only alert hanging responses if they have not been marked as success alerting_message = ( f"`Requests are hanging - {self.alerting_threshold}s+ request time`" From 67d356b93377eeb9898ef1aaad075494cd17e2a9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 20:58:02 -0700 Subject: [PATCH 2/3] fix - show api base on hanging requests --- litellm/router.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index abbd6343b4..e67507318e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -351,6 +351,7 @@ class Router: kwargs.setdefault("metadata", {}).update( { "deployment": deployment["litellm_params"]["model"], + "api_base": deployment.get("litellm_params", {}).get("api_base"), "model_info": deployment.get("model_info", {}), } ) @@ -438,6 +439,7 @@ class Router: { "deployment": deployment["litellm_params"]["model"], "model_info": deployment.get("model_info", {}), + "api_base": deployment.get("litellm_params", {}).get("api_base"), } ) kwargs["model_info"] = deployment.get("model_info", {}) @@ -951,6 +953,7 @@ class Router: { "deployment": deployment["litellm_params"]["model"], "model_info": deployment.get("model_info", {}), + "api_base": deployment.get("litellm_params", {}).get("api_base"), } ) kwargs["model_info"] = deployment.get("model_info", {}) @@ -1118,6 +1121,7 @@ class Router: { "deployment": deployment["litellm_params"]["model"], "model_info": deployment.get("model_info", {}), + "api_base": deployment.get("litellm_params", {}).get("api_base"), } ) kwargs["model_info"] = deployment.get("model_info", {}) From 6f948cd55983d44ce6ef199cc58d01094bf87257 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 18 Apr 2024 21:01:26 -0700 Subject: [PATCH 3/3] fix - show api_base in hanging requests --- litellm/proxy/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 071b882d22..954d2496ea 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -330,13 +330,13 @@ class ProxyLogging: elif request_data.get("metadata", None) is not None and isinstance( request_data["metadata"], dict ): - # In hanging requests sometime it has not made it to the point where the deployment has been set + # In hanging requests sometime it has not made it to the point where the deployment is passed to the `request_data`` # in that case we fallback to the api base set in the request metadata _metadata = request_data["metadata"] _api_base = _metadata.get("api_base", "") if _api_base is None: _api_base = "" - request_info += f"\nAPI Base: {_api_base}" + request_info += f"\nAPI Base: `{_api_base}`" # only alert hanging responses if they have not been marked as success alerting_message = ( f"`Requests are hanging - {self.alerting_threshold}s+ request time`"