diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 22e95d7717..954d2496ea 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -327,6 +327,16 @@ class ProxyLogging: _api_base = "" request_info += f"\nAPI Base: {_api_base}" + elif request_data.get("metadata", None) is not None and isinstance( + request_data["metadata"], dict + ): + # In hanging requests sometime it has not made it to the point where the deployment is passed to the `request_data`` + # in that case we fallback to the api base set in the request metadata + _metadata = request_data["metadata"] + _api_base = _metadata.get("api_base", "") + if _api_base is None: + _api_base = "" + request_info += f"\nAPI Base: `{_api_base}`" # only alert hanging responses if they have not been marked as success alerting_message = ( f"`Requests are hanging - {self.alerting_threshold}s+ request time`" diff --git a/litellm/router.py b/litellm/router.py index abbd6343b4..e67507318e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -351,6 +351,7 @@ class Router: kwargs.setdefault("metadata", {}).update( { "deployment": deployment["litellm_params"]["model"], + "api_base": deployment.get("litellm_params", {}).get("api_base"), "model_info": deployment.get("model_info", {}), } ) @@ -438,6 +439,7 @@ class Router: { "deployment": deployment["litellm_params"]["model"], "model_info": deployment.get("model_info", {}), + "api_base": deployment.get("litellm_params", {}).get("api_base"), } ) kwargs["model_info"] = deployment.get("model_info", {}) @@ -951,6 +953,7 @@ class Router: { "deployment": deployment["litellm_params"]["model"], "model_info": deployment.get("model_info", {}), + "api_base": deployment.get("litellm_params", {}).get("api_base"), } ) kwargs["model_info"] = deployment.get("model_info", {}) @@ -1118,6 +1121,7 @@ class Router: { "deployment": deployment["litellm_params"]["model"], "model_info": deployment.get("model_info", {}), + "api_base": deployment.get("litellm_params", {}).get("api_base"), } ) kwargs["model_info"] = deployment.get("model_info", {})