From 9805269bd7dfa8cf82ccc5e009bcafaacc873548 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Dec 2023 14:07:31 -0800 Subject: [PATCH] (feat) proxy: pass model_info as kwargs --- litellm/router.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index c6ff34c811..edbc3cd743 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -187,6 +187,7 @@ class Router: deployment = self.get_available_deployment(model=model, messages=messages, specific_deployment=kwargs.pop("specific_deployment", None)) kwargs.setdefault("metadata", {}).update({"deployment": deployment["litellm_params"]["model"]}) data = deployment["litellm_params"].copy() + kwargs["model_info"] = deployment.get("model_info", {}) for k, v in self.default_litellm_params.items(): if k not in data: # prioritize model-specific params > default router params data[k] = v @@ -234,6 +235,7 @@ class Router: original_model_string = None # set a default for this variable deployment = self.get_available_deployment(model=model, messages=messages, specific_deployment=kwargs.pop("specific_deployment", None)) kwargs.setdefault("metadata", {}).update({"deployment": deployment["litellm_params"]["model"]}) + kwargs["model_info"] = deployment.get("model_info", {}) data = deployment["litellm_params"].copy() for k, v in self.default_litellm_params.items(): if k not in data: # prioritize model-specific params > default router params @@ -303,6 +305,7 @@ class Router: # pick the one that is available (lowest TPM/RPM) deployment = self.get_available_deployment(model=model, input=input, specific_deployment=kwargs.pop("specific_deployment", None)) kwargs.setdefault("metadata", {}).update({"deployment": deployment["litellm_params"]["model"]}) + kwargs["model_info"] = deployment.get("model_info", {}) data = deployment["litellm_params"].copy() for k, v in self.default_litellm_params.items(): if k not in data: # prioritize model-specific params > default router params @@ -329,6 +332,7 @@ class Router: deployment = self.get_available_deployment(model=model, input=input, specific_deployment=kwargs.pop("specific_deployment", None)) kwargs.setdefault("metadata", {}).update({"deployment": deployment["litellm_params"]["model"]}) data = deployment["litellm_params"].copy() + kwargs["model_info"] = deployment.get("model_info", {}) for k, v in self.default_litellm_params.items(): if k not in data: # prioritize model-specific params > default router params data[k] = v