From a5d9a6cebdc36ddf35040637edeeee43bb2d4b3d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 6 Jul 2024 13:14:04 -0700 Subject: [PATCH] init litellm router --- litellm/proxy/proxy_server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d633075b77..521ded6272 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -207,6 +207,7 @@ from litellm.router import ModelInfo as RouterModelInfo from litellm.router import updateDeployment from litellm.scheduler import DefaultPriorities, FlowItem, Scheduler from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import RouterGeneralSettings try: from litellm._version import version @@ -1765,7 +1766,11 @@ class ProxyConfig: if k in available_args: router_params[k] = v router = litellm.Router( - **router_params, assistants_config=assistants_config + **router_params, + assistants_config=assistants_config, + router_general_settings=RouterGeneralSettings( + async_only_mode=True # only init async clients + ), ) # type:ignore return router, router.get_model_list(), general_settings @@ -1957,7 +1962,12 @@ class ProxyConfig: ) if len(_model_list) > 0: verbose_proxy_logger.debug(f"_model_list: {_model_list}") - llm_router = litellm.Router(model_list=_model_list) + llm_router = litellm.Router( + model_list=_model_list, + router_general_settings=RouterGeneralSettings( + async_only_mode=True # only init async clients + ), + ) verbose_proxy_logger.debug(f"updated llm_router: {llm_router}") else: verbose_proxy_logger.debug(f"len new_models: {len(new_models)}")