tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-15 20:20:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

(fix) router: set default rpm/tpm when not set

Browse Source
This commit is contained in:
ishaan-jaff
2023-11-29 18:13:27 -08:00
parent c1914a01bc
commit 7bcc23e8e9
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/router.py
+2 -2
View File
@@ -941,7 +941,7 @@ class Router:
rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
if rpm is not None:
# use weight-random pick if rpms provided
rpms = [m["litellm_params"].get("rpm") for m in healthy_deployments]
rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
self.print_verbose(f"\nrpms {rpms}")
total_rpm = sum(rpms)
weights = [rpm / total_rpm for rpm in rpms]
@@ -955,7 +955,7 @@ class Router:
tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
if tpm is not None:
# use weight-random pick if rpms provided
tpms = [m["litellm_params"].get("tpm") for m in healthy_deployments]
tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
self.print_verbose(f"\ntpms {tpms}")
total_tpm = sum(tpms)
weights = [tpm / total_tpm for tpm in tpms]