From 8e2b139f8d7bf73eaef603f5ebf4a5f3f3770b4b Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Wed, 27 Sep 2023 20:51:31 -0700
Subject: [PATCH] fix petals bug

---
 litellm/utils.py | 2 ++
 pyproject.toml   | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index 4d3481d8bf..a4b1a62dd5 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -1126,6 +1126,8 @@ def get_optional_params(  # use the openai defaults
         # max_new_tokens=1,temperature=0.9, top_p=0.6
         if max_tokens != float("inf"):
             optional_params["max_new_tokens"] = max_tokens
+        else:
+            optional_params["max_new_tokens"] = 256 # petals always needs max_new_tokens
         if temperature != 1:
             optional_params["temperature"] = temperature
         if top_p != 1:
diff --git a/pyproject.toml b/pyproject.toml
index 0f4786428a..3ffbfd1a76 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.788"
+version = "0.1.789"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"