From 5a1a1908c16a2cb547ae20c7406a59fd4dcfa49d Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 2 Oct 2023 21:35:16 -0700
Subject: [PATCH] fix replicate error

---
 litellm/__init__.py                          |   2 +-
 litellm/__pycache__/__init__.cpython-311.pyc | Bin 11989 -> 11989 bytes
 litellm/__pycache__/main.cpython-311.pyc     | Bin 52849 -> 52792 bytes
 litellm/__pycache__/utils.cpython-311.pyc    | Bin 139221 -> 139221 bytes
 litellm/llms/replicate.py                    |  34 +++++++++++++------
 litellm/utils.py                             |   5 +--
 pyproject.toml                               |   2 +-
 7 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index d806f534f1..9afa4aeeb4 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -308,7 +308,7 @@ from .utils import (
     validate_environment,
     check_valid_key,
     get_llm_provider,
-    completion_with_config
+    completion_with_config,
 )
 from .main import *  # type: ignore
 from .integrations import *
diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc
index 72b2291b4275cec8e830eea429ed763d8f66bdf6..191de47abc1fe9ee45ff507dfa8815c568ae1a79 100644
GIT binary patch
delta 22
ccmcZ_do`AKIWI340}yPVDxLauBku`409RB8vH$=8

delta 22
ccmcZ_do`AKIWI340}ymdh^Kzp$a_K$08y<60ssI2

diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc
index b103a749a9c4b04bdfe2386079a932f5e2cd25b0..6281df1f8d1a9fcd0f4cd4ab7945ef89286dddb0 100644
GIT binary patch
delta 1280
zcmX|AeN0<b6u;-aeyyyu1={jjptPec9k5b223wf$+CG?Wj1RY&7zxtV7+o?LV9W$k
z<wG`x(sG%08C&2FqKTVAZenCJnbkPM=8}P$u$VYMV$2dFkA&zV$@IPp@#Veq`~A*2
z_jm7k$z7Y`Kb_(Y?;8wyL?2Cq<Hf7<h8L{xG&yDEcu0}cf)gkt?Ewx_IGwi$Yvg=>
zoaU%-5U;ZeV1%?4YB_bP5A_veaNEd$dl_Ji94nFl9+FF*06;Uj<8=UhNq+Z*0bHcS
zU$$vz$K55p)NQq-4aP}hX&JyH++X%J-BjA;0^rF@finPg_~&v0e~=47>iHD;;R$MX
zgZ#Zase&$YuF9c;7@4i!4R9OZKk&f#T%Qno^CYJX)z#W_1J6eeV@pj(#=ustsplrk
zm6Qcx0JqmT)iZR(AP&|(3qv@*&x7x|1XWBG8^(8P1Mn8!sQnm5@Q0!g&f?~3D}E@R
zfOEJZbQeakBkasLzhyif8^u!i5RBnuc!}21<@xxHhz8!q?L{_xC6Y<mwa5&Ow4#Mm
zh;I81?v~2@BmE714KdZGPq&8hku6VS&-cn8n@?;UFG?klV28BVHEl~ysbdpU+*ORb
z>khyqp0BgvIr=@QJHx-*XPUTX!JYM`@E*Qg?}CfuX8j49wyS`ebvg0rq<{~%Mj?mX
zZ=*kcm}uMg>)<;QdU+7w5V`fr2f6s)!7QA3t$@r74y)q`-BQ;Vh*5;@7KLpJ-m5Nd
z_MoLAP{fi)-%`vk1@^S;;Fou(DPJ!3DTQxMQjvly6_P{2AI%}Q`X_(Lpy*#JLoNk>
z6Al(D_$Mb?6<ihaI`*Q~3g*7LH?Hq2<z?jS^zpJ!t?*K2zm^krD(&lgL_xv7GDB_!
z9|bq<|EXc_8!lC^7j1Z$+lJTNTgh)!M)oP4>Dpkpijl}pl^6-2bYvH6OGg4`Q9^uL
zLg6P*vW|l_QaK|>%EbU9O@$)cFWth67M6KN2pJhU<_|JF?vU7I>E~5q1<SPVkEj`m
zBE1-O^XaHQTa4yY%%^!b86Uj}Fh~4jD=K>FxTFo>F|kj~0$eAbW0ejH@mH5_(+iTA
zI;#UO5oRCj;V8+So7TW0S@`4v2fvcWg`EKF<n`+x(R)SwpN(sPBMV=obn)XR(91}P
zD;b$G&!h$!vt=!#I%LX`w=t?mIW{?i(d{V9p7KmOWdoC$XnRgV8mp0wOq!6+l;DT@
zWiyjm$Y4nv9XldtGigO;A!V5~$~HyD<y>TQre2-wkOfw@quj!2^<-RjFqwzU7CE2M
z0z?n!WYmi^*|LvOcJ$2w*0dbQts@%Nc*N<3tg?=^bQNl$Tt@!~JF@1k&{uG*2&C~F
GH}@Z&8-$Ml

delta 1258
zcmXw&ZA@EL9LCST_ukUAl+r<4+EPj>ZFw)q7%+!$+k1gHhKM2`)G=5W=S;RC_<;cF
z9Ahhkx#9r@=?4~Fd|M>sB#Vm1)EE~=mJIwL#weT7n3y;&X1+`k_1wd}AI|f8&i|a}
zoOACzA11i11ZRBPXw)P6jMX}it}Gf4TcL%VuxdD%CB1exP?>ZDH86{l*~{=78Fxi#
zA2m+mch>;KNLOx(MwlH!Lj@S}Y~)Pd8o));kuL-MLgsuyfOfLtcLIDteh!2I@<?%E
z=}w>n_Z1J)w9Vo!h?B=kN&)WU;nM4LQMt?mz?0{K0{~6<PC0>J$#ezHe3X3q5Doi;
z{8>54LmwHdaq@7KT-jd<u!6s-+b}&nWRJZ3f<`^mi--D`@Pf3DA4<7;26tE5u%rH%
zVdO!vt^YGxY{A8)6)=L6<*pR&#AQ19EDknw!YGc``tU7}osaO5b9lKS2<P$rh6RY>
zcf|mV;Z<<}F5m;9n=p>;VYgx8LF{~F9EZaP;35Wp0ACBgN9)~^7hb{L`8J%8(kQ(z
zT~@v177$(V67H2tGh)LBhYm*gos>Q;<rrR+EfB}o<zkp*+sSHd+x2Qfh~R|g>J;v6
ztb=J>Uu}ceXlupaH4eb*xU(q?GkC7a18<N|n~vBtg?fDVP#KhASJx<5$eUgCJ)a}1
zhxh5=Ym)l>2+&U+KmJaZb=~GtEQXx4Ekv7pY7=W!vT;kFiyRvn6{3K?6q<6x9zZwq
z!!}iK?Js!ThgS1JHLE^-djYo^+|%ym*1Q7cwSs_J_$pn_S9QHgcB=ZlImBk)E({q}
z|3h`iqv}ug<^omkYDBB5oA&IJd(mbUOW)iZ)t@Ti6cjiW;FN%%atVvy$_%^J;rI54
zc2)mK3+1W$SH({J-}Nkg+rtm~(YBAJZTroG)!cTqRI83mHdKUb7)V9DC<RecDr3Wv
zQqU~Qh)c>S{796IXs(yb8MKy*K?ZHPqL)ECFIpHp;Ru-+bQD!EeJWdKCrdsfh*hlT
z*@q>8K@aG~9xs>dF=U86IaG6Kzn43r{=fe}0tFKy$1jWmED~jWou^lEIc@{^m)x1W
z0<c8RV_pXyJTZ5JUYpKDOa}pS`>lWU&_Om9=CyExoW3@#fgg$WqauLciTLpc^tKUY
z>5>*Wa{BWbU9>YD^a@hjbwirMF{?$U3?+r34y9)*yBO+GrcE(0OhvmLGrlReVq`WA
zrDn$EiF(DvY&z1V$GNk^ikaEn$Y_Z_Ink<QFl$9-`;2AEq}Wv3tYjgZd*-F7lZu^{
w9VjbzUYLq1PG+-_*`l}@<{-L3H$y+tW+(xMZ0qy@cw@z?Wj+05d!<hEA4VvFfB*mh

diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc
index 6a5357ac2d5d527a7dc1e6416e5c3fc9817b54f1..d4e524aeb03844bf11a75b5ea5ccc0ae0614afc4 100644
GIT binary patch
delta 508
zcmWlUT}V@L9ESJ&&pDgU4U3g9h7^`Y`B67onKuy`(N#$)m33i3no&@cfkJgN2opr?
zFl=V9TH%GrseTxe7?EW|#jkLn5?+)@w6KqXTeZEn5AWB^P{zzqhCA$bJ8dcTbokB}
z_q%_;!)56s=QK`O^`jk%i&iqS2WqXdF_n$q+B?6+LxEKr%gdofZrRD&NWQfCShUkG
zjd>(Nq+u%s@Iu=2X;6$vVjHV4E3xhPVM&?_S<_gNYdaY*a7jriyA1Zp*>dVPb5lN?
z#G6T-EY&gSWm$S!c&zh7D(`UFz#}jHTu_wBrn^{*L*`T~UpZzD-lK`*W@#I5sF8RZ
zlZw;wv7H^7{Zf|Ab~S1)1j#Xr0;$g2PF6K;%lZJnls%Bf&1qaB<qznFkjw@70(~<4
zkP5{s`TK|)iilhfF$(jt{|N`vZ~S3AdTM1?FGqC@`PRp?4S2<T%d`{AY#e8nN;xpW
z{d7!plAkav<I^0@<h$uzU<Icbi_|D4<jiLx4vM94nRSkots=&I2Uo<1)2;JVLJ6wj
xxg?V5m9M4iAGt8rIiP|}Nj0iU@kx3$b;Lnj{y5bso!wHEt->yvWzDCe+JAa|xa0r;

delta 508
zcmWlWTS(J!9L4wf?Y~Vo|5#|k8d6v(WvpH-mAyoCqo)|OE@%%Kq>LhplBg(O7GZ*f
z{V|&rXhVGH@-M$h!cd~jtd-3!gC<eX9Y%IB+^X%o9?s!&IFr%5$!MM)(RHUKt5Y?e
zJ>DDoulwkx_MpK^z-$~lrwE#v@oK0yS3OaMoHlFTEGnYHtV@-Za6(Snam}M#T0KnJ
zxhcUiGH^$l7E=xb(!PWy#i*n!n1+{<T1p7sOLHZ&R=&!i6%5+Y#aqQn8$Q`pOXEV0
z%EKMF^J$QGyBTuxSuP)^SL3^^Z(+ZUVi^drSFu_aoWWFVG@iHe5x+5XmS(mZRc(y0
zU1r*tP}Iqzc9vP~`&<^w!Gw_vQ(`Rdq(DkLn6}a)b36H|>=$zOv7GId+KY5Uue=WP
z0j|sNC47pz^7jfy6p!U_4`Yy&HCI`eTN8><q-80)`q`pkmv7hTD`vhJ_lP^WWdviq
zV1ulE%K2PPVuBwqBC$A|@<|!)BwuieZHjG*XHx%`aYvS-@)L6$*_bcIsDlGyo1t5y
zPkPb>;I^bQIm)PX{i76O9fK-NTmmS+;+6DU)n*5O<hN7p)TkDJp^6k_ADi{4g!Mnn
CkGQA+

diff --git a/litellm/llms/replicate.py b/litellm/llms/replicate.py
index b0e9f55bb4..43ece59473 100644
--- a/litellm/llms/replicate.py
+++ b/litellm/llms/replicate.py
@@ -24,7 +24,6 @@ def start_prediction(version_id, input_data, api_token, logging_obj):
     initial_prediction_data = {
         "version": version_id,
         "input": input_data,
-        "max_new_tokens": 500,
     }
 
         ## LOGGING
@@ -109,17 +108,32 @@ def completion(
     litellm_params=None,
     logger_fn=None,
 ):
-    # Convert messages to prompt
-    prompt = ""
-    for message in messages:
-        prompt += message["content"]
-
     # Start a prediction and get the prediction URL
     version_id = model_to_version_id(model)
-    input_data = {
-        "prompt": prompt,
-        **optional_params
-    }
+    if "meta/llama-2-13b-chat" in model: 
+        system_prompt = ""
+        prompt = "" 
+        for message in messages: 
+            if message["role"] == "system":
+                system_prompt = message["content"]
+            else: 
+                prompt += message["content"]
+        input_data = {
+            "system_prompt": system_prompt,
+            "prompt": prompt,
+            **optional_params
+        }
+    else:
+        # Convert messages to prompt
+        prompt = ""
+        for message in messages:
+            prompt += message["content"]
+
+        input_data = {
+            "prompt": prompt,
+            **optional_params
+        }
+
 
     ## COMPLETION CALL
     ## Replicate Compeltion calls have 2 steps
diff --git a/litellm/utils.py b/litellm/utils.py
index e25dab254b..cd48ab7180 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -1016,7 +1016,7 @@ def get_optional_params(  # use the openai defaults
             optional_params["logit_bias"] = logit_bias
     elif custom_llm_provider == "replicate":
         ## check if unsupported param passed in 
-        supported_params = ["stream", "temperature", "max_tokens", "top_p", "stop"]
+        supported_params = ["stream", "temperature", "max_tokens", "top_p", "stop", "seed"]
         _check_valid_arg(supported_params=supported_params)
         
         if stream:
@@ -1025,6 +1025,8 @@ def get_optional_params(  # use the openai defaults
         if max_tokens:
             if "vicuna" in model or "flan" in model:
                 optional_params["max_length"] = max_tokens
+            elif "meta/codellama-13b" in model: 
+                optional_params["max_tokens"] = max_tokens
             else:
                 optional_params["max_new_tokens"] = max_tokens
         if temperature:
@@ -1289,7 +1291,6 @@ def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None):
     except Exception as e: 
         raise e
 
-
 def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]):
     api_key = (dynamic_api_key or litellm.api_key)
     # openai 
diff --git a/pyproject.toml b/pyproject.toml
index 8acac6eba1..8fc37b24a8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.813"
+version = "0.1.814"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"