From 4d4f8bfa5d338f03148750cfb59fa740054294fb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 13 Oct 2023 15:30:18 -0700 Subject: [PATCH] feat(proxy_server): adding model fallbacks and default model to toml --- litellm/__init__.py | 1 + litellm/__pycache__/__init__.cpython-311.pyc | Bin 13611 -> 13679 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 51989 -> 52044 bytes litellm/main.py | 8 ++++++-- litellm/proxy/proxy_server.py | 5 +++++ litellm/proxy/secrets_template.toml | 2 ++ 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 7900d04dfa..22af55ee73 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -43,6 +43,7 @@ _current_cost = 0 # private variable, used if max budget is set error_logs: Dict = {} add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt client_session: Optional[requests.Session] = None +model_fallbacks: Optional[List] = None ############################################# def get_model_cost_map(): diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 9469b8889171690d527950dc35e0268c0621e602..a76cace80fb00fb9c08bec596baec5ebb8d861af 100644 GIT binary patch delta 2666 zcmaJ@TTm276rEXiiO5qx5G_cw%mQkW5JbgSc!Tl~6j*2;>oDEGK=L;cL>`J7zf%09Qu#7JsZ`2}UrFtkR^>C5~8R{{0N%BQqXNTb4qW317Z!nPzPvRBqvTLWlBjqJQFEejT{ zsJsHd0tehbU=bEgL!}ERL{hqf8s3djm5IV|=?6mTWf9V0xb%ae^zsPlERb%2MeKZZ zx-AV=Lt2hfZv1L8IXHN#@|D*8h;OzTR5 zg6Zf;ASL5Ht(#OQ4u_jjX-ME`L;+VgfwGW5RYZaLa02Baf$E3?FTx2_gam3L3j7;R zpfV&-8$rM_krz&&DkM-BQJ^cFKy^@HBdQMw_}+ILEfd$_kzN(F$j9ucc+7egShJ)8 zhh%DlOdV%hB~u?{j&Y_L?E$FVy(mufuNI5wwaghl4V;&Sy4(xl%opdK&_|^0aS^qkS{2U>QOuD zfR4F%;6!`*%cWJ$whaQ<4m)5c?1D~yp5&*-&u*wgIF|rU%2*_0d-$3(S+ZA*`6T-I zB)TPI$cSRtk9DF0T~%N)9Kf(iv2g=qMq(>`JU`*O>e1@C_ zXQl1I=ZFvZqVt5))kZa(YbPVI`YWvILgMGFT=Lw}FQ;=`4OkRser1 z^Uj>ZkI578gxl%~CD*t5BJNLhgFlhDA;Qic4dZT2AMmbzpAc)Y74n|vul}4=t!~}T zP5(Nl({LH4(>a<{H&W}?HJtaBDPaZ4rCY=;d8*Ju{ELbVDt?$G6tl_XUD499k3COz zRPX1WT#G-^Y4%a1L8tC^nP}Lgl2`rlXb8DXCq$-ZV6RvF;Cv({{p$?KJx0$6t>9!e z(^B%)L#r&MKW1|iC;ajHpx1O0r`v;Enn9a6M_X8WX0GaBC7Bylz#N$$Zs-)D)**}u zZ5I)p>~dz=At4aV5LTZ6LqNX(BETVFKmZURbm%Do-qjDX@}m=;`eOnsGF-iM$7<#F zajQn(-CmRK5rh>Qy6LxJ(ab=%%0zGl`|jfwmSR7!nI0EmHp4{3>yH{h zBYxYEh6fSd$#nbHja?$zDd404O#ouE_B{5BeUpWbvDfxg_Qu|1q1RbWc9!o35AY}O zWqb297oVuxW9W2pjiktf4P7%4of2fMjMUwIh$4DZFfoB#we%rG`5qB9 zX4$7XslK~w!ETH-qS1STiIY?6Mv%Jo0XoMidSAf2fCnNXP$0NN76iR0U`hC_L^E9e zID>D;UaFaf+vTNs!n-Wsp@2sMRuuMAPMn24WiNA5tG?lsKhZ&x}0-0wQA)VvLy} z+U}wuE~w#>gk!35$SJwx512#BLsI1-+a(tdsr0XBVT^*4_o~15>#u+Peciu)^Waa{ ztSjSDM#fT2UPV9TldmpiWb607&O4G6$Q{>Ep+zHwGjePe64Zrum>SwiQb-!v)oaO) z(i%y}dBw74(dsmuKdxaL(OkQu!!t=zL$>gR_DJ15|IsbNo2BDktfPIh6>pGGB3r3! zWwMn!HB>>Y(&k;T5gV+Jtz^ga5z)TvZSmZi7HB|6$EjXdg$HSPafYr1cP_4u;^go8v}`+TzSN7Eva8X@uicS z)Ptq3+e{#O`nV!yu>XG;xL;=w)31!_*T(6`{AJ$D@1ESj-GmLQVg_|_22uY7^aH`4 z5;mxg85~I5Ao$Os`hoagXHCrDV8RBLV8NmW@%ptf{X3Q!W@xIt6~}=%u&g- zDdre3__)+X_=LoEQb8(5UX|G2_o&dc`5@1y2vj1X>U=hZN+UE;db1CJ8_r1 z?egxHw}E@GV=5EbRW4S!MDD3uS@uF-F2X4nVJN0gWt3r)_OKq*qieVfyQrfPp8XUv z>CSplFLNTN2&NSWR05@^*bo|0F{8t51dXWJO;58?G^%0`J;TP(n2NnLz|NwxDn3cy zXCI&sRLtpzY#fcN*hfEN=g>J7`{~E*JUXx90s0BMfG((bkm65S5Cv7|6#a}%pa~TZ z(Ti*nO{#d9USgNgWhRhdSI`x96hC=P+(pN0DtKfl!rTLUhGW3z_b|K1OWcOZ8zsk^ zMCE#0T{8@)4iYXU?S>h$dJW2W8?bNm7(U`B?ohHZGQdeF)n_;fGx`bV9WZVkq%swO zR)_dzeN_oqMYJJu0`vix02Bc1xC;O*ln(bo#OJne*p`y{IFuURrRjEZ+uX@b)p`L` zFtn6;FrjkqK0n_HgcX|(e<+!H90m;+uM%{s#j|&stu-I}FkdLCpA3hR z2Mszv_+}w;-&)-X*2KDwC{2I_F&nqk}>@K?t`vewPmPZH>-j8J=fW#!`h2$X;mLwp7&A&fM8g~b+ zv9XE?^-x1wt=OrFZRm{68jH3`#n_JZ5u=ToGKu5VR;yF(KcmJ}Gqv~L?Z3PK0wX&8 zXZGH6&%O7Y@0@$?x%ci5AC$fPpe*TxT%N$-@AY%GJ~r;xNqP-x^xHLmLQ$@0xT<}~ z>|VylkpY2kfE6j9(QCXjk(a|ybs7{0>mF0ZaK&DR50Lv9&O%9byvmhYizfwI$&5}E z!1x$G1x7r_qKJ8u#yV63KhCrxBay7clD+nbG5IDNvCgW3-6lJ=SuImw#M35=E*!C0)Z`|; z@0m;;Wg>@Rx`VymRCkUF3&S03SY+`LJ_J=o% zzX9A9`?vDmd5cB8IwUol;i`NE-pncLOO~wCS{PrEu5dL<4Ay~=sRvhPdARG<)b$ID z%)%~r=cphrOP3MuP@AaOt8nXa^eYVFvY4Y?(R>-)TBKU}e{y!ds4L9bZEX^x%;FF( zZDBqi4n8>A<(#t9ZFeO*?|;!<7!loOc;ZR5Jh-pD3cl=tY!&RVY9xzRAOeU8`9cXC z9ZXxRu;1UaiXo5HMT9(6xGbm;yHa6LMFq4T%E>@mVJfE-ZHJ3FC8!%>atkfpqHcz9 zMNfAeM+|ck{{o+{gztfkxfQ5{rjJK+E9HG+AKt1fU>tAo>$!XJI*;e6>Tznj&lT+} z#Y-|UTiar*XlKUwfrFD+Z@<2 zgSJv<_^z%HM`B#}VoWO~x+}$AcRtUFG&ZUl-YGS8ZI(JBBGm_R`^xzJVRM&)bxS_M z`cQtUB#qa?<$S|vIQlwKF3!DKB8?ZRYl+Cpd0%MOz!H!#3>>gGU~Dt?(oNw=8bmqR zj}!COwo&ZGYa5PpO9W1A1vC~I+QTJ>{J4_8OO%7xU}%9G7Rz2_;e;&M`avBu5QWhP8Cszc?f2nl#iN4?l;R2-c(MGGeS6q zpv6O-l4xuWYS7H*8%gIPV#=^v4-^@040nA`gzKgElputKCE`PX9@V2Lyni4#YkJ4= zD4ZkkFYqCJe-UZCbiv%iS@~~qtc#Ldy#)`+yw;ziBgkhTa!Ml0B{k1ifnD&k;zG?L z>fpU%Yv7KM9AOeHU!GMF4!%WHk0bs2^CCTX3X@2OdzX`J_4sl_Yq*4K1r;JjY}kXF=&skng>zxJP#`tJ}E-uz17}KN#F+0 zm1+KXA`Wo;3w*vT7%#J6+fS7_N{@-$c^b3u=^m63IGbG2^mNK8Cs}o8c`H69FO;it z=rOt9mp(7RZWM*n0Q*O|p)Y(_AB=F+BD>mG!#^y_nGZ4&A_X|Gk_6r~F7bn4=rnmE%N<;ClM9vZ&7G34UhP73dO7?!IC}@r5#lQI$vp=dy zm)C^m@lS)QwxjsBApq_%A~J$I5U)X{WYvQ`-tdRE zgFgd9bsgyU@JyXGkU^D&c;70db0Mz&Fq(zu>g~!VZ_@^bpAEym1J2iP!sxOaZeQSf ze4#lGUTjFwZ4b@h2SfB!FTGmFH=4g@L}%k_YP$+%8kLg#>Jp{Riz=b9L=vH)+}Xyz zB+go54DEJUI~p>=xA1Op3|@h~fyZ49@+y1^*N=SZ{GY|%d0H3A z*x+sI##$~nIoJVTdq5skI5W{(u*GRXffv{hoOx)F`teogMzjaN%2f1!hgHqH7F-b> zp8a97O}RRR2ETx{z5@P=Kd0r36eRd9ou*ji1IH~dqhUzu)|uvF2mATH!NEOq$$N%f zeS_UQdjiOi{DdR3}p6oq9i!jcXN!A zFf?$f?SkK$Hl!hWsnYC-OPrR;Mhi|Nm`RQxr@X(GeFG&^*pzFyswN$Rf*0r9gAds72p z%r@Jb=!n@?)=^A3ZRHL+cJ>CXLrpmyMNVqt>{_LRrJQxTmb5}jZCc(GO^MF(=0M-p zngqu-gKTyit8{GB;#$k%w!i3mw<1&tM-P{=wb=iU`(HSGA{t!<_gFi6*Z<6z9-&X* zt^0l2zv7jfi}8%?*wfE*jbs@=$A!%Z5BJ^#XO2`OCtN$y9^a0Ic*xBhQg#|)_faj| zi8-hIcOUh|MztU)zF!OaV;fKxxQ{jCzwb^TtJ^|F$L<{AN4P!@NB#!ku99L635Z5a zVZFl~-{<1qChRuMw#9Me-pRF+;ubQln-uFwv6k=%4z3GUPZqO1n7rn{W3n+eri*Y} z;E#`tjuNGTNg;)wVcs0;=K6>DkaDUoU~FQ9(mA$ueM`%lIobM_<~eqK%Z8SX98obB zj}37T4|%XT6q1GL4dU|POB6gmRn7Kec7|W}=o97*E^K2heu9h`k-N{{pw4{tt4&!Kdu# zL;vxoIE0S{-Ao1hUEKLGY?|4Ioc>cY{%HL^B55-bPYouQk~tP<2EK>$z@2B5Mh*|K z$+*YKkd34`feRbUjSzYtoH|pTLqd!Dgs|0|lgJD7NhNzA^{gEog8H+W%H^bss*35| zJ>1{h#~lXGSr>Z*^RM|YpS>KVzD{J`jf**%=Q|$m5^1X7mDv__qhI-)4541ooZGzY zUxZ&y3IPtB%UMr$1xI3-BXPu$C8N%y&fbWVuaPIH2(N+aY5ExQkT9zcpXfuXMRtm9{jAIlkA)b;6j7lRcqhunJ znm%crFbQ(nOkxsM<1J$iLNaYCm^j6_?2u1Lq0LlAt{&ewwnk9WW*U>Cn^aFE3mVcK z6|{`TG&wRcEa+%=I-@mDu@j?025st@6t!TWq>;e@O_a=J(u|Y66WfI>+RS5;HRGPK zJ%Wuk^9j1F34>s#%>pJVZM=7EyHH4*WlW4xD5oSgJ0@T~48N$3p}dX$PhYS^{TCrC BmEQmW delta 6656 zcmcH-3ve67b=R`w_#;`CtdC{OvaHCqEIYCzf8tMUpKQmU1d>30#14j7Cb8o<%K1zR zoT@N<3r=vmhM3{!&ZIGr3$bjk~wJQFMnyG_Sn(iVWun;#Et z0oxRWfX(1i$us)%&kO=Sdq3O@uLm3z5XBFv1Pa!}Cs?@^u=dOI@Y#M{BNiZ8%>^-$ zJjfBGgCO8CQP$IDw863xW}&AoE~rQETk41ZOi4rUKs$On&)Fa~JWJpYXgIq7 z1K{@@fLJ>IM}I*rpB1ohX(-?@C@>{zwAyo|a`XZvr7Eb+P9_4;UaQq4B{K+Q(?)7o zOO6JuSlF4w7oww9qYfLUlXwvfnbEp@hu14`0Wf3{;33f={uv&-B#!S;{!&ck5U_Q@GQMz{My6cHT@zd5%jDMjTV%d7qu#>R(zxnz zT<6kwt$OXqrMQz^jH(=VuQWy_!oIvVkSE(GKJo=2e=;X!)rqYC-|)J=+eZOp@$Kr6 zo+>fdSfk7e5ie6+#QIyOOX(?$EOen+tD%K2k2`)MsZ6X{v|TF|JXC^4oHp5Ebqkbn zhqYWT4;^*Z;hjLck%_inHxe)E(W`Kx68X+ zYDkShi4A!G8&M-uwpLgTlyU(9#!jgdV6@6)LWWYEEF(n|gH_y^K#95Lqvc2`(en9H zudF&Zph&4z765YP(jUHZ#43odt{dXqm@cy?rdc(2hah2ongM{u-OX6H<8H4_>OmZr z=ns2vGMWZIE6WbyuuJvY7^jz9>z=sPL|WHycM7{*L2KK0VULl^9KoPJlo4aJ<#I>+ z(5iB)>?#w>q>NuYWp`#=9zO#8ebN)yCvqgJr7<}nY@2xGudb@`I#!L#n0P$#T`b0Y zWPNN4yhE$t8rBqt7vh#cOclQH#}`>~KlSV)Yq>NP#H}vR8SQhlxLc9vH>5`vPp<4t ziY1nHa$Z48tnqxipWPl4R&L5eRD8)(Zb~2nUGO+@f3*4;lUlA%TZN-mYL?h9;0*uC zoI=+wXh-j>Nm%GpdRSnp}cd8ig z!!f$}3eRifZz6v(y^HFrN%6V8xnGlo!|Qu9i?~;?}N5_fWT%dHPK$+*^# zdvQrBj>GXKR?k5}#ytIPu4~0s_SShFQY+U9Qg9xO`|qZCfcp|OQI~E0-bUnHU7|*$ zg?m8|I#1QEttY84QNPp^zDmxE^`n?G(6H2V_$oO+YG_uAtrh;H@G>fCT<-bxRZ6*7 z??ipo7OcZ(jm@&ANxXvt3jERo{SV?dd+CP@Fw`aNgcAsV-jie>56&Au6|d-a|?QwA@%ojrg@vxf-M3K4&P3hmqptGkkDg0B4>aH3 z98THlwTARePbK)RqI)5E{=THmN}eC681q^iIFS#@?1U|CU7_NHt7@H9@u5|T;SURo z$-*DgmO07uyRn=?Ym0oz6j<+Wi>~!lrZK&G z#cWT5p6S&nW_vYCJP!}tdlQ7U=$-qjlnvP1`@`SfH(@pgS9-mnKl&=fu~Zj+JPM-qgZ*WKmpk$ss%bb%_z6xxhY)#^Y>HXXGndp(D*so zC$)Y1<^jKdaAXVn1<~a+YCqUmM>D;Et`CUohsb&b8I(jcDctPGWH6EGMgo0_UOm{M z%_Vaxf_f((WgW^-8zD$Cw1?{s?Nn($CG;(1=qD>C^y-UD+9(0Z@M>5YUYP{XptXlx z@Cy_=yhwRFMxQ`09=-(o(F-rR;pgFxUt%FNp;fQcD(}RL|3Krf^g>_w^ef?HIEJ1$ zVll87efnGcV`s=tHUj$!GhA+`C-zvi0F$PJ3vLsVj5-sX3>JF9|&yr@5C6=B_ z9v&R!*gYuqZ7X~#T>SRm6rolvOp5?Ao=n$76m+Trnfgcy1s4Eqb|jU8X&~DaNvB{2 z&{?LOleUPOPBkDSd!l{3Ig&}ITChMnp}0F3$)ZyoP#Y%Jj9(kk)2R_;Yv;canNZA$TUri zj&F~+=(Ln*=a{rc+;mz7G{%Yk@lBC(I;{dKeWaR#*lZ=W9%UXgswiqj*nP~N^gk2O BZzKQ! diff --git a/litellm/main.py b/litellm/main.py index aebb273ce8..656fca89c4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -229,7 +229,7 @@ def completion( litellm_logging_obj = kwargs.get('litellm_logging_obj', None) id = kwargs.get('id', None) metadata = kwargs.get('metadata', None) - fallbacks = kwargs.get('fallbacks', []) + fallbacks = kwargs.get('fallbacks', None) ######## end of unpacking kwargs ########### openai_params = ["functions", "function_call", "temperature", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "request_timeout", "api_base", "api_version", "api_key"] litellm_params = ["metadata", "acompletion", "caching", "return_async", "mock_response", "api_key", "api_version", "api_base", "force_timeout", "logger_fn", "verbose", "custom_llm_provider", "litellm_logging_obj", "litellm_call_id", "use_client", "id", "metadata", "fallbacks", "azure"] @@ -239,7 +239,11 @@ def completion( return mock_completion(model, messages, stream=stream, mock_response=mock_response) try: logging = litellm_logging_obj - if fallbacks != []: + fallbacks = ( + fallbacks + or litellm.model_fallbacks + ) + if fallbacks is not None: return completion_with_fallbacks(**args) if litellm.model_alias_map and model in litellm.model_alias_map: args["model_alias_map"] = litellm.model_alias_map diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9312522986..18f31133e8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -125,6 +125,11 @@ def load_config(): ## settings litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt", True) # by default add function to prompt if unsupported by provider litellm.drop_params = user_config["general"].get("drop_params", True) # by default drop params if unsupported by provider + litellm.model_fallbacks = user_config["general"].get("fallbacks", None) # fallback models in case initial completion call fails + default_model = user_config["general"].get("default_model", None) # route all requests to this model. + + if user_model is None: # `litellm --model `` > default_model. + user_model = default_model ## load model config - to set this run `litellm --config` model_config = None diff --git a/litellm/proxy/secrets_template.toml b/litellm/proxy/secrets_template.toml index 3297ce724a..ef22330b1b 100644 --- a/litellm/proxy/secrets_template.toml +++ b/litellm/proxy/secrets_template.toml @@ -11,6 +11,8 @@ [general] # add_function_to_prompt = True # e.g: Ollama doesn't support functions, so add it to the prompt instead # drop_params = True # drop any params not supported by the provider (e.g. Ollama) +# default_model = None # route all requests to this model +# fallbacks = ["gpt-3.5-turbo", "gpt-4"] # models you want to fallback to in case completion call fails (remember: add relevant keys) [model."ollama/llama2"] # run via `litellm --model ollama/llama2` # max_tokens = "" # set max tokens for the model