From ea8d0bb7d58d461cf984eaa81e5b52a0bee80c6a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Sep 2025 21:37:56 -0700 Subject: [PATCH] fix: re-add scheduled rotations --- .circleci/config.yml | 3 +- docs/my-website/docs/proxy/config_settings.md | 2 + ...tellm_proxy_extras-0.2.20-py3-none-any.whl | Bin 0 -> 30470 bytes .../dist/litellm_proxy_extras-0.2.20.tar.gz | Bin 0 -> 15576 bytes .../litellm_proxy_extras/schema.prisma | 4 + litellm-proxy-extras/pyproject.toml | 4 +- litellm/constants.py | 4 + ...odel_prices_and_context_window_backup.json | 4 +- litellm/proxy/_types.py | 31 ++- .../common_utils/key_rotation_manager.py | 146 +++++++++++++ .../key_management_endpoints.py | 17 +- litellm/proxy/proxy_config.yaml | 3 +- litellm/proxy/proxy_server.py | 40 +++- litellm/proxy/schema.prisma | 4 + poetry.lock | 10 +- pyproject.toml | 2 +- requirements.txt | 2 +- schema.prisma | 4 + .../common_utils/test_key_rotation_manager.py | 194 ++++++++++++++++++ .../test_key_management_endpoints.py | 71 +++++++ 20 files changed, 528 insertions(+), 17 deletions(-) create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.2.20-py3-none-any.whl create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.2.20.tar.gz create mode 100644 litellm/proxy/common_utils/key_rotation_manager.py create mode 100644 tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py diff --git a/.circleci/config.yml b/.circleci/config.yml index ef6445ca0c..ba42ccbc36 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,8 @@ jobs: python -m pip install types-requests types-setuptools types-redis types-PyYAML if ! python -m mypy . \ --config-file mypy.ini \ - --ignore-missing-imports; then + --ignore-missing-imports \ + --no-incremental; then echo "mypy detected errors" exit 1 fi diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index f70701886b..ad3afd59a0 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -614,6 +614,8 @@ router_settings: | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval. +| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false. +| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours). | LITELLM_LICENSE | License key for LiteLLM usage | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM | LITELLM_LOG | Enable detailed logging for LiteLLM diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.2.20-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.2.20-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..0a94ef6ff621531d9f9d3d4e398315f607384e97 GIT binary patch literal 30470 zcmb5W1yJ1IvMr3eTX1)GC%C)2ySuwP!CeyE-QC?GIKd^jYeL>6|8vj#l6&s=a;Iuf z6;=F->F%|=*IIis3euopXh1+fP=G}w0eAxf`S}KT5dxNtrHiSJjjg_elf8$hzNv?c zlc6(%zP_cMrHj5ky@Mwxkiu`zuU8j@&jFr`1pfbf-p1bC+|te*@Vu(Lbag)?+{O`_ z8nROTM@WzP_5APHn5BmJq2Gs)ZHdOLmB|T9$yCN zLbAY!;y_zC&{i5$N_Ej8CPEKvg;*=A93_KB`FveMUl6^iG%HJx-vz-6L#K5eTwY#? z{U%Nj;VEO+O0{=*o0%&4u$*IBS^uup@YQAUeq+k6i{k|N_jE)^LsYIn z00D`!0s)EsPwBL^Gs5E{!IKF$P`rkai;eu&w0R@eIJPAEtB+b@p@;0NMZGDC zfBMTehEx@qv5(6aG#>)zX)p%joB`n`~6t z6%+yFIN0kaCJ^O^ceRlhcWYX$hxb#*%MFRcGoP`b2AaZRB3(`CZI>|_@J2G*^khG4 z8s3vGpR+^fj(yS+>?fkYoH2)8X37{Zl-nkIjGyXsEM6{lw)UyFi&dN;nwFdP`tz7Lom(jsMwFkqN-M`C`YitP$U7_y%17}RF7RDV_KxJkt{SjPz> z$g6J9*pt-`$wT41QEUR;+p}%`ed(4JRL495A zRvkgX6*$q-HOJgGMDZEjFfim&Ubr|RbpM3iztwu;-S3&6sS8D{H<9eCD5mp%1x7=Y zSKXOr9Sk|DBk*0H1$g9c#bdy6j<^FQxv8ZIq9AbWiL<;6Qix|H%zYc%G4t9DnQ(WC@V?z6X!G8w6^RX3nP5|QZM!h>;<<5!ghlGwSbtZ+~>{6vD* z`|n&4y2BllD58Iu2##yCAW&C*FxqbR!kaARgu4>1jGXosf%B0;8mliK>?<@Jc&FBZ z2Q*^cIQJFz89Rk*M(!7;n_puiR!p%LX{}K>~;Mcu7{vxl-UQ zY>t!x1ZlNw# zMPZVFX^KE@=taFq>57%y>{E__f+rhzmWsjMfwd?(4R-o=cG6P{~P@Odk<6so)y>-6g~b!IA_S9VBKhJ>>J0TXNK7wOub$bl<+D zE2656xA_B^6sSt|ixZH8H%H7;FkTDOZcP@b7EZ=4Zz*OA}K<*5zO(I9BU0(XK#2B z9L%0%P1>?2U-9t+YQj7-^9~k|=sI5@d|gj8``E~9o$>dQ`}r!TP9grVRSnIMxeGf6qSw-1P8<+ch6|q@bp2OOcPZ=*LrjxC7CWo zPNmYujO1%S^{0DCNiE?}nL!Am!uqB!wrH)M8eCdA3MshoD0odnq#W|pkwe3`6XIJl zsaK`86D@bDLeA$G1+^BBA3I^Kw4N_0O_cGxU6t}lTP8@WqKV3~&*#fN@mjp+jgR4H zVAubE5Z7PQu?d_Klhbfs@vy^ZYNZ9KRHu~_YMfMQX*LSKbjPhHQ2*l(Cc)OQO=ApT z5m{hBK#KpH(aFKY$jGd3XkwyoVrXgOsqgA+>ZI>tXk_!p7|l`Ewhm^*^nFwZ%cEUk zTG`9ZHY1_6dPmYI-$!RC(FN92@cl7*m~*%>2?qGt?x-WXVH2bWS(C!zK@VzzVzvND z;SyTdV6PJuGn{txW;d}pU$U^Y6&@G1bY}W9L#Q`QzD5_CvHH3^bs`;CTYu)G3A4Gl zV}^Sw+q)t3cds@lNsGV*-}e^XT*spFP3p^foai(4{Y+_1N7`Q~%aRKou{yvD1wwB!J(s*?@x zu7KA2-KejdWhk}*ei0)T>d+!}BZ}$99z4eIq)LLsRGkpZX)CHw@{K=kX^dG4`%{a{ zIY_v129_&iQfZ@#y6rNa>gAio23wTdk?R`b75wjHh>^_3#1c?RJHRsh`-*ZfaWXP7 zGybZmp@XHqlc}Stsk4i-zKi|;qr&Kubuj&mND$k6^4CdSvt0|8M#$Av$ENhi_E2DD zlH5-ZCyzi|i~&;1)L#`Bk)nG9j|cHnoz)LbovG5~ZL}^sx^pr1ai?iU$hK#S`RrA< z!kfR)RttXnCf!dBm4G#^Pg!q}9rFzGdz?C|%u(Bb_>uwWmH+!VxfnTFIT_jX?M&VE zjh*c6^sVfT{u6Fp;{~ll7?DCRpV5WssgKJ>a_3zzY?3Mm>lTwNrVO<^#<3BE^~j(F^>nIo_t5FiXrzP+JOK&piP8P4-7|?R0&PJJ??_5|V2@dZbYOO# zBtBdZ8@6WWpdRO;1YC&lg{lb06Z-@3IFT7rlz+&x^I4|wOyE^c@7X{_veb{qg_W(% zDeHRM@o2DIM+$1ute6UG3-c{nWrCpGwzlEK5c=Kl{Uur&>p2By)hf8laGs^)?^OF= zl1)VIjt@_#?hgDo(b$*!m#(7N2G?0g&wM8Je@~|UswI}?2~KvOU^`M)Uor%4!^oByX=$Hx3ZkietQ zP{L|Z$AV=()H2!~bK08TC#0yu#J--EYPs(FYrO@aV&aT&ag^Y}H;5SV8v{!!p~VL~ zjd`B({a59sF%WLZk~kxr?dTj12Iag1RncgzR)~kRb5!myq{!Z97BQoh%Sro6S6oro z>aR6@ed&n5Cm~Y}hxrK*ydyxGDgSK}Sebq$fraVU&apMLGc*UpZ3c+`KSi38z+?~D zE1~W`qLNBPh?T|3QDgRZITza`7&Req_Sr*9WyxMHvminPsS~uM^?dTSyuH#$tj0IY$@dXf*FY5>tk+H5=6WQUT=fSH{_0x>N6x2 z8K0`<9C|1RZ;8WE#Yn=JepoIr71DogC3Ab&Luu1En*OCP^OEqTjEej^JKn@XdvRJn zyjhw|PYOYZ?%7S~`IE{5&ViMY>|Bj#O!!lUx5Ki3n?LC9i7prr@U#IW*aNUs{yx#n zjGRmyY%KaNriQlqwx+g5fQjQ^>SSx_?EFjC{TrV@zo#e})pl*z+ zt{ln;UX(oBPYV(&mH1llxy46+Iq;!abjem-nD)KE@g#n#vhpEQRQ2-+pdp3f$nT+x zH~UhVO=YD%EhQ>feZ8;5*u7+CaCj3!g{1`Znv~x9@nh^y7;$3ewE6*}>IW>xzmJZM ziH(V!mE#wTj17$~OaYt6&_v(e3E&L?ny|I0o%6pMv9p6IV21(roSn0w@jr_8x6orJ z>k#@0k$`4%YFZiB2REU?W5-g+y(YoFk|m<7Dd2wdKE2p}4On*l=(H!blb}gS>I|wR z&PvF{CU(|^K-IH3e&6kYMMhb*!}{@*&SwmBH_5jWS1U7G-qe$A>o9VHddF%^r+EPZ zY>6m{-TWZlpa-?5;2i6R9DZ{}=p%6_5%pCN5_7U&;8Vb^I#& zf8rM^Klufk-#`39@h88S5=&2`7qiFtwQpi&$XWK$B%9XGAIfRqmz-&AnSxfoO!^kxC|S$aDaXeN!q1y^}B zG^S8lj=XM|PcdT_ls}fIA?W%pD7sr+RJlCh3qD5^C39vm7G1?Ii(%)>LE3O7nhF|Z z?lQLke&7sJUPUmQPrXohz-I;duo4|o(f9l%YM|P$hCH+IVeByRm$9T9Remv_b_w8X zhG;{`<2-dQv+go0yL=R$KxkUE+12is%TlK$9)e6Ua5Mn;bD{zB}w`JcCD#0X$Vm^QsDX zX6tsdo|wtY&&CMR!?^kc=iuFc6nqcgT6zdj@K%5yqwx3WXJ=w!WBC{Ta54PXl=#Ix zjDELeid1Cn3mB2QZfPpBQ6ej;&mHCIJ=87WO(GN4k`8t)F;Z&48?1i##>%dVkwHsx z5zS0``{ib|=#l79T4Z=o_Vy(wfHC<5A=txZe32fvSv0O0+yrkoyxl}TZt>e+z+u@iaa%iFKje@Oxe7bOO4X%n0(r79u zxsTGxVtskZ78&o`cpxg=hY-5$W)obsGef!MoKgJaXPlpXGJ9zcQ{aMw8o=}w7_|mG zX`|T+YB)Ao&1$Bc2r`#C>m0<@UAOoaH+hPA&VvUw_5ZN%(y|Vzc>z%2 z1}vTb4P@AuSXj99O`Pl<0JGLy-__0%z*+5nQjnkQ)zZn-*3{1Bcla!PR&}5nXSkY_ zmaN)#g1$jixVm$8j8#y$8KafFzrK!jmzh-x1D4hQ=2LUBF|q#B*$i!L?A?E|BzspEQ)m7EB3*8#PL^ht#=p$Ue>(r~ zF_4Y^8g1aSfYCPA{jRiPZt$}Tk&Gx+kq90UCUyjJ4a2T>cQxoivFa{c1rY`TVWTPH zL>bj>ZFXa^c>7T5IQ2nU1vppuX2Qr$00E9OJwX>kQ)gB;?O>j1=FYcVF$@JLmyE;i zA3uiE8Gaw#fvG*rKZ#;AU`hXV)XZ#5oXkwjT)+CcGoV0#d9810@_Sgi6Tes>&@7+6 zqsdpJo?a?8$9QYl2N*lSuS9|Kiyf3vPnWwV>-xIES+KA<3@kX|XmJA7eLxt>hJ)+~ zGz_SR6TYx5}_SX@zFfwtmu>LE;|6+k|rp7MzPWsM(H&f@|gAGlX zvKnAS3c2)(5~aY&RQ|w6RSjcD#xv?) zYXNHY`h|E5%{(*M%(^cShCl(zvhe&IvR&P#pHa2WQO^MeI#9!x-%o7JYa()Q@JAYw3h*O(dwA_k%jUSZ_ zgKooNPt0kZ^D=gcuXmmByf+kTu<)8yXbg>9$`s+7$8YYOj<&peM6TJ#wTNGu-<4rd z(nL$qq`rH3Se<%`)JJ2dPLr|S3AHDBz6Yn0B!GH`_&v)tG^>g~v-|^)<-Z;iY>XUC z>`ZL`;sv(GKes)=aQIJSB1cj7r|r=FN~71QVklH+la^CY3#!?OLQ+{0e*&8%iIeCu z>(6hl%^~fsNtkwY`dtd;BSw>WSqmS83L`O=#LXa9NayXSXSI`-ng#z{Ts~4hP}sY6 zWMy5PAQO9(sQWAH=}WoB3!ZbiMJ&rXwq};@^-Hs-zK0f)&d)sN4@>%USCe`YQ)IN+ z14Ei;aO_qVqq5f=cVC&i2rn8||4coP50GEuzeVwX<@X;FYGG<<<6@x?V4?tRXJ%13>GDSjiBNL`4a8_U*!`BxXLw!@eJDU{P8oSZN#zL`0 z<zkQq2{8wD4bb4#P4&@~Mj4^*6lOJGR6SM@lVQXViLfj`M&~0-`3prk7=`+JZIIzv&1sQDCb@W-=V$hgsG=3tL)v9L> zl7QTZliUb~BwOgnslIxd?~^Y5z=l=BwAhQoUR_80Nrj4Td9|ABBqIu^&+gVRSRfJ1 zKH6Thq)t$9yr}WJHjhLIlA09s{WfS{2~e0WAY4YN)9fLiVhEt(D(PsK)^-2iGzts zA3Y+hMIzYVz59bXq#xBB4g)?o3qTX;|2+h`m;kznmF=g~s&8az4mcfl(KiE(mw)up zzeM5Be)vZmpA)+P`w}hGJ$E-uGq<-4iis3GHn}fy(P?fNBqg=4C2f13lx#TfyWb<} z5sx5{h~^n4KlL4y=`}tkv?bj86b!0OG~~3C;U}tvmE`xlc52$0_DWZ-%S&q*&0Pk@ zKBGnu$cA7mhVO*aE0dDxI?YAkLWF46hvD~jV?@I{X-x{eM<$UmK=}^a`M0$YR+sT( zb0pIBpjeAfQb;u5eT|<%w7m2fRq<|KCNBJ{x>{xiEVB_VoPsyE+Omoany)@|bo}8U zL=%}k{H(nhVEr{^WMg7vVr6DyW&MMO05WM}>Y@Kn2LCe-=%r)<^za|jh~VT3=cxJz zZTvS9;uz%U=%*)VqGx1)^I^!q{rPFjlUb9IPLf%x+FJiWLcRuy{3#%W7eGV$Yxf;M z9|2)71Ca33eK&VCbTV-QxNQI^|9AJ@H$m{{7z_irm;_u@iwki)MeI0Vjm~x{8cX8o zzQ&7EEcHKJ9*MxxH3aG;hCj;YGDs4hcb@aDwQNh4a|t7iE+vAds9qLLPTsD%RX3<| z#-fEr^wu7a_}Ed413%*n9t_YI>1?9JR(<~=gQ+D{Dr?Q7okwXCty7g2Tz>e(Cb+Qq z`UGrl!;cmb^KP_W_YO&)*?G$+~)0r<7`Eqx}_{hB?Tx8fH<3QYYDXP5y_l zVTtJQ9kAXLaK5!7QzQYjq}g@8>&jcFx0v0Sg zFNBN|#i&VEBZV+8W0?#;`q&pOfvd#H>s_ZP`Kd77f35dA-Mxz1D1$8YNkcj%(sYbO zsf}&+E1VCn0MkwYrf|~N%4L>&>qK38hGAZ<)YG7naGdY>0}*>+;Q7y4T4-A|oTvk;B=KSc0w`_0j)jm7p;>k(D;D>k& zDssBub5m@Li87eMDbvf9!VGW~z{=x9(zH5vRC^=iIfj9Zub*K&cTm5E(Ly0-^|Is! zIM;R?yZV`7Ru--iu*6tip%_Dzei*6RP*;TJqs^~^++Gy%fn-VvZdH`JkI4-EHWH_D zacMKM<=ZMNoP+p>E&XoJ8{ZzVKcN3t`vY*X1F*>%0gmiXeWh>bYGb1h5Q3I2p8rjA z{6&xdO%N(71(Z7e)8!SYC1uo=9DYg2TSUbx*!%0c!`*qZrP;+nowH&PvRxMCQ4X#~ z&L*geu_6KJO7cH%@wBgoM>qhY^8zdr!1^!!_uDO?e>;2sGTqsj0W^e-mE|8Q@|(i= z$%Fp|*#FB){ul;-5%8n@xRfkCv;^Jw6z$JW21=v)$8=-h%cQbg3^Pczm{}s+lTek`E#7n?Z)H(B&Ca>|C_u3G&M6DC-Xl@_>Haqhe=SR zJn>U-cD~Yt$Rm&{9M@5dC~7y=aL2$8M%HzYbVHja0S1Fl+tpY*mv8aw z@agZSxvq7)Me;l%vvF!-D(9k2WxakR5Qphr^7os0F!lO3CmXJ&TQ=M zolWijJ%fIoC0RM!+x_k!lseVyT5s z_VO0gNGNc%Z$CL!H?g{+FHxgG8Z?w{NEy`=6nHzjYf$zKU@a?v)o{JT4g!ta42#hwpUL_4; z(2l-;w)^JR<#kX4v6#I5eB?=;pzNOC%wJPwKysOxSOD*T@cjSx zQ2(3mC_5=FBQJaXt4RV^DF4wKe-i(BBeGDW3-o34p@K3sG1XzDfTRDD; z764xUQRP3yr=^{ly}q%%jjJsHCI6Eg|C?A`5uoJgsDsND|A=B^Kj>+)v(bkPyvD$v zp91`yI>(4Y9uoNdiI(eskBY9W7hDv8{`ZhTK$?Ka{%Vf>jEaSkk?|i-@MjcuhPI~8 z4u-}6L)69I-sacOze9c~dBzHGCO>@nUUal9^d8!){zP!p%9`Yiu8=!|l}T{3{`Dc& zzg-%!IICWiAp_~#K^6nc-YexvvfLVsoL;gy1EplaQvjnS1fj_c@s|jegZr$oW+TuV z%^^qw9gZ{5X_;~uFq#kFKa!#fF>jV#1Mz$2`s3iV>THu6$(nZc#_Hb9PMsA*;;sZ@ zek}N=8}JSZ`zQiphE&fHynrxDAw9t;FaLU=ezEcZ#cN8MF_Ma`m`{xL{p@DH&RcI1 zTud?Td`x~L;h^YnOCZi^YkF5s+*hyslP{|MF^D{j2TaQ<=_A0g=bN>|tf6lGl+w#5 z2Zp*+d*zA5vU<__V4GiZ)m$Nh-Jw1dI_er2bMnVR`w{bJ!L)zf;A41|acFv)hjo8k z@s(LUpi8%n?M9FgFQ}RFvoH9;*l>P>G3k%_{SUX3g4FZ27f>@_K>eitx^{pwZ$R(* z*Kv)ri;1QEPaOVUtWbpsfb2x-yhYE_YaK=0E(5`z6oO6`5rzs}LxMwG0C~69Q<2~h zkMzjN)4;=$3f_oSxGTCU5ls9@ynPpdk5ma6d3@B3I;wrn_2b4z?TFwf5 zk>i;ec#3BkglmB|C!M+a@NNsMhkCBXc22ja|6a=tHBfmnXuxUShj)st4*jtcNysWk zy`11OI%PB4j66w^4QwMK6p0Z*FsPLS?$qB$?Mor1P}!Q+Y5OI+*2-O->e%Zm?C*^T zKLV>s50D>Iz%uyzJaGaxE(g;;g7e?o!qCak_LuSS8+-dbdw9_uFaeBkK#zW+#~1H7 zRnLjbg4EO_u|*;ft$q+xK(=g}Vqr$&@=o=Se{8|?Q2q!W>un;!!NqO4=ekd(_euWd zb{ShnRd9888~bTC5h202ooe>lqV%ZtL(LHPTP@IY5XOaQPM$UILa0P@k)J@Eb8>lc;M+twVyE^HQmO50`0XlB=IG zwuc_Csq!=-JPE!bjrKGK9zcK-#jj9F&rRgWN?)glaS?LX??=)=28*E=*QiLMvT`w_>i!NsmN%Mq>i0)`r@@<8hBp_%o);$(ly&q7|;{;IPmQSDBuHGJg0kaOmQ)~k|Mo7Ni95a0N-ZthzA z7q-FFCAf(Y9R+A)JI=uGcEbuv_+}xc+wz(SX0ezezz^p=w6(>o!3(4gsSB>4BiBD@ z71^iMgROV-roPh{VvZbnI5&6_O`q%a&&z>5f&C8Tb|lN`VnE)n0PC*>6&K**8z&13 z+rJJH0WOfS{!erJ=MmMvW~>p~aaJevi zH8X_|XQSDd@f_1wP4V5q#}kMPh<*nk{i0u~Ek{l9Sf zZ^#0usj)S^i@mK44dpn)gp4GeQmv{2^(X_qB!d(k{iNc5J0+FA6oVf@0s=|^$UWl! z=}$QW&KPVB=^dOboox-zb!_c7T9MzfzoUortO(jt2tIk;vIx*t^}Vb&!zLxBW# z>{h!jNpW|rI3hl7@i6$Lly177?QaNUgimB*BaH1#4rJ_DD6ftepPgi&H1H~m()i^- zDSkdsO)b~*p=d6zw|v)RYUqj@$&Nf_<6V`cY1r*-+SDw1Jp}s zxvBP20o-Y&<~W~UyZWY$VGV`+;4QswsTwb`y=lduWb@>j*ks@bS`keK@(E^DuEv&= z{e9K`#UrOo8f-%;e+o_N={0T|j9n}r-_-K9vm3XP(|VVqYvYgFVgYsUrFT3%SOiIEwWvwkl7YxabQ4wh;^H zE+4sGU!AH`2JL4j7?*dNrq8*;uQcnE*=@=s<82<|J~a&w#55sj5_DcDF^>14ZCvg> zX|spVFshglu&jsj9DL3`iWO*+TuGjLHcn%z@>t`*q*N;fStcMO(QllkyHIKHg3bw7^z2DR0|$uT3!xlTKebtFaoXT4*$So>uQ-F{ ztX!_TA(9K0^@+m9YPi{Epnpt~=2Y`ZXRX9~)97;14lw+&Y%xmBy*LZH0k#n;ypjfv zCqhMIq8%lw>xOD zV5-JS6oY*l$wq!mNamVhjTg~>3ju`>Eoc|Ig?y)vRTVQrvxht}K5OFY03v_uDbz(qZp$TXvyCpgH9b45d>)YSK13DE=V$DdIHj_XfTMK|D96^zWM-O# z>w>MmfGSL}$vmx>8z5T!d>wC9y2(lcrm}oHSJ1Id!lWXdao`IaLwsZw@ZF&oy;xBX zRAy@bOo*(Z-a;_olxY9GA?=`Dcq`;1@K%dK3rb%k8|J6F(~eJiKb-X`ITL#|FT0g5 zD_h64^mOxwta$L(%_%e;v0h#o(jT5+OqE%gxvI^%beeh3p(wR9(Poxc zuCKVzDsaRSYX?a@5L?<)A22a3e)njCY;S>A4sLt(=#INQ1TNj6?A~8eHTSdX_m9iW^F$xoosTFzwJ9@WVyA& zmrXAN40GGe6mJKanOQwBP%M}>9r_}!c0SjB>-eb{O>}iR?uyW%)LYQ49V*A^rI^5B zOVVrJT3cqUS*4eYshHTI{n|!LunR1999ZaYYVI76-3S`2P~NoFB2*v*c-Px+R5BUn z&U^L~I-j@O&|%_Qn$eWW;>o@R>e%m?mdmRa*RkGB4QIIYOdE%RZaoXM_8K}TC7Fa*FfW;m$L6Q?n8$3o$x^|Xk0IA8z*5KrLn|Q}TOl~< z35u3^B?FoMx|CP+Rt`+eW@|dhy1B&Fge81=O*wXIjoSpH8$3#*>TO}qO^gJ+WPHJR zO+71H=-U2%nRb2r{@zx&Cfw$QONfCbO-<8)TEGc4X+1$n0pBp-;hIZ`Hy}S?0_lc8IG3Ah zRa8dzSB{1k0egbwu|1@ELZc2Zwyng;ntBt+ALpapNSSesnU*nl-9dGj0c9h}20}eh zICg&H9`B+bnaV{2*Y&ZiL45j-(>D3~8P(@bxd`n`TR0g~A@YV?3SzrQ1f)k^nRSvO zX>o^;nr2X{ep}U9tb5oU1sP{NXJPRi0+@eLoq4S-dkhqmLdTtj(bcSF{>tha@!&T_ z_IHF00}IA(bdN))^+ias(`yI<*S=~CJwH@9UU#AHP}E{XO=_>#_f_qJF>mcOKf1-V zfBIy&GE*Ab2o&1}tNYbir}2A(iBPD`L^-_VUh;OcdK--KE+-jhx$AmIZLK?|&(0E^ zVy;N~HWqm=L?H?BCYd-to{o=9aI_c9B5DBH@=B=S6Gk zB7L4(P3IN(_^y2Ym2=*Co(=c!q3`}+4F&pijUP<0G8ziQEJb4i(nSMRxEQvNbiV4c z?O-KnXLyBni(cv|OJ?q%(d^LwL;*sV)2Lm7D7(>Y^?oMTRJ8p4xlgUdEt3U%KAiR@ z^;yo6IUc8Y`o+qAFP3~cA7?yn%Tn%Ibjr#CMe&T-6RO_Fm#@Q2xt&xV+9S|NQ-)-_ zao5~_k-#bUi(YIIU+DGqI#ke)kc?i+-cP9vl@z(M4#z=?udl!zW7273gGH8UbulW3 zB1lj@te784VJagB1!gp}$dMaGn?Q?PIP}%k$&H2irKW9uMU@HKP`acd%czALN5JIO zi{wDlRPlhKVmxNG-XO5pn|m3pRmoU_&@q@h%WfEmt>)JC-@EpS5wr*P!;><@QAqD> zsLgzq-<4^4p$jm3w;bXJ@OGFG)hl24yV57L(??-!+(lUHh$Ijp;Hi zp%)H%7qU(Zd>;ms3?-!Mmx~ie{coi=H_;j%;pA?2jBS@@OG_<))weZcO{_8qAPDpZ%0Z6Sn3bT zk|*5el#^J7d13Ue|gvVE0T$ZLfczcjdndnS& z*mtRuL}p5ubi?9ppZrv^4!E2liU)CxJf|r*7w&kFY+)vhn3&KbA-v^@%girSGa=ts znb!p~F_M)(iW9_SdZCQ5XEDiWf_2Rebh%mN%Rgy=pmcywZeTuX&-6N1$utHRG6!=* zn?EGLK+hm?2n38;e5X!$vZ#JPh%+1cBh~nUMoLW0!dmj1=-N3Z0qSX@urlA-d}XM^ zK(fapzx8BN6`R8zBr>XVFapLYEvYMymjvxeWn}b}-iAvd z?%_!2IyFWJ6z2xVx!2$ zWUNZ3cCuH{RB30JMjpPN>Y^$@fLUlYx(jWlXK}0{q|Ks6s+*70e0$h$u~B5SGbXWYT*`a5ll5DU7vWj^v&0I*d3g$XsUO1o|LAX z9@A8pp8Gu0%AUm_c zMbH|sgBtTDEJ;qPDFt#Khq6E%f-ZT-g(AxfPl-HabOY4_wncOT3qw#3Z9Mp1Nek?`e#D)0!I3EY}-%Nms_`KTVml_n!n)+`iv~;HlbM4riGDutmUk7-d7kY+^J>?iT-y{)QAg3iD=H(IZ=Ktb>)n z)I6Y)nv%n!dhB@A=b?sed%BmCx0LT+({?dK>)z4F5X%hBtpcs;B5l9gYT2pxWR&`T z(x*EyRHyXLyLM)K%G2EwJr)*rBb7Bnn4_n6s;l1%!FdR0vFA;+6vX~mX+hF}Viqa{ z{Hz+(EiS>*{OP-z!4<^S=@sVDFqto^x~;Y<0|SN@Ia-OgT_x%`C!ynNqcJ zkh=O9hP4cS2mZBj(9J0qEkuI&k$@&wU>UI@mRmGn=~Vpoa#yhbB} z@;8pp5NvgO_}V?@Gs>{J&o$by@he7Dwba%=2R|V z>ySDxgs4T{iUQM34qV+gzf~N>g`>8ex4%)u;#z)t*S^b-a{zDP&wbKuGM>*xCeKTZ z;Wno}5yG>ztFhqs1IShk+<@CB@^iTL&f|(6VSFn%cumgc+hZ@ep6%>Mzp2D4LXH<* zxEIl!UhT^$SF}4JoqX$?AmaD*9R*UAV1-uxD1_iFqRzFeDdjVhkW*2Ku(PT~nDM>YnhGY(xBf^#N_K29D!o_4j z6(c++#ZhaBaa$1L<#Z4&Q0`|N{~cl^$<|_{6!9Bq`ayjq;8VFL?yCZ@BamT3aJulg zJ=QJ%kG1GD)I%~N^gT66WUvljvw?uIqq1$?GLZ?W?0Uo-vO*44DUR;;Z=UY%Ci!`u zcXp3&_Dcx9BT~GZHYdf1uiXb}9uRYkfS;K?rM?yB89n5&lM>d z&fO*5y6uc|I0W&Y*+ALB7;Uy1OCd-1RE5Qvo^8uJ2k=j?ut>yXduoTI$b?6&9PHCr zo%zdlx{hC zyLEyFX%OAT7^Sj^Zo{g$#uHwhw7-3OIVaPo@%4i#=C>@ir&~xTJufV@YwEy00345n zC^5{^GvWh}l0q3Y%(Hn3uZmRy6k}sMY|D9uE+o7X`naa|g-c%@|Hw3Eei`^Py2Pwo z>BIm!c(!mG9`UHF0yd7pus2yepn1UW7M?(6pyY()34pE0hG$~P3bNUHoZSm7GdC~U zx)|EMY7C^%B7Y{`RD2z@JUG~%QGf5|t~D3*y~4B^?@^TkeyBs)^VHh%7+82Ro|=yW z58fW)bN;n-YNo*J&hgFh=G=(!kE3X1aU7zRV%R81{HN>~=3wQUYx@!~!&DujpsGNE z4cK=2!LW~V3N9FiWAU}dXlB#MQl8uCN}qa zisP87W371A%^0EPT=>dh+= z7*0zS654cf%VL<5z{RdOz@nLnSWmX>N0hombq(tUqLf zpRL81OY}-Apjir$^e53KV~$z$QKr#1sMXS~g@&A$7s<{p*VlZatF7nl!t&|TP}x9m zEmbg^m*XGkR=ohWWS#k{D*?+Fd_|95x7PvUK*!x^yJ*|Xn0^Mk%4Pik;~8{Sie^Ie zxlqk_V;ViAFgaOM1WR*n+YB@$c+{DDQrp9R&yVN!GMKe!uIJIn)<9x6`mSfmp{%SbgDFBJJ z!jc3_GOG03^rXlxZ>Z_9UYjY9E-B>7Fx=+>L=LwLSwH*-ww9-Y~)w6eBsJP48-pHQPB+5Pp+SFUS^4>IO?^4 z^S=Pi!3S!P38~mFFDjLR!ivvj6TreOw)X zfW_;_P2#AyYFS8pnHAa5>qrCQlvgx^V+x^ zjA5rYwThJ2uVCLlLe!^wc?j5P67Sb6t@k;Xlh3kFeZ5{`V5Uqna$O%r2ym_dT;h&> zp^8lviZe0KlohlgwWFFE4R@c1O3_5H9#F6=76%emp)tM;Uuc1@lrBaeR#H_9Y)5@; zGSr5Oo5Bw>#z7o1{c!D|MybOUn zV1>2}xVi2QXgwa*+Cw?gWW#?wIG!SG9q9=iI!xFgZ=#!R8v@@du!)8J2Z<9p9b3Yo zf~;*Jb*`T7sw%Un3hTm#wnlz=hN$fj1-T}@j7F*SWP|v1=wgBjTd=WoFa*d6LQzyu zOdk9&{-kMUE>AlM*FvodQQ*zm@CyY9a(pr*g7R3oLw>k05@JdjP{?=Po7ZEjW7(lI z2|6ejk8L_HY2ocuW*JdY!G5G1rq0^!-t^cHI|CN2Ye*&T&FT$;@DVw0OEk&Aw`%Ka zq>}?G*+0I$>K7KQ$R?D*DH7~5%W@PjUaO=loxJpnP`1rF;1=FRtYTAYGkd8*V$w8! zNFxYieu2&C{P@|u-b-?D^n@+J@XXkaP1%iWC0QHMYuIBhnw|V}^{!zS+90pK)oP_h zwEvgGEsE9J>`}Xsm6!ff5$w4Nr`1ukMTd3?{{kkX0T)tWHo5CYFVI}VNzIVAN;;O{ z+~X$EnS^T8+@)emev0hGW`hN$k83*c;O#UVNPSYytoKLTYp^|fQQ9{zq;dG;EF87! zn%^4<`q{RaJgH$D_O+Zmzkm|)nQp@InP*1ROLbdOexC#;y~F?V_=BmAe_PbBF zmb&n@5QHKgM3IQ8EiSseU0BSJ76ps?4F#8+cb0{#MFD9Moy&6}@gk7s;(+;C4R)+H zNNbsnqF<%&<{!H=d@ZR0O&6uHix_DMY$P`hYyn?Km8`PX`c7@((+-`c9>zuA)5gl3 z?be#6oL)|-YYT)+{=~b|@pf*)OI%vp97~?P|I^x8M@7}HeHaNrx?Ab)5X7Nda*!73 z2I&$|kVcUjMCnFa+M!EA8iO3VyQR^?d(L{#d}n{J>x{lwWbyCRnf8QFL&8=6f}_vU38efHuGip zRY{N@r5>56%YAvy&RXcJ9H$uhaIyV}^&W^uek?J1s(kn3a~Dktp6ETC;BOlu*LJw? z&0B```Eee0_yiv=xpc`b5?|=ckBDSjNa;uuUpV_q^vE=HatP4zC@$_iO(dAJu7dG$ z6$N8;9M%-R+<=OyrMtVhrR*-e-#7fBq@!(-rh8g`s4UO4e)>ktS+*(ua{Ed;q^Ley zu3Ie1kL5xF)qs{wIl$U99uW~N%&KelQyYHw9_WB*h=ypD8z(5Nud8Sk?lXia*wzlD zX~aGFTG9*pwsffzeFl|G%Tl(&zZlM*5eN#9xsU$Am^ILP+SZHy*@YG*0rHYziKAeHw91-PlmB zC2EZ{-P-k!+%(Ir)I`-jrh#BjQFS4gVP=*rWvj>$gU?J-f2eHO*>Noy4n0fCjZd(!IpW3G>(5 zJLpfLZb%e^@yT=$9cxmWyX@RX^%<@-GNw2>w{O}sBr7u5H1XD7zMjBRbA0_Y%t|Ow z!rHSE?NBK3EUKa6O@0~mmP+ksi|sC^@5)&al}HXk%|*Hr1#$2-yxN9^%_(8~_fm)E z$84SK9(DyoLY9_|ucp#VF&>}OJnc-~{Q+2&CXSuNzI)Xi;dHg~h5IL`L0r7t{M`J! z+!nTO?p%L#y78#WX-UaSX-TDO89IHD#DCIkTplh=Y)r@kF4M}1A=T*S(thw*D!+Cj z`6PcQ+sj(xV545m^;HJxTvKF&KxO3$4AeoZL?F9QlKMv8I%b~@M1(1jZ`o3(^W0F_+!o8=MmY;zOXR+eli+Ojwt`q^ReF8J)hDUbFqhzCGaxLBOYfjx%Z zy~Y?;)vZd3Hw^+wIwJKLtZO;JzAEAn%j>7z7blzZ3H?c6iZ`w3?rH;SW5`ZaYeOt6 zTqLZM=O1qPgPxIvu}RF9-sPRk#^@x-AyA=Dkl(!*RvUEPNPtjfY zLv8tt2_$WN$F`oedvaB>7ScQNxej$=oX%lI)z;6hGi?%lp3-k|&?&O6t*Ml~K`P2L zT@as>K2<&c{bA8Ci>TvR(BKXyiPo z;U(ISd@>Bfz|2WUUqWz}oM9F>qa&4v6jk(jlgy)W~qMV!xP7mK7%~WVZjd`nR^bep$l9RHsE9^8RjIjy0jg6@@ zFz(q*lC!C{PhP4U*LejKZ4OIYs)pz=X_CM$q3%kla896mgIZI7tCm1hw(}pg-NJ&sKF3dr= zLxa*Z%HiWc-L0Djj)Hey)3I&r?Bx= zCO$Ye5(azQIE;>a{YWhzY^le!gP~8MZx$lo*;36Gqdu1hVcRB{s@1nrl%`t^k>_og zOs(}S9GWy{f>Gb5f*rNbO7~jLJ7)wZ%LeIe-0gk(623=q$CfK%@g-+}J}Lfu3xzFZ z_0#0Q_fO6(*hvb(Sd~(EBeAKoDq^Zug>p>8={@el@4nFFJhJeP50a^4ov=+AP7xCq z=Py)zi`i)gN=fmCp`U0;w2bl?H@8ZIte+`&x^Cfi`Z6-TZ}jx>3KZ7O3}Ox(S<~=q zES*e_qSebkBf_5WepFTuPMkh;0?%265}o^}ix%WGCgv_Gtv8pUr)LyB3=NP)uL-pn@8F{mJ+QOfR&b?^Zz8i|M&^ZC}Fvqz^W0Nr_O0OP!+$c5) z;!CSm7-h)4^H|e^ck*eL@S5AAF&1lu_Gw!@lD_1-igZ`D*Srg8P1Xbr zA6-)&lv^-6zU-4!knA3k_)UsV;1(NN9(kpWP#U)MU2)<8@`ksK@Xd#Zrg1&b&yu>1raG71Rg>I$r)t=BC=6*>}r8Mm&p{Zu#JK~D) zNX1@}3N!3>^(T_$%lD+UUOH(s(m%zwJ}rU_ZV-;>R}hmw#L(&_d!CR#q?O`a``n~k z_(K1sAw?Feyk*?o$NE^91TZ33wrD~A~`ued+XY$m9- z`B5H!Q@JBp`syGQX>y|x8d8JYc7GSO9Mg_D;?2CsIiLsJm{c|4#|$vmEKP5+g3|4jbrC1 zP&|%_;JZW}Aq^rK{*WtAArc`Ol{&gSVsx?Dko-&Ft6 z3$;F_Z6q{VdEi~2T`kDwu-kTcEBJ!r+;zBE1yZH>{XKi>8<1YTPGWh)w?gBNxlDlv zEJd5(PsA^*W*~DqOsyJFmMTU6M#nT}kD(mPfF@1G4v1jKwJ79gM<0+{Ik{#Bp-*H+ zH@lY2%Ep&mZK%C1$<$PN#0L01dE3Etv0D=c?{Q)p;*CJ>_^%0aU4(6UOF~}Jw3pMM z+m4)b1y;|XnjRr5HL%PNe`Wo|eXOIe)IGno&X*^n{V6Iq_Ij?e)>!3F)u2&1?$v9QGqPuPg>+4`88b+Uzh!6WTJZk1d1a(v;EQQ6}y9$Cm{0OvZSeXDJRdz499kBnKkYSUxt=xjqsh` zjV6xHX&ra@3YscfF=A7KpIPb+boobdi`!*m&@ZCPZPM(yn_Re=?0mt5EPAmf-=g$= zN8(=Hyq6)XInD@vBF+VIG%D0I#Lwz5439szlepHvi_3(P#%jAO+9Apsh>8L-d)ypU z;l!4@><}wxl?S@7oNhjp^3b>3{EJN#`A7}!ag^PY7b(ufAW}>@%xBiN%>h!|E4+Ao zzU+D{Y|?|!r_Xb<>&H|3r3d)l1__5W$(G<+`;qSpO+031L-jN`NpdnO8_JKRHbaVN z776u_V#3TKYR@BS|6bA=Z60WBvbQh>M%#;>x5Olt?YGZ}hl|kMl6R= zU~foSpC^pd^wTJy0gM%k3KaiC_L4K)dsuE&GwZ zrj%@1O!0!U3WQW(<9r~)3o}>_CX}ZH+W6x7?U3ud3_hL3^F7MoFIaszu^lnQw#;Ix2>@w^` zL1o3Cu`5WI2kbB!u$WmlS_ci<0EKukN=i75gN^H*ZpRm|nGW(5HqG^hX-6*+(_SML z{cxeYNp6I_LsMoH-E-)>>`0u*ti9%`;VG{$%_J@`X2C||>5(HXZlP*zpIcKMYaWfG z{vwlAvcK(dIMqSu`>S1o4}b;%|_Z{Xv3k}@`;F|lm&Dc-7|iyjvALt$3ABslY71XOTO1D(l7?&e zO_+l|owXU#&=B63Ci=iL!Fk}~n5+8Zb0U|E(s>$1-%U?Xhs*J|Yi?55>iLlSt78R* zQywODM|9;Z#KM#qq$1qUccg!WF-W}n#+NQ3R9&F|o_Cm3Ozz}jzRfsED%o+QD^ApU zRm_-^=YAN+QIlRwO+n-$PT2YPpaDTgTUJc7;z}VklOQpER)}sLf8UlChr8qKVqE&h zTCUmC&oWK-Gh7`)WjD8PRK3F5;Yu7j)MLFP#io-HY+Q0CJgLEvt^DHrVd<)+;-geh z<|ZdjlWL>>5_Sl)_0nzFy;^c+A`SBK0Oz&q*vhe!&3S~P@rLwz7nPgo8mC!~iYxt@ zy)}0PI9!_9VqtL+w6nOnU*gX6rVi|c^ho9)^Jak`d>QPiV;U{_V$dF`iV$k)b7yG$dIve|H_Pj0==X=DBw6T7`{EjZKYOd&!+g%N_h0%vHSnP0kxFdF?^R zCudI>^Mfw#Wf8=eOHj|X;Jb;5wnRpfhg7Opv+Y6Oms-|O<2g1G-Q;bS8!PLMtwDt# zQ-|=VL_?Q`oz@H(v`)Xd*KCdB7Q&dmQWt$K=B^ux2nxpzfK?r|8WD!4n9xO|wZzgl zo$;Q$KFB~Dh!lLxBcVPmUb|n}>ldk_MtgIcGZs{o2c;7q6eI0_Y@I+UXSmZ}G?+VH zrp1(8e|5CEL600jY?E|Z6E_!1Ahrz}oZh1=Wk?%OFWVE!jd`6d01bL`M)}a8y-4X! z=2PU61_Qfb*%&R;nI#KQW`cz8(n|CgQFbr%ZJz`*DU99gJydFKTX^3YDYS=LGf85V zSl1O&%Xh-mWrHk7NW>ZA^)u?QnZ$G$X4^L&W zKTes!coJD^F*e^Mekbrk{%Q6{CzltL^QL|?svYrG5RRvv0biKs3>4?dpA~7gC%rQ` zPGf>T*a_e#BLVYAxoJ(^_j!2P^m?Eawz>Sm>@HR#pJY!a6#b{@lo|A9--;UyNaUR~mLXM`)`AnrCf7$W;d! z$9Qj6yz33|xo*n8Qo_sDcVj!>7=5x7=J2YZI#HF`|B?o*d<~fx<>&h@Tz#AT=OZWJ zkC(q}ga1FfFaWCocVYNfEhMCz0LhWax%n8HJt;D2X$ z1>ylMt8l#atF<=>&7NOzt^(tM&QEwej1b{?gbkp;bf7*Go{mfM3+cZhlLV3gm5^|f z_Fp0S+k!|S7f{*==c*$`bc7I6I0A!#E=727%GJdj#OCF%niYWzK+_E2~Ko^DABJu zoPYp8JrW!MpBcd+5&9)r5)c6B_<;lTu^Z$<*p;17@Gy;qTnmFLGqdW-5{tYh&kOyd{fb;0`A;|MLJrzJApaTL< zn^u5@fgL9B!k-Z?4D2@nW&m%s|2yLvmoB34|GM!G WRz|!2$GbVNetfP@emzb9`0XG0pa)+7 literal 0 HcmV?d00001 diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.2.20.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.2.20.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1562aacc22b73a2db570b62ef49ebfe06c602535 GIT binary patch literal 15576 zcmXAwV|W;h!Ed?8?Bt~lZUeX;e$V8rvg2wJ%cQ!gmX2|sTIl|# zBY6!ZbLKUPA>?}*pQkPh-Rw!mEo7E4^%!FR!| zI+-nTF-)b;dXDx&d)%7~-b;Hfm6)KiBc~SWPR+jR#bv+|T4-_?hUa3naW`63bOW*) zL*KUp5*vu3qv)?v-qhKt(~m0FAzbOYO>3*|S`pbvYzm4Mv7!fsu**F&ua)1ONU`!WoSj{q(@S<$RHqP)gz=6CMcm*$sOS zw0Gbj=k4AC$OS?IF{FH-NT_6jy@AbeNZu{+HW+Qb3PyDY4gT>_JgmP1_&=k0hgz>~ zDgUfCk?*KsqC$28fL+M``~AZpAJiT(c2+|D%?U$6{|<2(($tPyr^F{`o4eQRWgvgp z&G!DTn&WFaysR%dfR%;EHhQgRwzu=b$%(%tI&3CZK+J=ig@@P4jhEFyT%Ef)N~M1V zZI6Z1$w9uHXKwkEXLu+&WG|@s+UeTT`Dfd6qlVY7HjZ94y`1P#|4f7-%z_{{u^=W+ zFLqw;Uxb2BAv5Qr$NLXhp4bFPa`c)rG(4pJdfQN(e%u^feN~o0Zj7%?(J^?L*&kd4 zogQ8So7n{Th&wyw#F5t5r{*K=ffD$52%^I>jvO2W9oTi6B{O3!e+HkANBFi7Z%`iD z{Q*RJ(Gp6IsG#|T2&Q-IKH6bSgGImSsSp`vte7bjqUzz2;Ypl8G|S97(XZ{qHV5WX z%!Tu#ynkGJW3h=Nh6ItCpW)H$p$GLXXDUDVpr#?_ep%va-9Xm>6zp$!~BNEe)3i{+HC)-0+G zR_;`bwdRz62Lqv@%0EvtH}nckzq8~Au5&-pE|r0a4fS~!PJrAs_37=pTN$t4cn*7$ zI#n!@)5cSgV|9=jf+{%QLk>@OtKu7gF5hstMj zO@fDl!~Q7`Y$l~tHAYbdDNh7CJcF1h{gzzl9NWb%%-0y<4JAgsw+mgj=K8e@XHT3F zZp(m&H|6;I;QOsJ*Htf=*R8V))z>Xi-nF%TkkYXd%;V)!eE%swCNj*4iob-QV0ifW zR@KRnvD`EYl$oWcDd%aXI4+J%@0jf9a+~=0tS@J+pCWrAr~B9U_-s(choH*}C6xKl zo)qC@`Ydo85Uf#}=~6&vVGqHnKDQ417W{C2YayC7wyp1`S1Gy)eJOR%rS28rC|Er9CdhDeTtLTcEy< zxg@C>1~>i^s&$BLpkSb*?3Nzt-85w||B>iVGf}Uqy7VMP=Z`vuv~ zq0VwdPYccjIrQSE)BAgK4*3a7SmvCnsK85EVw&4Ec#JM#eBm{F2j(z_LQL`th=zJF z(5|~u8nkWHY-3a+uQUq&F!OK1k9abc<>0mS&eQF#+N6u_p&P^?4)s?{PlS(8k>-K(u zL2v$HgkN;F9UbjwP}it}O&XBc5Vb3)034mF?w!{mcfl-~4ZWqE(PC5yY_-`QOlNdPt?a6my4$8yVOAqA@M$x@7nz zzYw`TR1aa`4`L-uomfE`O=!t@5J#hisgRD*4O$J$JvN{YqTo?&Zux8lj=<#;-D-C` zrkyz7>$p+cj9fyMMZF|~5yhq^XSq3KgyHs44V$FcE@Vll_S{QNzPW7{0!ey)4+2K772%H>X7^s$T0^16YR=%0I>CR0eM z{sH;6U6Jg#AE*vgc&DB1Ik-k_bBRUFFL>3 zhX3-9$3lYMqk8ipOu{SRCE#1c(F}1gMlSJl9Zq~gb(SGS)A!iZBi>@+5hTMyK?m2c zne_Yma1lgrW)>RtjkrI1LTjJsyH_F7CQ7k{`TYcsjSdKo5`)5gKHgX%7JSjS_hsb3 zOU^d1e&V(=MFAyO6Jo7KuSu$x`lhPFl78|cU-J)P@~1JzUn)^NLC$9Nb;zTPvn;U* zv~<>(P&A(_Q};AcITJn+PwA> z;9WKa_}s~r+-1@_1E+(Uh=G?bs~7Z_9hQG=HQBWT8Mku0e>Ci3Ze6?pTC`pYpUt)m zaX{sg?UP&iT{qI<`xS8&^(oYmx6>tkyHsg#1!`d;nqC5ipFBEM{iuB5qMJXvJr1dv zpRn%f+Ic*g_f{CBgRupJ7g#S2o|NsHn0_<5Tm5a5a6V-~9l0lL>|4_>h|A@CUyiJb zHi0-${wDQayccM}e90?CW5H&s(k|>-cb44a;&j<&>Wtl(0>Ev};-kA>=>))?1-w2Q zPXMnhvkTegNpu0S`)zy3(D5+>_l{S;ojS||;MBh^X9j4$@GS!D!~sI}uIu-*k2r~o zoHx>#P5hi#c{{nRens<^B=PEfzdt~APBP$?^u84d*xA|K0v>NGQ}wQT^(T#N4!zQ! zldTnn9|b8u@VIU&P~V=A2xQs10R5&PenMWHNE?3L*#YlByORgKtQq{&==J^vlLb$2 z!8-QgKcYsQn|Qbm!L#9^U67u=-4Hd)V*qrE1Ph@OP87$2tDu%Ghv7YvZicrKVv#O zKfhLDw*-zTP7canalW%={Z+#THO^3C;SZ`1JSXRS&d3<@s$%$F{A+x4-8bfjSz@N* zO55S@0l54;C{~$?W#RXH=Vu9nt1|h-_)az2<~cy|Rgb+Ac%b>PV|mqcvw78P{-~&5 zy0V~MftS0j9E|r;h27p;0AN!(89$Axvu=TVH|me8fM-8NP6(1KnW1=0lfudcf5}g& zH@@)a=?%$GJK@i#r6b^^A8>VhOeOU{`UGd`pQ+L*rH_^l%_4NA1!IS|jxme+n;3MZ z_K8XF759GCIRD0%^|i6Q5g`AAFfNKf2bNYPs8clv-w&X&G32xq3?g zthj-wz-^Bs4>xvKP+2pL4T_D9{S48h4$B}P^?IpZJ+f;^Enlu z?%7eYhA$Q4ImVq#aCL+@JU0*0F<9%2;aqNXf+PxT$(#WflE7BvCtz*!3fPv+Uxfwr zbkQ5OuTX7|OMdwMzZ#B#PFqi)dZX$QAV>rlia+=mfi2kYk-Rh(J;(K9?yZZS;;eQ7 zs7bR8t_2cAAFp6&2c~g*X2ija4vr*L=yQFQ_C&c=N*QRY0F)LdNUMAk7k59 z`hwOUx4yxleT~7Al+WMBfzB*y;K^qcxLiF4bldMbnn}S0Vxdo2zPcgh9@j{fh#tJ6 z|F2QM>Xas+{4NJK0Vulj`{?`rkXc!l2}Rqro<_;XER-G(QS{H+N498$@}yFrGt7z} znXn#ivaA}ntr~99IH8u4=d2}{^msP6$?i1yXLCg40DQ9d6MY(~&B1 z|4@Vt3{PRdMXz@bu9Fm9&~54>qDxSe3Qd+ihC(s~y64Ltfh(!cr^kP^fCot~&0j5k zwFDZjmqHmCOYC!A@kAs%C>pb#y?B6gT$c=R^ZZc$EO9jf>?Qy^tl!nR24|wrZ75sB zL<-Q9YP)83$YhH_snvWPB@tnJ@~{jSa!pP)d@pOK2z4|ayXF+*^I8o*c4%F6#9GKd zMX3S6blOV{Yz$JHHzq^%W zbph>4qrjJu27UlaZpCYde(vu%Tu48=2T5g*YJEQ1i&DF+=2Kg4aanap80$(S47bLq ztafOv`CWUBn{&9#2{d#?ZqhjEArH(Gvmp;g`uc}*ow{7LEvL@?W->5lWtsjOE66Oi ze_@LE|6|HNVnSp{b;#b%Xzz80IXQC%$1l)l0t5q*>5x7+h?vbR{9O`4_$egFkp8(I zaTIYJXoEO0_7`-R5=z4F9?l8V#>^H(h2g$pJ*7Wu`KmtDHUOuX$F0>%;G{XH@rgg_ z>V{wc@JS$rVf@gLvwm#nP5h#}&xC;oSd@74drSIwl0Waxe7m|s$SP_CC;hjPIarYhQbbfbp`F#8G zRN+f$U5aHMnf*LrTteD6(AMV%YjNGE%}81Lbk^2z5)6|vOO2Hu<7!jl;u^)EoV~_r zIAeGTCe+1q(toLw?&M1pn>XiIfBihJ#Q&u#IEC?#-uL34TfJ+uW}p_s^UzY&9I#aN zC1t`ZIY3YCbJ(|JW98)0%@s8a#J?2&RiqS1BD|hG0S3o)kAbxnnHS)Z8rUkn2X=Yp z^PuE4w~X^oM&y=5OWaE9Sk-R3+km^?F`#h+Sh+g)Cpx~oPa{gDb6K+QMZW&~#z)xt zza8G?oV@Y5mA}hnBmo{wp>};LNd-v#i&K$Yk&?pLNOg0hw9aZdUIMSB6Mv+=PS=Gf zLMCEI!0i)zV^MQeyB)|^g5ekMf8mZkSduLf&y7;Q=#2ubqX4}(qa&X*7#Rr|%?nU^ z;CEApc>%b%ah<%!sC4;>;L= zE2QX#U_AC3|MctblZ$L*C!teVkW^~0G0+Xs4o#R7aa9d#oiU?R@<=yy1~ph2L{N!n z3iHGRynNiIsa9p_R5S?{iTH(T2duE9!H6Xh$-_j882o7kPO|gxLW!ZM{?kb3D9aD103~kyPKQdt2fND%s#RR z< zO5r-D8 zgRLaHWP0DrK>L&qUtGsOeBL$wWaX=U$pZX%U8I^+A z94ZdhtLQ&{DsH<6>M)P|&WaEGl7N@q5#YStsXybuP(mYL%hwNhEu4e+5*W#+ki%zv z9l%Zsw0l`40>9L|j`E0l5ZLYVXX?9Dukca;gqF0XmJ%W zCctuc-E5^v?H5|Pe3QEs~b?!(t zGo_b)xT%&cnp{%IR+3y&Lp}CfyWaY{5KO!m0Mfteyt{GPul>RjcRKD8AK? zW`esvQ8njJG+%U+8oElo@CD2VQw|U%>`c z?a?pbV@vHEP<;W|jDhO2cYuXwc+An61MOA`U-jbAhpe5L6V)2W0-cf>1yRh~VSCsyMLBwq82NvnEdM3j7vM(R{L5HK zftVl+a;0JN`-qd5;$NhZlAS(%DM>XZ9p*jbDy zT%Iz20Guqn%!HwrYjbyd0j~Xr=lj%LZMUNesmToO+1Ab7VF;ev>E`~MShqsYy_k-;@$!f(EkHkST%j2r}iG9W65XOLwjez9%6ZmAat9lPJZ z+OfXB9q~DN=z$1tFoJ`>gqn<0wq1b5inxgIv*XbYF~waeeyY6yS8|`#GyrfnRsUG8 zTMcNlXq)AEz;Vr9w_qx^63tVURwhASx9tB{2Y!{F#_vcY!0G4Jq<8iwgzze#%Mv8# z{zbt>yNs_(vap-Lz8;qpp$Kmk?M<~gQ3vmk+6yUYUI+uNq8Fx2{umgABa@Du-`HrZU!P_}z$_LfrLiACK7P1&5jD0teo3jGoy4>`uBTKiH zrj>Tf7xNjgY4UpY=yehP$$kqXs3Y)IZ~DKQ(nA<%S05MV`>}q&1=;MRZ)!7C17`Dn z-Srs_KzmlFZ4zKZN{Bv`UGGdu6hW7;%KL)v>`z<8KQh>xw@ojBWeR{2wJgS0f~KTeb>Aek-_!P^EL7M-h7`5`;3=d>~Q`5m#NGdo5Q+B zl*lQX2s7(Hp;XzZ%s2YyMSIoGfsE#JQ4L_J^9<$RD z0z-lIq8eID3;K_zcu&UcxJljDh*r}bXlz>DG?tt7N%tu^x`Efv_M>Hn0s~yBonPBQ zeiHd=Af6M&#|c8~MNCKNM*QMZbk2-BJgbE zE9TltS+7OJYAcf~)eCN-^)+19-jQ-VvFx$EM*cm0K!F`Ig>+3oRIn-Ej)2GRN&?bp z-%s-1LW1CMK7{Q$YHIaLtz;)*>?U4>FNBRM>}8F5ITK3<^K7yp0prOnBj3@r%;3dE-n%D6Pwpa zgk%{?V=-dcAQgxE2Q<=w%qqxK{-79)A$B$uA~^ELd=8l+4Y+mS&(=HkbWKhD+7PGH zvk6{PqG*|_&$lHmJJ%@%P`-}cH>C5{kl_su@l^N({dwC)qsB&cjejxu84EsHv*LPn zU}svM54(N^2WoKn&_{!Hj5)!QEAC-FZ3~9utEuS=%pE?ryxNmUGP`qBE@i*+Zs9h$ z>-^*lUjxW?f)b2nq-ScHMe=2~k&W&KNr=)#b^6>Xld;*qVT)g$gu-K$$y3A7o1|58 zU>dK9Ni!JdTE}nk2SXX0IjkS9(I#3&`=eEY+fXi(5wVtqVJ!gXX)P1NVQc;r#?eJznKQCa2UDD>Y#fiVBcqaplPdBkp~h1B z+*30MvBF;T0Ejn#nkg}SbHQqKbN@{%+q={RL|iMeRkYhg_(Gy4rWTOhkfFvO=t`oD z?VdAZ6sLObJ|r*j<>ak9{LD!C`w5my#tpmXO9Xb-w@8`8VIsupY{jT(1$(<=sun2r zhdJj)Jg$TPFfOl{NIhRTjjD=zIpPQ$2BYsb&cQ`Yv~A|s)v5zRt5ntL0EY@r`$}kR zo+vADU($Xp_Kzs*tT+jrpW=vixP}nXDcFExsi&CYbwAwl>p}yl{~)b#mbLWZ2FkEc zE8=6Ppi?VeCM$w4*3833i|%bbJ$>p<50&bchRyt^^{cmQI*66J19}%X^Mzbp*K~hE zlp4Ds#O?(&QL^O`K0~=JlsHOGk|oqwbR?9q4H+KTU*EKkT;nLvazqF6{wLG};b>x2 z@nM+EqlHW^Q{DV%=i4S@KH{rnUfBAiMndfT^qAyQOqy~wr9JbjqE)pPf2#nSOxEI- z%88;YEhxqS5B(UVDdBYzP+v|~Gte~|oDfAaP%V9wPs*_EK$j>n@gd<@M!nsc;bF)J zimF!q{;*$&?3(*?UFB<=eGWd0_JX1p9*MK4>UC=Ae)YMvqJeV~UaY;&YOG2{wZktS z3hf673+|W#b35|coywtM{Q5;<6vd!K`88+v(Cu|G*X8i`*ejSG5OX@T7jlef84pG$ z*$g@X)DpHQV8X8b#@4)y;Z74-bx}9e;tao8n8BH+6XOQJrP~$aQ_DU$17&$@I1V~? zyxbGD?JY%x_M~^*WoOw)v3nFBv&1=Yjw@3KA`<=-X7B9z3zBuRu4a7Gf%<{tpJk#( zf}V$OaI^iY;GC8*&%gDL?lkbwR0-LxD+onwne4qsRc(g(Bqf;2y{sa`geooID0JUE zEt4f~otesM+a!C6u$hs?rOSJRZ7#YVDyintt{S}r-LJQVt(+bT!(ZunVVqU(U$q9M z8-(uW28ULCUACi4*ZUu?jyXMsnUS3_vhD@UY24+{abrrGAvD?KQno!_%FWd?;uw9z zJt(G8027nV(~C0RUc(Ch%FGT}EF3dI`m90DA1q%D zv7bzvOLfcU;y)lkFn0uq7NrAJ^1}Nv47q5}VxN%haIDUfzkL&3;v|@nKsu0sBX77s z{;0Rrgr|3^oh9`&hs59ZUq)6BP?ecvU=Las;kB)jU@C~PTZIGNYMVV$W%dk=@cm8| z>~-k~4c-s^nJK~>x=p;<%RteI$P-yHOM$W?qwo$Tm3GW9kucGQZt@$9xXh;G3Wiq8 zTvQwaxn0B_d)T|&OAVyHcO|quna*+dc|v%e@)I?6m!Ww4^D}6(VFdXa z)ihGTxaL|eRrQ68N5I6aJdfGncaAjdO#rKA+{qX(>6+XaD#6H5Jl;UWU#Y;=u9{M7 zv~np!@cf+R8t5m+=Ra$X(bvWhE0cHN=KL|ROzQ`;l4VgyCQ`I94zAp@%hUa}BFq$< zv40EOWTFFv!zh_#O1m(SbS08}(tbn+3EZRL^>2fl5xLsu?=eJ<*Ql6+CYvV_vV=m7 z@I*EH%S(P83|Pq7lN*Qq`TLisNU8=cyo7EaQlU>tfunec2EI3uV9e!!CNCeeVG?ef zmZ;vo$IAOxovz9^4oMi(aNtYfBkLKl&?)?-Yh>`KiDNZmENtk!pswf_-x|H@`4kD;|4! z%eMB^JfB- z;%%x}?!;4xNWc+T1kBdcLgkzhK?e!wf3W2#Dk@Hgd$`3zSjO8?9XtgsVL@WJrYg_X zy&%-ne*f|N2Xcdf=PXj8gNb(`-rtEs$jY9=R>vs}Tjj--NA~a|6r4`BV;AT&%AVjf zR|+u}ASqKxFB9xK{g`66{b2ngw31PkfqKVIJ3GCPN>#~yKyp>`$xIGpnic12P&>UB zJh5G248J$uBsSSUS4boLHoA#GYoD(1J3;l3wM^|@-{VOTYkkov?7j-*i_}h4?Icj> zF8$*wj-An~Gv20R@#109wG{1b9=?A*jtI5_bz62<`P#kw{v=${AXml_lW)Ql8XAzA zkF(_Ai|9%D>d)3CJXL>l)H035U4t2RCwxnQ8Y9B;O<`XT?q6OB#g31zv(vQ#zr73Wu75Rz6~qNp}dn~x?CCbl84Zn(#QsxDb|dj;JZ^(>hhqSvvfad-JByaB%4_-!maDP+CHfuSo2@Hs=*^X1$0AcipoBQ;g4+mS~V(nc^v`HSA+c^L%INWYT|CkYr9;U0Qj@xMLM5~j_R^>VF;1jlgo-3#jjzlEzV4h47Uh%xbn z1dk^{G z6XBjJD|4E!5kVPz+|XR9^gEJ$M0AaAkGcbti(801uG?{BE^4o&Kg!7Wnl{Ge)~E^9 zKRI^6kg@@O2%;Qp1ap|P)zNFv*CKv({FxhIHc;u)k!7Qp z2WhdbcgZ-#(&pD^zC@U!T}5QQhOhdWB;8%omz4-xLyrA9fwiUhXH>E&qyvW zQM7gj8aDH+=rYLu2KlI@SmuNL7GX-bs(NBH={iWM4%SqdK0|94Qg5=u)L#6$FBqkr zZ2{u=x#D1y65R_c^n17;L><@U7atJs@Q!m&N)k}YoW+9UNfqdkBSQPTk3BkGr8^_3!0lB|LhKBX5 zO~#YIizzxI2%E=#ph8MPNUV+tbRf6%vZZn@E=sxV77SW1~@(4;`Sp3D}h{VzkeThbUSGWU0uAfV~-CMHjz_QKNh00U6&A!ppiotH$f%d-vw_&)h);1fCyK78@ zE)TU(h@I*Ka(!r=*IqPJid_?=3HL+d#CMtNjPAh*O4>VScMnLu7dQZt^=bQsD&b$7 z${vG?`N|xv8C%3+Z;t-Wq9y-ePo;K8>@n)Z9^{(Z8qym0n3KQ`lnegW$W!y3Ei&JM=t$Ri>%SnBxz7qNEi|XFE^qiG-E> zjCTnviNYsqFh6@5vNJT*zkR#Ls}{MKrohU%Iuk31AXZuxwHpeGm^}$|<-l3uyY3OI zilNf7u8RmvhHFAE9n_+z$BdNnEfr&L!oTN`?o4*v7hH@;A6kjWaT!SXEsf z*qxWwAPkrd)Emjb#bFO=D8;Ghth^76%F9aZDhiDV+7w8~vjWZT3>w z2n#&`Nhlc|;im?l)|e{%v|3aCZ+=}NCZco#Su!lWq3acNtZvJlv1j@sgCtW9+DLDg zUr}@shu)qJpbO=fIbbI#r)A~GqPB_~=>3)EDhMH-k7zjsYAn#<%sz)LW-X$8N>7kM z%w~=GVdej%qk`PO{>4u)ZHUpu6x_gA1WL@5S_YY=o7u04QKW3o1)5!plk1BK zz@g|sj`sJjwak?E8#<_>IVkbVVDQ-%rB>zJ`mg=y!fe2UZr}@l6E@;GqGCYb+a0aaypxNZal!O}6gr`h(05m-0*U(LAx0*eF>+1t?|LJH zOzFHKe*$HeIum$8aa+;etg%7PS|LQY&c}@_0R25^e6q`^AXsCjIIf-AJQHZSAcC|k z>N%=asj4%yaU74z_6KuTa$si@_2j~+5Y3X6&dq{dvG756^qa8bo2Sx8@{?G-F7Bao z`h%?(Zc>g7WYalwS3Lq;>J4w#P#T^6RcL-N(uo+6L}n4try<`?+>iY;>+Dsv4YJ;k zgSvP#Pd9Y(VfD5hYs%s0S>s6UrEmNLw)4YTL~t;o;B*6Jq*$ACfCjO=F(kCAzs_(wK~Q}6*!kk zyK>2ItpqxK$Mw*r9bJ~(9Zum{i(sj`2HS)htH*pD`L9Q~=IZR44+Gm6Mltuk58FtL z5#bI)ezl}_p+B7)HKflvD7J=G@^^V(vV3j&_~>%^Puq6fQm>cRJ&q|Zp%%lXBSTYH zE)7ZZ;6moGk|ZQ>x(y^;E3g)Y)8Vbc?cZdZgo~pg=0l|p%t3yT{L)`D&uhBQ8EW`! z=gF?Vkt+Kn(WAjCnCHlnJFCtj7iCjNJu}R*@rjC&d6KAWSI+=r+H`8|Ufk_#8hkgS zogi$y6n-Og?7iYjF4}n%mPU(D2@28!<2DNMOiUFp)LbUH^ybfSKt8Iicp^3ncgVvU z7P#!o*f5=B3*#98hl1f4uniP4c1Q~A0;y3%IHk=^*Vd;Y^!5(b1PN~xVA3zP`mH~b zLx^|JxFz)ap8D1lfCk2dEGtEB}chmFdO=n2w^{@qJ`LP!&2S zQX3t!?y6wp_o6&_9rY=MJ5`WUyWH7~ZF!@}fdW;qK`>5r^PdAA0)?PIjx;sH5NUeI zeyb1_Anj3lP^4M^B+~^Afyc9q@^= z6dj04P9s6;40VZ*j2YP5s}W=N)>Mm~xa(h77b5KEFg`=7q&|tnCrZsLXXK*#Kfe=t zSwK-X3Bp)uH>Dy;pJ`?Az>;QbtqLQFYsZ9P&qxr@WES{}GOfBIAE`@jUO z#ozV6{Of7OwRz$Y)2jUuDmSR7?s1@)?7MX`4UVRo#^(+kwni9D2Z{66k$Oi>=@uZS zsgNRv0@=_~Sn>0wh&~l!8<2xB!Ps?DG;<67a4m#N6=zy!wJL_iFr-kkjmCc)b~lp< z3*3=cCI;&x4BMG68-^m-$uwcKC$wmy;IxK|g^^s6uoLs-DK%lIu=77@5zfnbWHcOc zQ$WnPG#r!srEDxd$+2fsq1cjblgrVlc)?$hO{+a`Y|g#*n9riHtfLvA?OU=%q^c%Z zn;pG>jiA=8j_zR+)@^`L<4o@%wVqlobA04`HYSCE&U<>EaHfFX9**PSOsrHc09(KZ zA6dY1UILXXW7{X)3cA?R^@49TirL&5}%7iqz#c)puoaTNlZ8lT;1=nQ0~`O^e)iq9Yv5)M}!ACt^K5 zaQxnfK2yUZZYL2Lz!~U~L%Uwtg~GJ_W157$eq;^#*s8Lv&GwT3plz|*kWH~W$~w#Y}}bMtL!J$Mi@#6WLnI+ahH?Y4*~+@ zjG7sIc8YvVsVi#h6a3=w=r?GiJi;~Fd!vjH-`HT@7w`^JL#c0W$+^p^w4?}!<@(=U zkoI*|wPW7^hS(z%IKRZ&@mUmOB6v9B6Lz%ZYNC>pxiz7q? zi${0l@XjAsBVY2!Q+#W};G?3}Z|(B&kRk#v+c6)v>Ynw;7`rDea=QX!&l1-k3M_DuPzR zKDBa4?PXVkm+;dbuJ}rUDQ}x@`$vZ^?IB5p85!?Rt@(Sbo^QL{S#Pc$u>4H8SFL%*phZ}a?9UuXbL#!`cd%LAM-(AwR{mR7 z>FwK!T6h-1*>7<;sNOIl9!F%F zp|GFBna$@!bt&k$5kqb%>NKM6_(Y0<$yp6s*?@eaz#IDjNLm+Z=o|a4GNEhjfV0&6QPT(?%?a8#v zzBaUQwsXSzTLT(Sx74H`f*0wtQ+c*ak0il@U!3cq#1F;ll&ukzGlyO$oqAmK;vK}0 zWFOL5^l$q@#BigtMO_42?zCGk^kRDT)6_&;Sng>h0wXDi;byrAUUb0%Wa*t}< zGo0vod!?C?nd~k7%=?qhO?c_f!NEh&bST=XT0UsDpxKuEl;?UdaRN3k$-mMQ9u0r& zd12G*l|qjS43aQ> zW}SUE<&nTYDXPOeJpD5aTF=tOVVnJlW$|Fjyt&tAq@ zxh@vVomH#nbUMdQ>#xoira0I)ec5?eDdTgkOkcVWxB`B}gqtNr3Og&aY4MM{L}^8y z|GghsUleu4I?-Q);t?8o9GnCjfpBZd-ldP<(OzrRqzIde7DaaPfqvy?y>!9e~GS(hAJ literal 0 HcmV?d00001 diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 2b1e20820f..9d3726d29d 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -221,6 +221,10 @@ model LiteLLM_VerificationToken { created_by String? updated_at DateTime? @default(now()) @updatedAt @map("updated_at") updated_by String? + rotation_count Int? @default(0) // Number of times key has been rotated + auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated + rotation_interval String? // How often to rotate (e.g., "30d", "90d") + last_rotation_at DateTime? // When this key was last rotated litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 1c368d5807..dd0b2138dc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.2.19" +version = "0.2.20" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.2.19" +version = "0.2.20" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/constants.py b/litellm/constants.py index 6e70ae0671..1ed9f237a2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -989,7 +989,11 @@ HEALTH_CHECK_TIMEOUT_SECONDS = int( ) # 60 seconds LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check" LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli" +LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs" +# Key Rotation Constants +LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") +LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400)) # 24 hours default UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard" LITELLM_PROXY_ADMIN_NAME = "default_user_id" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e5cc96ea52..e04c75655d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16660,8 +16660,8 @@ "output_cost_per_token_above_200k_tokens": 2.25e-05, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6f2732af37..345322a002 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1,6 +1,5 @@ import enum import json -from litellm._uuid import uuid from datetime import datetime from typing import ( TYPE_CHECKING, @@ -24,6 +23,7 @@ from pydantic import ( ) from typing_extensions import Required, TypedDict +from litellm._uuid import uuid from litellm.types.integrations.slack_alerting import AlertType from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject from litellm.types.mcp import ( @@ -787,6 +787,14 @@ class GenerateKeyRequest(KeyRequestBase): default=LiteLLMKeyType.DEFAULT, description="Type of key that determines default allowed routes.", ) + auto_rotate: Optional[bool] = Field( + default=False, + description="Whether this key should be automatically rotated" + ) + rotation_interval: Optional[str] = Field( + default=None, + description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True" + ) class GenerateKeyResponse(KeyRequestBase): @@ -1802,6 +1810,10 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): updated_by: Optional[str] = None object_permission_id: Optional[str] = None object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + rotation_count: Optional[int] = 0 # Number of times key has been rotated + auto_rotate: Optional[bool] = False # Whether this key should be auto-rotated + rotation_interval: Optional[str] = None # How often to rotate (e.g., "30d", "90d") + last_rotation_at: Optional[datetime] = None # When this key was last rotated model_config = ConfigDict(protected_namespaces=()) @@ -1932,6 +1944,23 @@ class UserAPIKeyAuth( key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, ) + + @classmethod + def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. + + This is used to track actions performed by automated system jobs. + """ + from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_id="system", + key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_alias="system", + user_id="system", + ) class UserInfoResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/common_utils/key_rotation_manager.py b/litellm/proxy/common_utils/key_rotation_manager.py new file mode 100644 index 0000000000..319d5ed523 --- /dev/null +++ b/litellm/proxy/common_utils/key_rotation_manager.py @@ -0,0 +1,146 @@ +""" +Key Rotation Manager - Automated key rotation based on rotation schedules + +Handles finding keys that need rotation based on their individual schedules. +""" + +from datetime import datetime, timedelta, timezone +from typing import List + +from litellm._logging import verbose_proxy_logger +from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME +from litellm.proxy._types import ( + GenerateKeyResponse, + LiteLLM_VerificationToken, + RegenerateKeyRequest, +) +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks +from litellm.proxy.management_endpoints.key_management_endpoints import ( + regenerate_key_fn, +) +from litellm.proxy.utils import PrismaClient + + +class KeyRotationManager: + """ + Manages automated key rotation based on individual key rotation schedules. + """ + + def __init__(self, prisma_client: PrismaClient): + self.prisma_client = prisma_client + + async def process_rotations(self): + """ + Main entry point - find and rotate keys that are due for rotation + """ + try: + verbose_proxy_logger.info("Starting scheduled key rotation check...") + + # Find keys that are due for rotation + keys_to_rotate = await self._find_keys_needing_rotation() + + if not keys_to_rotate: + verbose_proxy_logger.debug("No keys are due for rotation at this time") + return + + verbose_proxy_logger.info(f"Found {len(keys_to_rotate)} keys due for rotation") + + # Rotate each key + for key in keys_to_rotate: + try: + await self._rotate_key(key) + key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") + verbose_proxy_logger.info(f"Successfully rotated key: {key_identifier}") + except Exception as e: + key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") + verbose_proxy_logger.error(f"Failed to rotate key {key_identifier}: {e}") + + except Exception as e: + verbose_proxy_logger.error(f"Key rotation process failed: {e}") + + async def _find_keys_needing_rotation(self) -> List[LiteLLM_VerificationToken]: + """ + Find keys that are due for rotation based on their rotation interval. + + Logic: + - Key has auto_rotate = true + - Key has rotation_interval set + - Either: never been rotated (last_rotation_at is null) OR + - Time since last rotation >= rotation_interval + """ + keys_with_rotation = await self.prisma_client.db.litellm_verificationtoken.find_many( + where={ + "auto_rotate": True, # Only keys marked for auto rotation + "rotation_interval": {"not": None} # Must have rotation interval set + } + ) + + # Filter keys that need rotation based on last_rotation_at + interval + keys_needing_rotation = [] + now = datetime.now(timezone.utc) + + for key in keys_with_rotation: + if self._should_rotate_key(key, now): + keys_needing_rotation.append(key) + + return keys_needing_rotation + + def _should_rotate_key(self, key: LiteLLM_VerificationToken, now: datetime) -> bool: + """ + Determine if a key should be rotated based on last rotation time and interval. + """ + if not key.rotation_interval: + return False + + # If never rotated, rotate immediately + if key.last_rotation_at is None: + return True + + # Calculate if enough time has passed since last rotation + from litellm.litellm_core_utils.duration_parser import duration_in_seconds + + interval_seconds = duration_in_seconds(key.rotation_interval) + next_rotation_time = key.last_rotation_at + timedelta(seconds=interval_seconds) + + return now >= next_rotation_time + + async def _rotate_key(self, key: LiteLLM_VerificationToken): + """ + Rotate a single key using existing regenerate_key_fn and call the rotation hook + """ + # Create regenerate request + regenerate_request = RegenerateKeyRequest( + key=key.token or "" + ) + + # Create a system user for key rotation + from litellm.proxy._types import UserAPIKeyAuth + system_user = UserAPIKeyAuth.get_litellm_internal_jobs_user_api_key_auth() + + # Use existing regenerate key function + response = await regenerate_key_fn( + data=regenerate_request, + user_api_key_dict=system_user, + litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + ) + + # Update the NEW key with rotation info (regenerate_key_fn creates a new token) + if isinstance(response, GenerateKeyResponse) and response.token_id: + await self.prisma_client.db.litellm_verificationtoken.update( + where={"token": response.token_id}, + data={ + "rotation_count": (key.rotation_count or 0) + 1, + "last_rotation_at": datetime.now(timezone.utc) + } + ) + + # Call the existing rotation hook for notifications, audit logs, etc. + if isinstance(response, GenerateKeyResponse): + await KeyManagementEventHooks.async_key_rotated_hook( + data=regenerate_request, + existing_key_row=key, + response=response, + user_api_key_dict=system_user, + litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + ) + \ No newline at end of file diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index aa56cc68a8..c230018ece 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -14,7 +14,6 @@ import copy import json import secrets import traceback -from litellm._uuid import uuid from datetime import datetime, timedelta, timezone from typing import List, Literal, Optional, Tuple, cast @@ -23,6 +22,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, s import litellm from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid from litellm.caching import DualCache from litellm.constants import LENGTH_OF_LITELLM_GENERATED_KEY, UI_SESSION_TOKEN_TEAM_ID from litellm.litellm_core_utils.duration_parser import duration_in_seconds @@ -642,6 +642,9 @@ async def generate_key_fn( - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. - key_type: Optional[str] - Type of key that determines default allowed routes. Options: "llm_api" (can call LLM API routes), "management" (can call management routes), "read_only" (can only call info/read routes), "default" (uses default allowed routes). Defaults to "default". - prompts: Optional[List[str]] - List of allowed prompts for the key. If specified, the key will only be able to use these specific prompts. + - auto_rotate: Optional[bool] - Whether this key should be automatically rotated (regenerated) + - rotation_interval: Optional[str] - How often to auto-rotate this key (e.g., '30s', '30m', '30h', '30d'). Required if auto_rotate=True. + Examples: 1. Allow users to turn on/off pii masking @@ -1558,6 +1561,8 @@ def _check_model_access_group( return True + + async def generate_key_helper_fn( # noqa: PLR0915 request_type: Literal[ "user", "key" @@ -1611,6 +1616,8 @@ async def generate_key_helper_fn( # noqa: PLR0915 str ] = None, # object_permission_id <-> LiteLLM_ObjectPermissionTable object_permission: Optional[LiteLLM_ObjectPermissionBase] = None, + auto_rotate: Optional[bool] = None, + rotation_interval: Optional[str] = None, ): from litellm.proxy.proxy_server import premium_user, prisma_client @@ -1718,6 +1725,14 @@ async def generate_key_helper_fn( # noqa: PLR0915 "allowed_routes": allowed_routes or [], "object_permission_id": object_permission_id, } + + # Add rotation fields if auto_rotate is enabled + if auto_rotate and rotation_interval: + key_data.update({ + "auto_rotate": auto_rotate, + "rotation_interval": rotation_interval + # last_rotation_at will be null initially - rotation happens on first check + }) if ( get_secret("DISABLE_KEY_NAME", False) is True diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 9897e17d29..b35c7e95a1 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -42,4 +42,5 @@ litellm_settings: datadog_params: turn_off_message_logging: true datadog_llm_observability_params: - turn_off_message_logging: true \ No newline at end of file + turn_off_message_logging: true +# proxy_config.yaml diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 23d9acfc1e..f51b65c915 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9,7 +9,6 @@ import subprocess import sys import time import traceback -from litellm._uuid import uuid import warnings from datetime import datetime, timedelta from typing import ( @@ -27,6 +26,7 @@ from typing import ( get_type_hints, ) +from litellm._uuid import uuid from litellm.constants import ( BASE_MCP_ROUTE, DEFAULT_MAX_RECURSE_DEPTH, @@ -3811,9 +3811,10 @@ class ProxyStartupEvent: cls, scheduler: AsyncIOScheduler ): """ - Initialize the spend tracking background jobs + Initialize the spend tracking and other background jobs 1. CloudZero Background Job 2. Prometheus Background Job + 3. Key Rotation Background Job Args: scheduler: The scheduler to add the background jobs to @@ -3838,6 +3839,41 @@ class ProxyStartupEvent: except Exception: PrometheusLogger = None + ######################################################## + # Key Rotation Background Job + ######################################################## + from litellm.constants import ( + LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS, + LITELLM_KEY_ROTATION_ENABLED, + ) + + key_rotation_enabled: Optional[bool] = str_to_bool(LITELLM_KEY_ROTATION_ENABLED) + verbose_proxy_logger.debug(f"key_rotation_enabled: {key_rotation_enabled}") + + if key_rotation_enabled is True: + try: + from litellm.proxy.common_utils.key_rotation_manager import ( + KeyRotationManager, + ) + + # Get prisma_client from global scope + global prisma_client + if prisma_client is not None: + key_rotation_manager = KeyRotationManager(prisma_client) + verbose_proxy_logger.debug(f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)") + scheduler.add_job( + key_rotation_manager.process_rotations, + "interval", + seconds=LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS, + id="key_rotation_job" + ) + else: + verbose_proxy_logger.warning("Key rotation enabled but prisma_client not available") + except Exception as e: + verbose_proxy_logger.warning(f"Failed to setup key rotation job: {e}") + else: + verbose_proxy_logger.debug("Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)") + @classmethod async def _setup_prisma_client( cls, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2b1e20820f..9d3726d29d 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -221,6 +221,10 @@ model LiteLLM_VerificationToken { created_by String? updated_at DateTime? @default(now()) @updatedAt @map("updated_at") updated_by String? + rotation_count Int? @default(0) // Number of times key has been rotated + auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated + rotation_interval String? // How often to rotate (e.g., "30d", "90d") + last_rotation_at DateTime? // When this key was last rotated litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) diff --git a/poetry.lock b/poetry.lock index b6ed7801be..c05400e139 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -3804,15 +3804,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.2.19" +version = "0.2.20" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.2.19-py3-none-any.whl", hash = "sha256:cd4cc5fc639ac24bc99af70b8b56b7cc0480b0e72a2a53638f557beb7e9f64fd"}, - {file = "litellm_proxy_extras-0.2.19.tar.gz", hash = "sha256:e53592e39eb0b9c3b6cd32f29e6a0a53be51d7ad31dedfb69b58c24073b5b3ec"}, + {file = "litellm_proxy_extras-0.2.20-py3-none-any.whl", hash = "sha256:80ebe03be210eed99ddab0bf8e1a14169f24a502ecdaa98f4bb5032ebe65a944"}, + {file = "litellm_proxy_extras-0.2.20.tar.gz", hash = "sha256:8dea657f965122490be480327995ce534e75dc96778cb3e05c562c7996f85cce"}, ] [[package]] @@ -9598,4 +9598,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "b775339256b9eb6ae6ffc72cbe7d17d379897a3d72f762cb0df58e4044cded0c" +content-hash = "1fc7c0289412c4ea4b35448c3a2857eb4c5656796ec51df5f20d4c134c383271" diff --git a/pyproject.toml b/pyproject.toml index 77d80732b4..f8704342b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ websockets = {version = "^13.1.0", optional = true} boto3 = {version = "1.36.0", optional = true} redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "^1.10.0", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.2.19", optional = true} +litellm-proxy-extras = {version = "0.2.20", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.20", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index 3c55f682a0..10929ab819 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests cryptography==43.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.2.19 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.2.20 # for proxy extras - e.g. prisma migrations ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env tiktoken==0.8.0 # for calculating usage diff --git a/schema.prisma b/schema.prisma index 2b1e20820f..9d3726d29d 100644 --- a/schema.prisma +++ b/schema.prisma @@ -221,6 +221,10 @@ model LiteLLM_VerificationToken { created_by String? updated_at DateTime? @default(now()) @updatedAt @map("updated_at") updated_by String? + rotation_count Int? @default(0) // Number of times key has been rotated + auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated + rotation_interval String? // How often to rotate (e.g., "30d", "90d") + last_rotation_at DateTime? // When this key was last rotated litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py new file mode 100644 index 0000000000..8aab67bf58 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py @@ -0,0 +1,194 @@ +""" +Test key rotation manager functionality +""" +import os +import sys +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.proxy._types import ( + GenerateKeyResponse, + LiteLLM_VerificationToken, +) +from litellm.proxy.common_utils.key_rotation_manager import KeyRotationManager + + +class TestKeyRotationManager: + """Test the KeyRotationManager class functionality.""" + + @pytest.mark.asyncio + async def test_should_rotate_key_logic(self): + """ + Test the core logic for determining when a key should be rotated. + + This tests: + - Keys with null last_rotation_at should rotate immediately + - Keys with recent rotation should not rotate + - Keys with old rotation should rotate + """ + # Setup + mock_prisma_client = AsyncMock() + manager = KeyRotationManager(mock_prisma_client) + + now = datetime.now(timezone.utc) + + # Test Case 1: Never rotated (last_rotation_at = None) - should rotate + key_never_rotated = LiteLLM_VerificationToken( + token="test-token-1", + auto_rotate=True, + rotation_interval="30s", + last_rotation_at=None, + rotation_count=0 + ) + + assert manager._should_rotate_key(key_never_rotated, now) == True + + # Test Case 2: Recently rotated (10s ago, interval 30s) - should NOT rotate + key_recently_rotated = LiteLLM_VerificationToken( + token="test-token-2", + auto_rotate=True, + rotation_interval="30s", + last_rotation_at=now - timedelta(seconds=10), + rotation_count=1 + ) + + assert manager._should_rotate_key(key_recently_rotated, now) == False + + # Test Case 3: Old rotation (60s ago, interval 30s) - should rotate + key_old_rotation = LiteLLM_VerificationToken( + token="test-token-3", + auto_rotate=True, + rotation_interval="30s", + last_rotation_at=now - timedelta(seconds=60), + rotation_count=2 + ) + + assert manager._should_rotate_key(key_old_rotation, now) == True + + # Test Case 4: No rotation interval - should NOT rotate + key_no_interval = LiteLLM_VerificationToken( + token="test-token-4", + auto_rotate=True, + rotation_interval=None, + last_rotation_at=None, + rotation_count=0 + ) + + assert manager._should_rotate_key(key_no_interval, now) == False + + @pytest.mark.asyncio + async def test_find_keys_needing_rotation(self): + """ + Test finding keys that need rotation from database. + + This tests: + - Only keys with auto_rotate=True and rotation_interval are considered + - Filtering logic works correctly + - Database query is constructed properly + """ + # Setup + mock_prisma_client = AsyncMock() + manager = KeyRotationManager(mock_prisma_client) + + now = datetime.now(timezone.utc) + + # Mock database response + mock_keys = [ + LiteLLM_VerificationToken( + token="token-1", + auto_rotate=True, + rotation_interval="30s", + last_rotation_at=None, # Should rotate + rotation_count=0 + ), + LiteLLM_VerificationToken( + token="token-2", + auto_rotate=True, + rotation_interval="60s", + last_rotation_at=now - timedelta(seconds=30), # Should NOT rotate (30s < 60s) + rotation_count=1 + ), + LiteLLM_VerificationToken( + token="token-3", + auto_rotate=True, + rotation_interval="30s", + last_rotation_at=now - timedelta(seconds=45), # Should rotate (45s > 30s) + rotation_count=2 + ) + ] + + mock_prisma_client.db.litellm_verificationtoken.find_many.return_value = mock_keys + + # Execute + keys_needing_rotation = await manager._find_keys_needing_rotation() + + # Verify database query + mock_prisma_client.db.litellm_verificationtoken.find_many.assert_called_once_with( + where={ + "auto_rotate": True, + "rotation_interval": {"not": None} + } + ) + + # Verify filtering logic + assert len(keys_needing_rotation) == 2 # token-1 and token-3 should need rotation + + tokens_needing_rotation = [key.token for key in keys_needing_rotation] + assert "token-1" in tokens_needing_rotation # Never rotated + assert "token-2" not in tokens_needing_rotation # Recently rotated + assert "token-3" in tokens_needing_rotation # Old rotation + + @pytest.mark.asyncio + async def test_rotate_key_updates_database(self): + """ + Test that key rotation properly updates the database with new rotation info. + + This tests: + - Rotation count is incremented + - last_rotation_at is set to current time + - New key token is updated (not old one) + """ + # Setup + mock_prisma_client = AsyncMock() + manager = KeyRotationManager(mock_prisma_client) + + # Mock key to rotate + key_to_rotate = LiteLLM_VerificationToken( + token="old-token", + auto_rotate=True, + rotation_interval="30s", + last_rotation_at=None, + rotation_count=0 + ) + + # Mock regenerate_key_fn response + mock_response = GenerateKeyResponse( + key="new-api-key", + token_id="new-token-id", + user_id="test-user" + ) + + # Mock the regenerate function + from unittest.mock import patch + with patch('litellm.proxy.common_utils.key_rotation_manager.regenerate_key_fn', return_value=mock_response): + with patch('litellm.proxy.common_utils.key_rotation_manager.KeyManagementEventHooks.async_key_rotated_hook'): + # Execute + await manager._rotate_key(key_to_rotate) + + # Verify database update was called with correct data + mock_prisma_client.db.litellm_verificationtoken.update.assert_called_once() + + call_args = mock_prisma_client.db.litellm_verificationtoken.update.call_args + + # Check the WHERE clause targets the new token + assert call_args[1]["where"]["token"] == "new-token-id" + + # Check the data being updated + update_data = call_args[1]["data"] + assert update_data["rotation_count"] == 1 # Incremented from 0 + assert "last_rotation_at" in update_data + assert isinstance(update_data["last_rotation_at"], datetime) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index da64a866d5..14b5833f14 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -23,6 +23,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.management_endpoints.key_management_endpoints import ( _common_key_generation_helper, _list_key_helper, + generate_key_helper_fn, prepare_key_update_data, validate_key_team_change, ) @@ -1091,3 +1092,73 @@ def test_validate_key_team_change_with_member_permissions(): team_table=mock_team, route=KeyManagementRoutes.KEY_UPDATE.value ) + + +def test_key_rotation_fields_helper(): + """ + Test the key data update logic for rotation fields. + + This test focuses on the core logic that adds rotation fields to key_data + when auto_rotate is enabled, without the complexity of full key generation. + """ + # Test Case 1: With rotation enabled + key_data = { + "models": ["gpt-3.5-turbo"], + "user_id": "test-user" + } + + auto_rotate = True + rotation_interval = "30d" + + # Simulate the rotation logic from generate_key_helper_fn + if auto_rotate and rotation_interval: + key_data.update({ + "auto_rotate": auto_rotate, + "rotation_interval": rotation_interval + }) + + # Verify rotation fields are added + assert key_data["auto_rotate"] == True + assert key_data["rotation_interval"] == "30d" + assert key_data["models"] == ["gpt-3.5-turbo"] # Original fields preserved + + # Test Case 2: Without rotation enabled + key_data2 = { + "models": ["gpt-4"], + "user_id": "test-user" + } + + auto_rotate2 = False + rotation_interval2 = None + + # Simulate the rotation logic + if auto_rotate2 and rotation_interval2: + key_data2.update({ + "auto_rotate": auto_rotate2, + "rotation_interval": rotation_interval2 + }) + + # Verify rotation fields are NOT added + assert "auto_rotate" not in key_data2 + assert "rotation_interval" not in key_data2 + assert key_data2["models"] == ["gpt-4"] # Original fields preserved + + # Test Case 3: auto_rotate=True but no interval + key_data3 = { + "models": ["claude-3"], + "user_id": "test-user" + } + + auto_rotate3 = True + rotation_interval3 = None + + # Simulate the rotation logic + if auto_rotate3 and rotation_interval3: + key_data3.update({ + "auto_rotate": auto_rotate3, + "rotation_interval": rotation_interval3 + }) + + # Verify rotation fields are NOT added (missing interval) + assert "auto_rotate" not in key_data3 + assert "rotation_interval" not in key_data3