From 24c6dd05bec712b768e3860ec380a02417279ca2 Mon Sep 17 00:00:00 2001 From: hiento09 <136591877+hiento09@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:12:30 +0700 Subject: [PATCH 01/17] Add icon file contain image size in file name to fix linux icon (#2344) Co-authored-by: Hien To --- electron/icons/512x512.png | Bin 0 -> 38651 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 electron/icons/512x512.png diff --git a/electron/icons/512x512.png b/electron/icons/512x512.png new file mode 100644 index 0000000000000000000000000000000000000000..289f99ded85e1e3cc91b17af6b607d2cafd97cd7 GIT binary patch literal 38651 zcmd3NWmjCmwr%4Uf(Q5D5G=U6yL%EW1b4UK1a~L627 z(G9z*man;Hg()jap&}C?gFqlu8EJ7<5C|Ih6&i#H4?M2jW*>nElAW}UBM5ZD{q_T8 z&oB)90>XEa&~{R@HFI+P>R<|Tb#?t@VPolN{MF9%ldXe!`WZg~2t)>w5f@Q+%Q#te z|6-ugba#1PbBFJsSV(Qb%YvJuz(BFo1ShX2j&b1EkHYO==}_rWY1nlbu6yJML!SpN z0p?aUY0PXeiDi%p9zkhy>@K2P$52}Ci^a0nQ4;T{Jd^LkVYpPaz^!}BP2JZ zV_sR8MO7MZs*_}@1M>^Z88lMOI8qIsPy_)FMoUmN)_*=B@H>I+eESBTj1THB@GOTK z{Qu}rXg|DdyxjjJ3qdPVE0K;wH%N-zgfvs&C4oa_zxq)~Wv-ZAJINH4yW@wh!M=w_ z4PyNIHCkdvfZ@#Q3FG(gDJ%FQ{Fb1KHif4Tu{Nb%)b!j%ZzD;>FP6s@j~&!L{D zuEZtV&HH_&0VSF4=6mO;JB#4>ZBOysTNs*Fc&O~tQ_E?U>(a9@gh=sX?^&X)XFEP7 z{KNkU-*-W;S;sP1TbACova zh$++D;L%B=T@gKY#|5fLeGVeR_~gxN;6ME5MK^n4Gq?4)LVSrlGsh11yD9FXl0dmD(^GRWw?u{ja8t#4YqB6QIT$>iAH5h5rx zMC8 z*}?FGu-osNbO`rz-qMNr%In4%PMIc5e0Rww-_dZ#O`aPpBgCTiIir7?ok<*%&xiNA zq+TL1!htE7J59&t}go z!DF=a*y&JjBfZ*NL*6SE+(abh_gtFw4de{gNI1dtGew0xUROnOu*D;N1#LAZkh;Js z@(=8SxOENAY>j9&zV6UFu^-c{Zfq&QcR$&Ez~Ao5lS>w|N`@_8r~<5>0I~Ygh%5rh z5pf_=#pfSJ==R?2Z=&Jv)yHOd{w4;ao|}`kzVYO&i5HRGw4;3Eg?d zS;;rip{jBkmxH2x{_-^Dpbh-%_^5?iGEc6WpKtKzUb4B9^JIbh@7d=8e;*NAkS41i z%bfF=#d^*@yMABHvRTSrhUR&#P%#x5dwAT8`9?fDU07&L=8j21hu~esKxIitt#lSa zF=BYf@rQUmFV$0IAehasdO7~&X-}SSw@dj*4{ksaRPJNE#&0j^%v~2}<|a>f!_W5d z-)8NV>YT(9iF13ao>~Ja_9`z5YI3nUb3Ex3P!A%FR(4rg_h7A%tI)*(o^$mAvb=AP z&=QhII_`qyUWOMq{$cn^@@^f{Ff9;f5LxOWDJ|xNIPPh#nErXA(wYPj@L_u0WrBjy zA+QaU4Q!{ArT22Lq*NshNXp zJ1?qA>u#Lbsr6Cy3=J7qBfE^;We5gJr~2C*ufOZ<4ZZ}arcwvHdH+xc3j4Buphs#9 zxRlUQiRXAWEL{v(2=;9uNl)Vwm_@5wOIlklP4z{(_Y1rvLgoBMdXaNXu$EFn@Yu5$ zmBh^aa%M{7a))}ZPOS8!VN26EW#))S$AoS4WyEuG=Hn^N-A)x^I!m=jm?{+@QEpB; zy)DdiaQowBWlE(r4zwq7C%sqnd}fOK!NF5}qJ}HgF#Y-&wOr>ct9_1sVx>oSJQK=J z{2nrHU_~)ZtBQx+*#NXZIgZRHcA{O>Yo`vSa26e69iKaJVuy|cq!dfOkT+rRM$KoZ z#7CDS!Vro#SIYCO3G&*JnD(|Fa5ChD?Bn;wV+TeX7yH`uFK2P3U8c2{$Y8Q76Bzxo_3*qT6kVx6Mgy%7T#TVk!tpb%^hYnM%Zb5PUC{@}D~eR&VrBA+ z>$zge6Q$-t&V5`aWP75D_jH({DQS#-;@2JVGOKQP0cD>)t6#;uo~*$ zoD?u<8hmZ7z~Om2`N)xqbHe-|==NXJ`&!X96f(_bhyhVPD2Gcq+%}BT;Udr-HafKH z^k+%6V5eE6+kvn?8C38M+(YeOYO^DEUrt@6lD_S|7Oisq*h-%#_97)4>_oJjG!6s3 zjVrn2%H~cd0|s3nT;N32WY^{|Vf@)o(*y@Edbt$kjC zYim4<@jgGl6#kM`25VOe(yH;C`OF=fWn+tvE}BPLqP0_1fX45Ilw&pP*_Jv(!$t4# z710w~Jn;D3@N!VR$N3N9B9fm5-u%2baxKQ7x4)ZEul;KsKRScVLnUR#y6!lp==~&8 zwoN#1mw)JX42k~nwn!9{l3mpyKdN~~HEL{jO2t!JE%!Luxnb22TBNT&{-Z)jO_5TQ z!PEq(FxoAVc8xiCrDJ$OQL*geACezV8%N$J_=lm_`NsfQ?Jjj!3gx)T`X`)R0{)b~8kqvM3a%@VOY|5Ce*+>0{k!qai46H3Y z;q!iBj0gT;8YpNfZhHe3+pL$91GJM4aRyyWsjL8L>HQSXFvP<&@`U#{9aDN6tADIl zB;sM0^kcLJtk0kgaGfJyFfTnB|0a<5eBm9}F^^m&KEGCkIb%2QB-IGN8v&Hn++1}T zCLX%YQsV=jqZnm3(oy6)oJ}-hH(D6x0_-BU=CN_RvyCaj_Vp>Au`=K49;5vZj&{__ zJ(Uo306AH8QebtvDJj>nTxZFZ*T=eEwsZ)9dgaAq~9dG4z zauw~509!sQC`CueltdQM0DG5EE~%0}lyfG@7cdP8<1Fr8;|Ae%*T?i2TBbZJ>ED#= zq_&11lhcN=cwl7({#OBL6{q9{r$=neDgXpMX!_gPW0$oeL}pkRufAt$wHs0 zT3LU$JoG7*I8<&{S8UrI1BF*HtNy|;<=ctuC0Cn@A8hTsV`(eNG`5XlmMmf07hQ@* z+3@K2SC_=0&S5}ySV*$)#3!$rFKTWB76SU%q8~wK^GXH_RYl(XcIMi~AhFng-~xqB z;wsDU-v`e|oib3$S#`B}rk$s+BIj$t!)tI~c8&5HiPY|3S=g%Im7aJ#zyU=aSnsP8 ze}5o2xO>$C!fYu8dGDq)%lf{5#LU9?)?#+#^g2ygt~?}mmvn9~CKu|j(?h8el70~S zd~;hVIFpCH&__({v7q(DUn;~y(Iv67iu$;;C_6>K_-H}_ShlAf>M!_wFt!M5H$=z} zu|O;Y;gCo>c4+cmH1A%>IE6XORSEE`KKs?Rc2pkK;k}58DMaY=h86_>Sba>D|5S) zZ;RFL@Jh7Qbr)cg=W5r-)RxGj_c!1~h(&cFjv!zX;N!t5TZEDb-u`x3%MkLYa|rfF zhL!tarY&EsuY#c~7|wz1(g}%Jau1JSz9x5aclVlWA@htW4xnsR@-vKBw4H`fTzop+ zLz7ESb66){M5rlJc=;5}#l=e@g!rVLXPT4=(tg01u~(6Dqonz8wHyNv^3Mswtl+{M zYK=O{L|&}aPUi81}O}@QW=Pj zSBT$^A({`9#__U4wOBq5st@_x8yL9kY=k!gg&I%eS~`dztfw%Cy;%B@NlsV^k4SxCP zc*GGPy{V|YMy8>$U8WS!d41al_D826p2aW9sQxps{9=P4YSF>SQl_vigp%i?boI*! zla(@D%!XuqcpwegPKz{z*92!Hy#;A-MRA$s>fP6K6o$~tbAPm@g=yt1% zlJHqIiIB&Q%h1zuVepKir7LgTIeGreho4?2XE+Rz>lX`?lar_J8AX#e#U}FK95PQ) zmHz>ib z%>VHtpcV=K{>zmp&wx5fo_NJCUQwg4PzYsOH|`Q{_*DCyw{t(fxswL3z`2w*k*K@9 zx{ZFl*IY><%0y|l2h9Pr+Ku=Q*b@j8PObgO(Qigx(25Osi=BS*^%scsPbl%m3IJWe z4>uXVM@54^&W)PVx^$wutgUQ#pvtlHG2yOJXWE=9)a{3S^(|sj-TaxUsb{{>8^V3ig(g;t_KQA z{o^|H^lsp=CK$fDQj=OJF!T~774}DsK=tf3*U6g_?h;T$99q&YL9j31jMdcsMsjG< zzRA4ON@+?S)n~ zlYd#7&4R_J9^o}fYs_u*J~uFr(r}5t6@B*#yR*g!0k=LIlmG@jkbTn`79yeJdX7Sn(=>d+q$0LlDsq8WXF65j zKh!^T%z&yX(YLhUC?J?gMOU%4mQgxPTeC4t63MB4vUQXLTgI5|ATy%CixWYWyTQ6@ zHeDg1g-5N>%9&_i0C`z{UW1!+(`offHMJjuK(b$2n{A<-WtP=b~2kJi=u1WI83d98_YmXhgcn))g zbT7j2umV1<%`*5SZTPn;5~K3_($rZQQ2Ekynu-Xg(%|_j5Dw}(^>z87-?wCzSxpswWekGPKa4b!d4o?y`|C5)h)hS z@_dtbm>9B_Fn6Eb&u{s5__5q0R31cTF&uX9a>;&IFYi84*TAxR_77=MrU;Bw%Inp8 z?V8IcW$+kpt&2Z|`Kwl0QD3>kIG4XcsNwxP`+Op25xl{qmriNoI_|#-qDB(=Y~%fo zk>C~nU3jWPJ^=e|4Es1Fo1K{y6kWeU7e7i3uJl~~l?|nd&qP0dC!gJRmvJ`2^4C`? z5C!*jZ?oGnUZC?$3I#^u#GRGb8#zAd(Cehd;wkLmZXIzh1=B(F}RyAEC2`_Donw!tlGE)AXzBvULzk%KZ1_0IDhGm z@O-$d@Jf!lL`L){TZPS1293q{qEs*W*0AbRtn+52sv)j z@XEAcuP|gUcGZ|=bx0&qKW$OVJHGfs9En-HP$RS7YB{Vc^15uIfbp{Ctza2x&_6RN z{9#iLUY2+&OJS5#c>p{Fgpcp;i+Uoi;>0~@VuaAqwoI&G*|~`7dj-TlDg%Q-q4#U( z#Vn0QA?J8e*n^`M@wV`7W7;AGr6DAFcY>p7oHv5AokJKQ5H|Gxd)-ZaUKMyuU|)Slm95GlhEAg#vffCzU)V2d zfyH=fDRL)HOsRE9;4AfnvtpH$O(t^Y%~_Vy-rj>n&}mX6-1OMc@nZd(qs%+=rGn1V z-w1(`vZ8cK09kV#a5$W{AteF0X*}n59YhPL4%EaTf28wqY{?MbexX-niakS-eHq3? zurI$_%51A&bgBaOXvQO5G{d)()>g!UyM%Xa9!w7KnJBPnttejKU!NadE0=4%0PL$w z*rXlxNFuYl(nNx)D(;Kt1OL6tmGzXN&J#0<5Pg3GTS=Pk`W;8;#j5WbeZV^@WRu(4 zp(~Ndn-S&4>$4CN-f8u=JEinTw8=iGk$1!{&w7!cl(fA8j9T+6UdIDzF|2nc9x=vr z0E{hSIa%=3m~K1ZAs>eY06`}bDk*O@@SwkAP-2-dA;EGvomZl3{Zg2g6c_2eApgg( zf-pSx4bH5K01)puv8R*D$@J~?k5%+69#nk{Bq}6je_0PF$ zhwt!Y6TiPML)ozgROj%TO4HChJOC229o%U(RbUuMErB!7T4RC2=5L!9@`BA%$$iUVe?C&W7UiQIV9dLXpGtnL8zdRC z0;Sm=>n6}aB>gJQX%IUqrV-*)D@#7+PR9De?1vbZ72z4xC9O#cxV45h(Ssywg}}M# zH!(81TD?9~_$^qiz+_kv+jwdk0NqXPY|+KV`Dd!&SJQH%im_;wpJF#Wji5OF~#jNA$8;7UdZd&l2vpZN;5KeKTWQM=}W4 ztA}0RZ93JzNTuLR=XsqLl_ur6NmakR^)#-D6?4C$vpb|&+|Km+x3(&6*qAu&O$-A= zhVV{O#64UX(_MV9{rV}SFQy!&v=Xi|=)$?>E3S@Y{ynRolL?}*{{c>?8D1R5;Eut) z*&DA#;5ozUxzu~p2?mAfPh6U%{`ZyUCwoOxTc__4(dFRR(L-6rno+y1YuqcVfPbT~l^KM+b3VRTcw#~fZP~?>Tem%mt=PtBs+w!=Bfi*f)N;;H zG(l+}pszEj`DsdYt zAo?B+CKmMF+2aiTYx+5V)>4I^`&&leO#7RIe-jSITk1$zL=R=qe*t^R5K8YD=~qe~ zgntfDDk(i88(|%&y}t0JzDluEeCfr-ZZ|DVyVK|#G?m7kT{u^nOMljyybI4^obLic zipD>-mP&YzS$GZtYEIZvw%q)TreKPPt6K%;+&*`4HKJton+*`${kJDko_T!=_Ik~a zmEx>+@@i`yci@l-WW%E5^j%4^ZT681Dqrmz7d4%7n74rPSHY(pi+Kx85N6tp7uLt; zn1W|Y`J->^D2}wf;Zj05USPcRy?1LbR>kMkfxut=QKu^RMp$*~z5f*a%`-V~cs2w( zCVq{0%9~cbAAR%qyOlyC5G0gW5`C-LpJ1kynncYe1J~5nFDyM~})|DU>)~|GJLnvZCefTSJ%A0vtiqt5(S^jO3)l|Q? zk-uW@&W8wQK|aS1x`s0360diNTi{lE74QM=T#=%qK$Hty9(_X{=+2&c>SJ;8N zKhA!=_uSC&RT>!rpn|)Fk;Nl+z%s*o?osbRXp+Q~c;p6zturNv$>%>3Huz9p2q3*; zw9(?6KEfqkuj|mEcKVJC$#@aQy-&x-T*w6X3BmNJa%V<3_Zm2zD@~398a6i%Zi|z* zXP49t+6Zsu|K*@rhyLgST+bS+zrU>j@$6iW1~vGedNJ1<1zj#FhOzScr4`Z0T8B}J zAi^YYfpQ(z1)ee9X&VfY-+{W=_|V~sTkLkume{i-^tRQ3d>T>-R1Tpqw`wLUmlfTw*N5yvgb3H!6hdFlpUxl)@M3EJ)(tE$FpItZ7dD-1rl$>z zVWicwpDg?wtUDHEr^aaV&!ysdVEUAE;0lAQ+B7Ll1Zt+bvYa#!B>B@+WZ~Rp&_A2~ z7I*`j+a)gQ#zXO!(2Mn_tEJa|kSU)XHJpebP2LIUZgGp8|G=A39XD6lWi1#xY=UxQ zRmRvnS+6uCKEN3=M7_Bo?o<_;xHV}Dqh`~70!B@mm-rpKIJB*Wj7@CS>)Z1&hL5B4Vny_3DL_*7lKf-=hq@i8~4lEEo5F*rJmg+mu(n-(m& zc5Jys!QI}~j<`eZp`CrltT5b<`$Op$N3Xctf=9F>A4eDtf7nvi6tPRE9=&@;yidu| z_P&dv$ueC%kW9D&3)YjDbcr3f+x%J|fZNq2Fg>v}U_cR7Lb-|$*kuTRv?idUcqD~2 zQaoUiy!~Y7=%Gdx^JD&=HoO(bEeDh7teqZ4bD$dRSiNenUwWitNIg?@x&xs5{F%2~ zCD)H`&0cgBo(RUIzMoFYb9B`f{00qF1`z3I>OzPoAnf1qofGf_#usqlDR#|XQe3wV zdBgAj%~$-XoUF?mySvJLR+@Y8$n28#XELY)Ltrlghr7__&D{b9(koNw2KtY8Q8) zLEvfDmA9d-T0*OB*KmGT0gc_UnA*w;RugF4inXf)cYb#gL-%3%$mZR*gU1`Hl=y=s zSuR;?Y!n*S&T>AUTzHa?`bun1-D1CpIv-Z*0a==U#t;`97Zf!JXg|HRT0} zgxJ5NqO*IOr_hqs(ORr0St;L_@gHf+hLA{B=T0wBvoXi#)YL6nc-xGmw$!&h3cK9s zk(iAg43U3C%Lc>!J}@#ZD0^px7}#7-(r!hebhd*q6=4~S^-Bd42pt*}G8gRk@M9H1 zB?@H9>+kF{77M`}GUHDmqNxUa!gUW~J}6qJNScqp$j2e$U$%1L?W+(RS1g&1p~zv( z<~!a!?%`n4O%Wau?x8!vqVJNCWx17zVfwtJziA~8Q%88X5-yBM&imLW|GsIhWF9k{gkhWya_ zG~q@f$w{I6=6XR^bM|7~yie$DTAL&n4RN7wIQ;Wp$`Woc-d2HWQ*^;_R+8D`|Hc1PnO3#+oPev*Sgh z%UUbQ5-E6$_hoh;#{lzpZ|_suJVBFpHP8dV#AevsU?Y%rOqRFPrgY@b6P6P?LRa1A zA@X>qhf>MWlKQ~zhLG--b%iA#e%+U2ez(~6l4#@+9nqRaqc{lhs3zU!6bJy%6dR`~ z>soHO{$VkGg60u&%OPqa#LXorM=|J)GsS6?fsFZ!<3WT5zMe=5V9jEzH%NtVm<$oL z68f#&LKS1XFHCd;J#)}ORWnwXUX)$LI_9N%S~z7ixyI?a^=no==il3DDdsws?Wl+BYH64&?`>`xq z9z%{1Cch<5z7*#@ROKXk47|t+a?ikd=DvwXd{_UksSW4NBYBV>AXzwNjtX*JDZVb_cKx z57zb=SvV_hcwL~CrN5bKk!@zUJL@M`_e7ED9G*wo?ce^HYpJ(}P)5%xa1f1^1GOzB z<`gS;O`K^WK#cez0gJHvc;ntt&q-*t?+#^$C9@7`J7L=Fu8TF8y@p3cjprk7>^r}T z%A1ilO$8cC*U*cQfY+F#Ph&^!z_&1UgSBtf6>{HFY!P(!dq+f!oLZNZb8jdEYo^_n zG@}Pp%Om8c;0k*}AGl_=BRqek1bF>lK4W2&!}l3sqTZaZCHojmk7w3^ibAnA7V_EZ z-2I#6oT0JWkRSr=5egtsWYG#5?MhpRFYN`Hp$j-;rcltzPW~9xiecj?*?Y_5c_?>J zgT|siL95FRj~<*Vnrr$l|1RS3TP*T6wfFdW%`l@Hkde2Swa{A{Y)zH){C@{uJiG!JrgH-K#^D>!TbHcR1*=|Rz_AyEspOkCGsMSmby1I8j81<2WMwF0i^`} z5r>?q=0h-L)O_6~!cF1)`x^q>j!Wt-m&^bM={I`$pC_`x%mi!#VQ_rtsH$QS5+1cnvjGjqT&b ziYQ`Am$oGbW(7*wcLQui*`E>E>VWR1X6eHS5lrmkxMO%TE4rWb#-1?Qv+D?jds(V@ z*U8{XJIaM~{n{MAEp45s={iWtIbA6eD4 zD961|mi~r&pw=gu$9Th_*VW~bB2f5p9GJ47xV>h#&+|z1{z^_9r{%LWkAEncCiWXK)hakFK&bq|{i@b*%gtE{N@~}%75&ufLBhiY?nwU0%Q<+;OWZYe zcxRy1otq}J`iLj6$I)A}dG|i`JoQ-}uL(IC(Na=DY^SGlQTXq#Y4rw*6-jDG%J}T? z=ZVQPsL<&iDJ}mG;voV6bj|sUXq3LEF*kIw^io!DR221|T-WaKUg=THk<9H>JB{$9 zFO#0tzNxB8CBn&tFaLFtd_K zV+X%<_>SnrNkE@I9suLgLG&1A^k24LtXEIVg#@S5-=D&g$zEQ&(PrrNd>g`)4S9ce zQ@-y$x3qd=30|`+MC<2Yi@Zn#(#;nuF|vsCETo3>-eVIsA%@%APT_Nom98`xa=Xi@ zAcxi*>Gq~if=VPjL8}NKLld2M%kepq^Z9>WP{tUG5zxVdzgC}ZzkNm>X2R3{MbHcW z1^DNI!+&RsWJc+ynP76AJ>tt^|o;C!(!lWz;<&2nh^A5gGASuS(D{) z_vHMV#&*7C)HH zGoz$~(i(H@D3F_@ubaeWc_hg~(LyjMM_0Ukq5A81Ww}L!-^&FWor>;6$d@ZN->LXK zJ*@0ocl9ZkT+@)ezrw#pxDd!5!r@+-|0d)S?hilF3o9SA_Z`sqjk{ z4V*pCVAn@zD`eQUqr#@VjZZs6W*smD2QJB?v9wH-hPodbqY!Z$*63*PLY=;6eem|& zX#1Hph%Rj^P{D>hkZd)tmvqctD((|b?n+TkDA3@FuS}n-=x7HPnUC}^`j__)NjM*% z9${TelolqXoGbW`S{d{<0Qjic)*f-^3SgvhB>c0zL<|)=d;6xitGwN|y{bl+X9AhLjlm#I04^`~ z4cViy9Wd(*M!yyh-nYYvjAMzX>9%JGZQ2}bm-1@gy;=mou)l#cYNeH8b0;jIoOH_RUTt8Hxx}ltna8a=D066vPqHG z&$TJK_)v*u<&K$z|7P{Wbt`snH=1+`NWD%Xa4?~Iyw7J-0_?gJ*W^PAm-l- z8GCNj)~}wR&7_9hn!Fc#^%`JASO#=A)8v9&aL5)?+SO$$Ugz|mZKey?b5}^6-(iUd zPV2(>)^`7{Q0(sD+Pv0sfw~eSrBRYCh2}yKCHiFDBmrJ~2T!(~cNZrX4UaK)7%B3R zM6fx0I!=g8K7u#g*8~2z=eXhr;OV97Oa2#b0GJ!gSou-s@F{RO)&Sq=J|a#VT+Wc`5=JUq3VL`4WcG4aMiog&w=(%ge#5{KrOtXtm5y zd(O@u+J8=Zgpw|oyeDNoIQGltCKu6~!wei!kzy6in2cgrLK43A!FBf2fALiNeg2A1 zYN_)PV#%gPW;ns+m)lVc-x<~??}a51xIKdDH7A@e_HO4T#b-K8M_gJ9;p`)W&Tbm* zr#W8p#{_eU%o;d%(~ho~0sdAsvjwFGx%Qe}6Ro{{;o|gt@bJu}&AVDGzv9IHq<{}z zLVjEh-&lu(AJ(u#5w&U=6YlV$b|-qx4dL1eo!_a&C2BMdZIm%q^dzU@%4GUWBXGVB zb@){vI!Ex!CTO8EAg!#r%FFS@t`xC4F6Afc*xn%(`>1@y5T&$hlB)Nazoy#;$gU5S zDmzCt9C^$z4*(n=5qH74D@)!ngwHY`)eM(`k_(`<3sIad)~n~Z!R_D2=&>v(7RIRA zAErO&209H%2Il2OeG@fPqTw@i2n@gAW6LkRVM~W` zlIq5jW#d!5O4h3pZrU+{a|6`KIGiqsHqP2yYfLmjsM8lL&h?OkVC8|qJZ*Z3T*fa5 zRyq1aKHsUQK!0!V8z{hzkINxK7rN^a%0QeUPx)&Br~Fv}YK_)_@AIJL1TT|0>lN~E zzJ+|6b6Aovz<5g1$V0K7+vyN0{<){awnA>ds1M2JpV}$N&CSBwY*BZ+Tchl%i^V=x z(kRl38F#=VAFpAkeQZs-oPtc(fixzw#WTB^F>e*}iL-~+3B;Q_4Kit;_ z5e(1XZq7$Q=vdRwsplI9AQ9q}1$$55rfjIrNE+IbcPmU<{F_UK?yABKb2z_O7qiJC z0LY0}y=&yr&{zPtH$jU%RG$qqrZR9b64vL*1^1YL5mes?#t4XgPW}6pV-gr_sm1Lk zQgarN+jZUYy*iz`bai>5_vq~gojI^3(o*XtM3pS4ypNR!NvC97Zcw&z#irZeVyU9~A-{?!i`Hsa^e1`mWy>OH$+GL&rwY``>rm zds~x7Qk~L{WW^^g5A9+6FX4?`A6EYa^88zECH50ICC(vk1KdaSjZu&=0`wQ^7N;yL zf#;3~J$=T9C|a@?ck?3OZ~1=fQI364ZZ7;_AHYoY>tZ+a>n(iD=-hRDzwm{w<8J~O zqqU&ux)pM@pSVXf^HJ@f*gF6$Vq84?_vegJS;-bw(UMqV-Fla%ld=z2R4p>ht z;=~n9p)+hMmINGlqH75RRpaw(xLsR=W%~I7Un7knY&dB^w;JFjc!Vd?bO~$P^V~6n zyZJmX0j3Iwb+|hzO!xB@PwV<^q9YN*M|A7__}=z| z5=>aQo^CrD?7F_n?^gNXvrIlHWhDs7o;J`khxXexA~@*)bGq=Nk)Jj{o4uYWc$U?( zGC00}xNOH;ye~v*Ti_wQ2WM&cT>Q``oXPTbz&}V*{vq{I^V@G8bh7S-p3@=a4H80j zq|NSPP2DF#oiE?CZB9)PfhP0##iM*C=E+D63)Nx@D5i7c-D&in+>8=@8w)S%hnBPHx21So8%I%m&XB9 z9f;to#O5>5@fgCyuF11;!SIQd5XY|V33n+OBYL%ZS7AR0+zFNwI}3_l4+M(M)|A1F zy*_D076mR*ECZgg*obchR|3z@!5efNvR4oCu)* zrP9GIwDB87STVX~Rw*^WG~iVdR>hU3b)Kf;tdxV}-C-=mzHB2QZRi0$qPGyWaQA#T zZKv0ZmeC>HCwuD!?c@n)!~si2HU$20()})slCqVkZ>s5cj`PlNgB$N@i&0NI_F5e6 zHX%GkTk3+hK?-h;cuoUPOe|Fj5`>+tH!>w?uQ*HTX9O_rNoma@0cfL-3y?pPJihD` zAv}eE*qSf(3`K4z9=jF~#C71P0-)Q5Ao?Q|4}m58JOFgSNB?p}DD829j`imU#^%;M z9GDX(*3foqCSB#>&t)?P`z#S+(1_8!dxD&?d-dZWc9bJ9?d_hgySj0G<-XWiY1&lu zJB9s?C;zV&wvpoqj~bwg!4OMeZ8pCVe7mH*_md1!tfEX#(7A2OU5$0^>~Z!&oFv=U zt>Hh<5lR29|GWw@`n8?p4UI?Q9Klj@lx_$q3-*Kt0Vso8LUZ;hnpB>L?R$PAnmt<{ zqr`lK4y(73O_a-b26>e%l%5TG}6 zfk>Xe5rjPZXn*;7yT0;zeX^>uC42*RQ+4nBr6TQ%W=L7(Do(j3ezm#6tdl-21 z!>cQV-Fr1+_zjmfle_MuT#S34P(%|)8rH4uNf0z7T3=l^e*wX;v{ygmSoC-luTFkI zitOi*PnPrf%0!qg%?cK%O-#Gb_@ppI`?SA@m7yycclKflr6dtpDDAAMZoXxQXOU`L zXZfTgNU+j#?^tIRB=3DJ^VH#))rCojy?O0K-!!^fg>{YU*@{zXtk)<7>QV%SPiUX| zXGPy6KON1~iNX8{3TLadjveL1E3s{@sDydh(<|YAUmzC21gGEdN9DQBmp`WQD*n~w zBYJ1#fC2{sK_UJpl&oUEP@hjnh!2yiRoOEqFkVM@>m1@szC2;6=Y&+|j_@x-Fr#&t)VDvMBDsXjb$Q_yXmFe~A1E3#?+Aswx&o z5sxxs1CBBmqMjwWD_<%MH(BP05IX}H>em_J=N%}j3(wxMcxBXrOj}xpTuyx%l8+au zKxNDT@d0kCKxKfXpw#OTWFfmhbF#mee98nT zZR8lnpElzBvo1G;l1WIlc>mF^G5UkD#hmg>*cD;MHNINP1C{3Y3`Z;ebw0H#Z&!@{;8s>~Y}*Z8fSeLZCZt8zns z88l8u%pQM;wc$r+$?)Bo;bw;~X0xo|ufFCGqnbM4f@4PXI~V9l(uj31@>Z&C4*>>O z)^98A-&}t7N{waKRYb-hkFqVR&;8M<>W@P0!8Ao$m^EVO2(N=R z=ZzbP7cmrY_v*k%(Wf5g8q1FaZ2s#fkTcyT7bYEbKd?`z=LYbajgZv(e*V6s)odmm zBdGwP&S>%W&%kL#XEB+`6gHff$%`qiV7)SbFEwD${w-JeU^QYZ4a zWKX06cEC*g6h6`s^YTk0sq7}5x(@OwnDUZoNUJx)~#dqfxa$wjW{yCyVxF}sm> z9;dFazh>OcEfdk+Q_K739|zjW<;f^uitTseD;TRN@A!ypFK@;lzFCK4Dp zS`}TAlk;amqFq^=L`=#)`FMZ(+Ya*uawX(_>lacWUHUH9?OS$Yy?ONAUT2T}H>jl% z2DLH-$F?SQ1QuWbA>xbu$nBm~j%RWNV_|Y=w+CNQJ*Ee3g+@|jm-++a9no^qj$C1Q zsz#?R&ATJPUf51&aKkGaAfS#p+r@wZDgUhEyp*jsh$9-&qx!r%DG2k`V0(Q0-2XP+ zIMp-pJ-*qFNdbMbRfoVvq`$eTrYhz^RD`$bCWE_m%p7j?TqfdM#qw|V z>eln-yuFpb4zvca86H~z=gGS+^DWb&`!5&3^VoPOuCryjZ!?4mWmh1W;&lBVu4|ry zMX|9s&`f!x&%xq4etTxTM~&jm_Hn#+hD^_yrNJQOSCa(QeNG`JtAy~mk~+8>eFLs1 zr0&@~zyD9-1gI==4>~1D>7V z-b^2zfZO-U<(X3buR4}_coxnLzNN*6@YCy@H@T{PuXv4^&e6 zLv74oc~h34Z%_iwznqkvXsQY*95Y2H&cp!3e!o!|ZH1f`)&-hHIl^l6FM2Ny0 z(@A{}`LW;H#kg;8AeL`eLPZNmhJ&tat`0BNdG7^|x~axccA+W#Z_T^=nWP~{BOtS6 zXul@^BxwFhqilFT0Gf{=Mq%uuY|bkM298kd6of+2@9UmOJF$S5b#nsrm)}N_yulVg zhWtsAh>1~Qt3wtqZ|V_`+(xuJ2egyDv#Nf=n43w|9db}uTmlh0u2>q_m<6W&0OaoV z+cZ|(5hh!Y6NcqlDBrGRKlbp1e`g}h0QxWZ+jF6%A3$4A(bVO8$Aw=!WE95BXknAk zqhRWhsY2-b^R5nPkjhPzIc&oG)u4v&8=`kmUb7c`ARj<14Mo7MJSpu9D!{XdW`-A`^ISrFuds;#l*sc3@IwqFUA-g-2NOA+Y$7LE@_!l|a zI?9g}38<_!*c)A@ZifRF!_bYSf}|h~0!m1XfPi#&r$|ee z3?bd!E!`j>Nau%;2I-LQZkYG-{XOsB^V~CcpL6!wYp=b}xmbReq4dKbx9zp&pu6wY zyn9A4%6%qaF7DxSW!dAbu)!@z?vlt`LkRc%0!}I)q}7s`k(aU%jE`whB{H4VRdX?r z<5zg_oj05K8oDsSC=QP=%ALjoef$K8_{0hR#~*y{_FKjL9)jY+ob#3;G%c1aoZeap zv`k~$$JZ6$9*mBd=cKyU14(FD;h>Zn+Ma5mz3rJOhDBxf-v|~(Jre1I-~{Xv+_^Le z@ZGg8@$NSZ>~>i${LZ9?A79kHunTXPXwSc_mDkSXWB=g!%7qiv=@!wuEt)@Re?dRm zG84v_BXP!_P@bCVmKj8v!oOD6(bOpVGC~^jIF*dyvAmy%D>O9pv*S2n8inia7TV&_ zvlaZHin}0HwTS9X7Sr{w2XR?q%q4d$nq&3WjVj#nB;X&l;bzWZPyG^yjNOp(0Ppt z_@g^RUPr7=NQA!ZmoI>hHPHO=Z*7J2&(Q-c%5xV?qc$Mi4?ji57j7ymI$$vQ+Om_U ze4#T3w<9apf{p(ab1;@@m3?e`DgmZvlEDT6TG5i;ztu3ojkK3C-MSsTIGiAkSu{6~ zSyLe}T|Q2Sb(Udk6565Tm{`1dp2he_y{|1WoNP(nP&YlRoQ%2{x2bE`|z*BHkIn5 zzISUT*|%KYS{Wedc=v!S*FERHGvo}rlh@np8k!Hs!xS8?k)pH#3Bo)=dt;(Kr?C8E zL4(DE5nr{T-HF<-0-Kywy97esg{WO`2`iFlRp;`UJYRIWOUA+Xwnt}sJ8T_jY?Mz> zxM%ggz_gC%91QaxLrhCAb*VOZJ4HI3QmuX151N>(wive%Gm7-&5IaGl*h zqBGWP4tZ@3r8{QM_*{VnZI$>TN8!P?ls46P5^#)Ck0 zg6&_HzOPfZ;Yw`yQz&`GPTgm+A75%~4LRA0@sIezqbK=cgR^j01&L&J)f;PD{H@kO z;h7IqiaBD3@WCW4NMLvL`U7kDCo;Dj6Uq#i1*&>YxukT~XKa;<(-grN78zOH!_s~& zRH}nvY|pt<_#Us9AA*kq2X`nEhyY2$3gMH~_G}tE2^MA%DstUo$pKUvP0ccE?5%IL zJiWcsH#|CKiuAp2*&xwzLQ@g;WYr%MkH4 z)E~P>v*Kos)TjfokcA(w{x-ca0Z8pgucb=tR!{)T8l;sLQV9dx!Fy8seHLa9YPggZ zf2d?;xZgm|sDEU^2;^a+v2`x7JNKn)oR5UbwDls|s9b+XIRFH!V>|rj7!H5Vc6K86 zkqJV$Y=iR8#q7k3QCU0EP|)$1`%bYd? z$?tBHfC>!)=?NSn+?SD`d+-bAJLBqzNKBZD5eJ|9k|Xr0wtICms%^z{9}s`ism0%R zcD9uC7Dw87Feg=_Al)HavB@EHWw!Lx_#w6|l&qew3SxPHxGZdLvO=2W6+8Do15idq=25a_`OR4 z_q800TD-n|FrzLCwV@*Gm$RWT;-N4*1k6XQz}GktiTX^R|3Mu8VZ8iUg3oV_;58rl zXh-8d5}}W3Qi53Vp&imvc-c_E-)!k^nxG@w&RH|!(8mlpYg?r225I9PGRqgn{-uhF z{6p$-1&T^qxKANES$n*XvQ>m;d}wqyw;RX&~X6{ZN+bxih4XGpvV+ zk0L>~J2;)*3h|vv+wD=F*Oyqm$$3-3@hcT=iPidHhi#iGIdNkA;h`fOV?*OifK)}G z;9ds;5>?af4ZMAk6WzC+I~iwLhiQ_IsvGP5l#cB12>rsNzC@Y(xG0XRwq9@Dn? zOv{b!`f=9P*yliB!!lqsR#{wnkbs>@j8mPF!zZniem4~gfq%UIr1KKT+HSQ+=`|gm z{v*ue4gugZ{60`-hQ?Ze2$lV>4$pa`_yuNAr2W?`ty=V}`r$lOq!b-84Pr%n(Lcs# z5f%~#`(O55-emk6s6;YcEUE<<_dcdZJ|@6d8EE_El z>#)LFD)hWEYRWH-8R<643V-jh+oSkpB-Wb^b2>L+0MANd{oPJDJ_c89%&J$Cg>(Jo7&LZ#XHfyrX-IKszc5s330tfi5#{P=(`odi2GI zTK9fzV&(oGb;729RgH92m{zn-RFCBwDpI2FGWLojn&K-{5%a(V*?*Oc|6X3zJWsx7 z2H;Biud~s_7>i!mB2?LPC<1kuLN8^nD`!4ZG2*L6gCCOH z{@-WSPAU9M+bqeYcS6q_Ea`&Vwk&bku}Fw$b$7j9FhkqsxTDe|L=SfFXxcp~Ueldw z=Zo4TnWBDknH>8|&InL*^m3(z@Q`jxsRLYLt5KItfFH`oKTS>qy~*u_nF{y|bN7yb zmEdrPSxgNkF`Jq{>gv%Q$!JJD-6R-(B;-kTJG~_s_Z#olXaaW#ysm-~ubgi~fYY^I zi6$0+6$KVZkfL8`UnOPvtM=D4Pm1HU@&?$jaUZ73NpaS z*>5c9GHE9RJ#jh9g1Jp8(b67$96*KO2Hm|}Yt!DG*zS64v(79zrTLa<3xP3NyD}v} z1}l+(^dAI<03{GhK^6CD~^Xwe~%#y(3?IIV{?{Dy2Bul4j zFv1X2aKXJ5dew1kXpFM6#@Tc9@tDo;WOGadlxvPVwh((@URT+i=Lzk^HDvQusTCmB6c#*cx z+0m}PV}=nhyL`KheQsctCOPvLH(L4|PJA;jXTaTU8kEU4O!WGu;%>*EUamLf3)PB( zIx|MT>yNh5{TGM!ELpw-wMek}b!SY;yyvKxB@%osu8$MmSc2Bibr=Q$q6d$fB|z~1PiwX+1kuLcpY zR${XMWP%+|E{RGvwak1u0`2^O*wNd$(eZxj%;r@-@9|lM9?{UEG3gMY!F-E zj+QKBL(g23whQ{EA*8`s5Qdicc$N!r#_g~n{C^ufqMeol$qSPh(RLaT_@l(ju@AVH zDV9EihRw^YYT4^sfbG@OLW0&zQD9P(&{ua)JO9DZ5rEylEW`-DCkgIV)phiyg5<;p z8szw5!>80qh0O{kAo#!=m&SN$ZEzU@K%Fyp%4~J*S%3^h#%P4^aiayBe_z`pJIvr@ zefGI&0G)KJ(3{mVMaZM|9z);t@DK*XxJ1z)GtNJ91Sa(s+TVaMnJFGM8r`v5 zYHP1F3oP-kY>Irw$YNmlCD(Ol40QfJ4i`56@e2i}47lYtJK#j4+!cri;}ko2{I%vI zQ#eXi$drYN`ymL!)fF-`K8$@3OAq&W1Xk~Y^zvQ=J4E0BBmxSs^;#*d@8g>=b!O+p z8be`=0#B6aAHcB$P9ZRKCJ^CEJ+7$umO{H-39V^Dto? zAh0f;d?`f5@mahBejvi)$`B_ec`PwPs=R15vH#^zDl`wHvm{7gVN$bdL5^cDcq_Td z80BV^G~XiS^LzY~{mgyrx*(Tw^q)|@10`IN`*8t#A{;(JPJ!0#^fJn2QDMhsAkyj^ zVYWe$6l|R%Vc0jBKl*Uo?On}T+t}nvnA-L|E6{8A4JCl|x4cRAxgkD^*voKy)b{q* zbC@R<#C_8f<~?>0H@n`fG$UUr)!`i2NExNQm^h&XJ6gqTQc4&LZjT5aWnah~uFq^cZtL*!9ga91sH% zY420=Ief6dOabK(avV8hna?87e}ZmA+sSbyETvI8U!uj4+6 zyyB>U6f~e1eozPQ^GbI{3~kQ^BAj9hO37#_Cf`o}6q1S~E^zVky)Fke=WTAGXH0Aw z&Lyd^>ksotO?ERd4*|{ivj+A6PaeJD2qSZuPZVv&=Ju(j6sbgjurkO#?w9ooje=qc zI>Q-x1nci0x3119FZ)t-t|0eX>mBL<(2c6WME+|aoIlVvNqbZ2;*C#q_91&x|9jlN zSSpH7N)$fElpAN-9NVan44u3Mx}-UoMen|odlgn(G{*D>^v}6+RXRe zDQU~w^72Z&)sTcm+vlcH&XZd3#v~)}Rqo%=;}N?5_eZycu2<<X?OyfiTA+Bs7a_$@OrE)wQ@-X5O3L*8F@WDRWnmme1fYdI7lAZ_ zRBAso9!IolboH}#r=(mb;rqjaqSUfT6Wz0~oXZ(87B?`&wc$?_4LnZ?%U?D0x-nUQ zl^xv_rs|Unes}`-07y){z6|vu|NTucH0$o30Dq$I1bs(`PgJ6&IC9b%nLm;8-#d7N z`MQ>Hc6L;k((~2eJ(R$HZx?-Q$=v5DSt5y_T#^-cV7R7VNhF*l}!$(XydLk@SzGa8VCc%afnoL6YLP3Z^#d9q8PWZ0CJrt zBsua}TZUFWnMW%pX*6a3`A?sR{!-$=D00V1#E9Q^Lxo7+Yup8FMIJu%{5{4O?`XGt zRxz)t&1N6WJ;@SmzCZL&ONXRGj(??=duLDn1je(@wh6VV;eI(?FPFE>R+&0e&=M1m$FFMZM$xG(WepBp@`84YzG&G>E-<21EH!ia$i;Fp3Rbjpd3@x97^;@HZVX zU`*Iosui^faBRjy;r1^-$AU*ksRL3|-XrNhZ{uUoQ4An-+^+xh#^-C$T5KZGy5GEM z)%AFbkoYJsIc2X*hLV%i?$lH|a`L90HGj>!Di%YNiI5sLfwP^1A8yD<*T*IPXf!A~ zKg}i`c-10X0#H_q1eoo*99JXop#Qe>}U%sfVmG6tWa! zW|R)(7I6*pN=NWT0U)+TXaE;L%|(%3yG~g&|KtI#(mwZPgiDpP&9n=~LKIJVnP>Y)-2%kUYf;a1iaRk5&Jjs>4zu*>H1;(85 z0>GUGK>G=iZWD_{t*1Sq3!edv0428I&H*E4+<0Q0fFCw^W6zmS1!&86Z~=~ABnH8P z<_v4wi_MZ+zaI}1fB7g*9b&?26hK4XCdN?@LGCC1jIq-_``CfM72!ul)=iRWnWB(K zobTLNogUD`OC9!}8*^3CI(2)Y`em6twj{g3!~5h61{e@6+rc~EuMGTsQ&IeUsKGrW z5VONiRE>L=r9<=A7DPbpV^x>zXAk0T5qtW2eD~m4(XBrRRk6FfU+A6EmYP=7Sp|BQtvG4W{Or1Z>BLCs%NY7M*dqm*F2kjv8nR^rX%&%uU!GfJb+e^Qo z#Y>bw9J+e67CG41Tt>jy_zDGPJ9@)_zjP*mL!4{okI$;em0}z4QFHc*C8A#%S$A^T z0kwA8mY6XS9+&&OcDH%a=E(TL8;=)lQ0odnGkGtodP^DLdsFYu0h`(CrtqW%H+Dxb zI1XIk+RO5kzB=phijg)Tq zw3|cPkEf1bG6AZ=D6J?fW|b3)zC15MN9VI8s#_+@s8;qSXeUXwMb37CssvLcZtseZ z9A;E><0i##+;1;;{Y^;$x$S!98*;+YveFnjj#p+8egM8vIHYX5DGQZF>HsmEVEc^p2T0CyO+^SiMcSi^{+W1SG+@;vI~T5NWqEGJ9X zc#Xg_w?ct|k2i*p>p)m|D^L$fnOQ+8VV1u&ir#7R=!QkD1T*+#*axW6)e@wsq0k{| zagdFMr1v;Yux=L`NjmPm3^c@zU1?lIW8V1>1_KanbIH&-Ytd9DJ zCIT3AKLPcFG)W7A$hSs=?>G@jeq{muFN$0BosQ;KZyjkj{7=mnj^4})K@D&AGWJSM zzb2Y3AswrOQ~PNGuMYd2>URob$b}8wxM^QZN!k(ggcgM#W7X4~D@2}@{DHzyJU4jV zsIQH#=1!801#th|nTVS_tp>#E7v{#ABP;S*=f@^?0Dvflm7B*1^cVEYsI ztg{VQ_sV2q$@jAacUxH4cmFq>uooIUNdR7&(*`)#R&a+pa| zWKTKP_E$-zUb{sM3HgjR{58X&@QRFiBZjpKky1mcgn_BneG8wTA2H#-)fT--$u6D- zAi&J2`qDUHhxUP!q;5EmgFjSxk&mj|IrQtZP7M#8OX}H)ga6G2esHg@kG#u&5lHaA z8{T=c%|5d=n!c22`2_Mn9(N8}R(bjJqd(ZP@Adg?_;~&i1EEd3*OE1b^ARom2KOG^ zkDiCRX{2@r!Z!HIw^jg-a!NeDzO<8bwpHQ#i3eN~R>rxu^P}6m8tIw_w2b-X>WkT$ z(!abusr)K{;A~YZmnFRqdgm07i}r}Xe)V<*p|AgTC(Uu|))c?1Ns0TzTwZ@in|jC} z+zS)Q=U4+sjg&vFD=T+IB=jJ9I@#}-Xnq^@{`*3f#90<1>iN!lx5fDUj8%2t=HGaEz%^f!dylQU4G|U54Od3_y+}{d((5)Znd-IgJ z+}0>kS`EIdOPV}13%%hLdPg17O`a*iPZ%3Cm?M(JV;zeGOHuV)jxZFqG!!ZK9$B;= z=R8@|f3?HR+00kn^?r3tS$tqh?g0frlCNehv&m4Xop?N+O(Xws-8>!;(4P&u;1%9O zOkihD*N@q#urDR~fTF}i9Z&1jS-|(f>A!cVJAt3l-XUF=fbU1ItR48ICL|*h0Yz46 z3a%?I(noCuqVl%Q!3#;d2OsSA^k=eE6n|6=yRtjl|J%7}^jT_d{nX8Fo}T$`eJU zYFqfvPIKB;7U(Zd9HPB)K3c+H**H8&kh-A`z)Xm^CF!#_mc;}T=Em=DjC@axNE>5A z^xo=R`8vG)TBdf~vA~Rk&IqiMg4&^o&X&@_Ndt=JH1EghVAp-In&ycqivXEHe5T=F z^{-lgmX3%V=`Fu9CeL|u5cRq7zX(-|=gW<#D^zY7fob4Cm*mjSuJS@7D==Z7UP3F{ zt;%v+U8e!$ome0=s$6jx=lhEXyWD;#--8oBMU^@6-(XgLf2c)@zKDR6@RqgU42j5W zaSuto`nP#JXQ`q~nVG@;p9JLV$;A|SN8bMpbu~V7;q*quEs(pWyr2>Y zjkmDfd84TqES0kFh>56;{d9ilZ08UwCH5hK=hmUS<1LJBC+n)h?pbwdJqZJR%f`!T zl8zSo@#9kFvzGfm;Vi?d_ z(L>rN0YTi;4%~u?I!~^}zL)&W4$0In#@d;CRCvgx z4HDOQ9lNMk(Sw&1Pot^0t$pwH0r#7MbK(8j(uEvtWR)2xAx``V(1!QPe*oh_B(pOOS8+pm&(A;+ zkcMX~KMEI_AV#Vs)d5|YrumV{{V%gY{fJ_D$&}sRHpo4E#?sWUfSi#+fCAKSa!~d& zX7IW(6$MXU1<35n;gWP>sdI(2dLQaVuoY3D$|Z^U4Dc1@feH4c$qI&NR{r>s;gv-^ zL!LHL1H8ItkB5t1Q!!vI<~7~B#xI8cpTPPr>r?#~)cPnqsJFkxiZV`4)R5a3zn01O zipIWH8v0Kf)A@F*B@NuixSBN{&NQ#9(5T-#vKlDtFA6u+*H5cvCNo*OF%f2%F5esljO=LfZkgalt+pAvQe4PF|sk+IB6hnp?gzI_`ZPQw#%(U;h z=d6>p@hULMpwFegEMYKZId$T-!XfguRvMMaTg5j#56{n1cpL@?4ez67EZshEBcT(M z&}j>!qlg9OW`89EOON6sf8D3GEI?xqnX`r?yz@iP*!lm68}Chez;2= zP}NmhytV?OKstgi8Ucn6u%uVf9f?yBYc8^u{Ivw&^zV>NF{<@PjnZ24KUU-vs~ z9r1H?b*jR@=`OcJIYNQRHcxD5!YdntuEHu5r|m$ox%;R4}GBiWmiJ~HLet;y_1+_z!SP?1#=>Y z=ZYcteg7D7&mMQ5jj%M=m1|wU2j&ZJM+aPnfE*GOo4{j4dF zs$`%?jS4)1NMZo3^0^~LrxEU!rjOa)SEc0D7A8w&eYn$q6_3UO$oBS-T?>Rjx*FxD zYE{hF2>@LAnbGx+EQmL9YQhz%qpEl-@|6?#qS&$^urUhWVQ2e{DijrITFS0QRtPuq zPE9(N1JpQq<9EPj5qR9}yUG0X-=m+fY4`rj+qDV3ulUqD<@KJz+7q3I%RhzS=x?E* zt@SAW#+#lo{+;zW<2#EyE7Orgu|! z;vwcn@zoRJN*SzYQNt?Hs_hb!KwY8Wc3#BtMW-3Dv`Hw$z6#mUd8>m2Ak?otDK6cc zu2AEu54|g-;MYUp%L#`zl=Q!*19N_iQI6m8V(1QhED6d>m%PgAG-wFH71Ka{tvAZM zTn@cIk=kVqoo};pV__Ty70f~TMvKtwkx$rxmj@tyFjdfD3-yLRDfeRNCq^L^xIa$n zU_g$s8JnuOK)Nd6L1_&cYIQRvWPEtPE8h4?4fm%?!9L{L$U>=6llxlo3)cX=nu8&x zGl9C!JtGu<4SFK~69txE);Xr@u^RAzWZ2d0UWEbxhGjJ;#Y|xug1i!1nAQ)|4*0p` ziL3?}4k$aOy|WQ^9qZ)Tw-qt?og~~T@aL{|WH&V0)WL=wM#E|!1Nk(8m1^Fn*#Phz z{WGFsY`s3H;c|~;)kHqGdKoyT#r;j}kFAgCbU;B5b-z16<3&C8qy9vpPPg^JbI=7P z`fI&Uc49LNkZNvKL#gAXrzhrLqm@Wfmc@@NntcBG;f$A2>a}Ch&YmGZ9(BMB7*hs} zcRHy3szuFC0&w0fuVTL}sCe2(fB(f6|KnqC;f!?wp5&ig(hui9ni z_v=X<_OFYY@|~Rtz$7mvsVH`^SwVc{15Q#r^z>j^=UrQuiZC$%z*fK*7l~mIm`i{4 z%W+E^`(P^)N`NYvx9;_pChm`|7Z%Af`qR6B@ol%_(GqL|$k~7Jj5t6DM3fG7FCrOYek+1$ zL>&nIxn=>>t6kuaX{c_zv6-vmBY4pKz&PISquJzS>2k4@Ur-6X4gwJ(MuSZ1@|+Rl zV``wHQb??3CzB^JY{>>PIT^l#RMhduBE?~qZg7UV`I|<-wGiyK1tZeXiB|R88^wXk zg7vl08uQaTaP`Xeb`+%Q7<6>fMkl^J`|oS!q8MB~6P$(ip{V(LMii$qmQNHp z!NWPFLyE3cs=nbijFu~H^AD^L-sy7P_p)m2{|U_>p`9_%caSx(=~sJ%f=|0=85it_#I^@xSF9mJl}76P;v{&{}Iz^z|*=T5tDZrXJ#t#+gP z&Uu^=5c$FIBBLAQwk1{bIZuTphUOnA`YNhgmyhW9b)V0^EoA*4O*=IB815e>8Jm8e zSYrx&ZnuHg{3koZTH*63@UfapGRYkx+Sn2P-tJbS3777fT6BbB2%9GgJfV^W+_{O5 z_~|Y7b`?6U9{D9+>=^<#ZvofRP_)l&?&~RPg~ee>5EK3oKo63528b5LOY?C6pdanr zZ;82J0memXm_J|=SnwTE*}osDbqJBJuuz=56tqNvCeN}X;BPsjZjKQd9xo410CCS? zc$A$7x%F^$O9g$vHNH8P`ndZ@lHmuJmuv&1^;kfsa8EH`>`5FwNm(1;ji7IZfpVKi ze2G`Z<0QWgz-09fd1VL}_g=#2_fYca{j2PvPqFy!f25iD9x5$s?vGTg5zWfY1A^%T zZjfjSE5S$gADw;pOIT$|+uM}EJS_ILvNT?+i{ z5E8GL2NA+%t|&nI77FYj`3(lFd07d+H~c$J>4;MNv*eVHO&`Al95#VxVLMl0yIEAF z%=!|OH5gnKfvN5oqU&tLEMUW!ub*rH`ClFzZ~2icHUOw{x+P3s6nPzS!i^N4(8tZ5 zToc06B&H^aL<_%^T67%2Q6}c5>$O|JNY)TtJV2*8jZCp`MFLd-B2(Ihl@)x$ZszG`1p=FVtw>)9=xGM%*B+Kdo^oRyZW5!j1z7K=DtExMwq_6!A|q-prxyw@+J z5wHI`2T|lk8imeYSkYx^Cbkg61L#5UZbwE;TVhY|RNN)8si-q7dz}q`;yTePy1$f3 z7FeI;{#MAUU!cm1Ac8M4MT)M!9b82>qZOM@hnQJbg=w?|WA{+CsrgH|HQZvF$w$W2zO16x*Vy-tf@ zxzP6)3B0B&Ms&UDk-O)zm4?zVzzn6m^6%!U2;4h)uvI{3V<#Y_an_R;CVwSP+f?wN z{Ymt0@ySH7nJrG$NLy1mx&yngy{xQkc5_r!-WGICg9`y1nwswuNg~17D1KjJ2?n%W z5D#3JMZ?|v3Rv(tEDmcl9Uo=<-8ihI%9I@HKX<}YK8h#g%qB>|_~#%02Uly2hcA}z z-{WtBay$NP`z4mK2w0oALkOFDdIseKeBs6PKL26&xtjC5Y%8()tmxt3(mVO74aqHJ zAQ`)&ZfKG_O87xHVW6=BtKJ)Tg#qM3oktX`1W@4I-o^RZ2rsU>8#v6Q+1t1CJe>x_ z0WY4ftYl!Wc}ek=EdsqRJw^&+Vz(2{3YBas6f;6p^Rqa~Nzm;}0b-i_A(y1ahjW;e z5<@3Wy`5V>NqdYE*M$ixMZ+K3mBf!MGwP-NJ8(oYU$q)t6RKelxo41}7Y|uorlNbq znDf7e->8GPeqaDuwTQ%mXYV8Kp3k!aQuRRO{&?8PVu^41Pqf2A!jl%6Bz84zQ3C@%bX}(_OA&MRGqjK54eYkpeczdlUs`WY)>5Hn z{{{tLTHR+0Kaa+K$QX=7#=-2~dXolNJW;_mLo{OGzsI)96qhT+^ZA!-IKR4GQQF^5 zFgSe{Ad1K}!?AvF-g&|z!Xs#9na5_cp?;CT;kT^NAXg!^1~2z&nUkqvGeVLIq-(uO z6@pH@^z-8hCowHbQ6Uz~v-BpsBm{)&iOwyzJd6M>C5#JI`Ih%uN@Z8W8SiJtJv(QR z&i!`(0;+J2mkYK5HKJ-|8nh;<~F+KEI>tkg(M^6J28R4e7O)rni&-4P)=c0h8+-> z%8z;HS%6n8fMxS(?te#+vN9A70{PLpNpE`>`K@4cc_9m^r63k8YxnO%BUv~-Xxn2F zI8Ms7El2F;2B6&{{}El`Gw!#Relo-58s$&(j+C)XT97LGYS%lvlxlsWeg*at1MlWe zx1+RgOY0s;3-7Td^SRGBNTw=p@+x)E6y{D*$!28YsAHniS^vcm0j*{X)~+Q-WL!bs zGf<%44=Eee3V3FJV<>XH2Ay-XwWY}DKvwMp_nr4{{%fAxL9Jn3e>&QgO_~F51rFV6 z(HoaQuBnJ9=yl8!(X9qOsPQ!55o@KY>NH5tSijP1(DZ$z zffY9n69b)XM1^^>(?kGZKYOF1Z>%Yz`aBN4Ysi7XVPC)o;98_|?uoO%rKY_6^Jo_o zJHh*B$~;ziG_p+k0(x<%)t7=lp^J=DoS(UJNozLtNi$eyS-#_80^&c_r#q4oJm&B) z=D2|V@IJm*uz%>Ruxx@SRd)SQAK!O`YiP0wg!CbYGyZz;qjH47M^DjhVe>Nz3==Sr z?y=!8bRp&K$dIqbG}VJB>R=IO^BZmq?k?-rd>!5*>hoHD%89Jd<&cO-If8PY(JnN& zZKU3Wy?7zL;H{q8v}SBQhDp#su8=*ZsQ3TX{Qlapb&R#sa_}lb9^8bJwc6 z1C}{T3?3|d9F|=_4gYO9I{!HXcxCB~xKXW2*Z5Vkl^A&X?v1=C0vj+4TPaxO{KRc* z-%lq!+*R~j)oTD=zY%z*mJm?#vtkfh=?qhSJ?br!splQD)_BzHJXd!OE>NY#${rxw zkO>8-u~E)yK3-UTJ4}Q6P08te@WN?7e6%C&6a$Bny|G&59ST>?8XnY5BmycvDV~M_ z?^tljjl(Ftoo;q(iTfU+Q&e6*s7^S%Ss4{CxXXk8|fMyoor7ebia-CU{WJg-8R?{CfwJC2ribb5+}N&(&q@uGip&1o?-N1ug&1Em2ji7q z&G>h}+^3BUlZ}1|L+#@l+|hF~7m0zvcyQe6=6i+q(k`pnpAz=WiWLj*^;Fw7U6G48 zkkQ{O50MUM3d~;E%FC!%@F#U#`G3xzADKI%X<>BwM{m+M$L{cQp7RS2>j4VzF>`}D z!1%vRTc_45@SCz>V1H5mP#c}_=~!!neo%t+j|YC*`_^;m)}HV2@&DZt!=a+y2M9dF z3ypm;HOS{ds#1doy8#m_ZriubW?NEnuzjGOZ(lnBt&PQ^#6uao3l!&49Y3PSL99An3`! z(}KCe%|-mt^t}+7G(v31a2ECWyJ^G2C0Dnbx)F25f3x7MN;95(G!{HmcJuGJV%7b1 zOU~XSk~=r2?PxyS?I}t0U7oIYm|B_3asa9A2W{lDr~b|Dz9*+6tOw`MoU53R&`7eT zZ&UPDUad8mzG_;;`>#zvQE~C79=dbO#s&(p7h`&18WoQZi>}KyM@vz1CVi1|@*+Ot zSuVt=k_1M$a1Dw8neU8svTC-{g0tr@IYoAyUx@xSYXGY9f#Yrvf>R>ZLZy~b%J zQQ>-Mx;}(&CDoEpzzI_i*~N~F5JDPF>}K@oVOPmk=%1VUWO}XF?c^tJIqCH7@c9n6 zEw?PAUKu@yhkDudr21;Rx-KpY_}Gp!v*=%=fN`o$%YtX@rnKX5EfF^WAkq9J{y{Zn zyWR(PmaAG9T#u(lTIg=3r>!EV84n=X+tXK~IX$b(pUz1O3FqE%L~NB;5S^a0^Qgbs zNCYjL;+NgeY(H=7K?TXQ2Nc&u58KXyE^iUly^B@!&~cU;4Pa!@x{5H z5>j7d^SYK_{0DY%KVD~{jyomQJnI|(WkMeNt^o)>DIPDa-8%lN&t5z&pHlPM=zGDl z&p*>fXpYUQt8T_SZ?_POPvelf>W>hvu4syt zTvQDkW)R>UZ!tvWo)E~iKjTME&;4M~@_8eb-&XiFhwR^RU)8S+a1xMM8)qZMXwGF- z$9}~h#R@rOe=XFX`9gotb|}X|ZVBYqiaIJ2YO*0I$gMMruN}hCuh=RK9vYf92E!J; z)g#lcZ}+J8q=8w-Zy(lw!gx|IEx(s2Xe+UR>^7~@^Wr!=AA)m?(uP-uDmM9&KNS1Ihr?)D&4-GoNb(zqJbI z3}?9|QUGlzUEm}Lxdf&E8+0488keo~5PSPpLNf3RxjpB!L~ODZ&@~k}1~|@MY&4B>|`bmKwHBvs;g&?d7X}n>YVN zAY~^YcZqzBFINe0{s8(J%_TZ#PZ3DpcTB^HfS_a$*N6u(z0x5wp8vS*N#Y()*AlD_ zLEW>srFZql8f;n}cQz2Nu4#}|>c0PP*kU99M8_2=8N+mDM<>RB zr%UQiAUh?vIX;3fK5oOuSWw@A&cdwu^tUx~d}QD5Os0h|0+kxt!v7yqnTYd+v)4hY z*Q9Rew8tj6j_cdUPCLN!aKDmcfZGP0x#e#Yms_>vHzOUU zC@XxRE_b3uXXs54E^w9A`vR85flk`iRP=aLeJH{aq{{mTdQ1pP25X;@P(DQxTO? zg9N*2F`Kb($P7!U_a}gu0JCYnSVN;RxKHA6{5jNK>v!Hu>sjH=o`OJ6)y+6dWgx%U zWguRNyFk8myr#?&ndjGiqa&&C*@8Q}2wlUZ^}gRHZpi*As9fL@OXq>8kw4tavv!^Q z-20$hUUX7Jw<8TKSfMZh)~U`N{0com@%QiBmr-l2hW&ZO+J?&i^6%~UHW;2-17E|n zuS3m|4aHHHUi`JS5ODAcsgT!{*JdX?R8NZF%?IqWU51joK^}P|drq%_awcTxvbx4D zlSlpjkF7RNjV&tmjBY?>04AC+t~glp?F?^yC&}Hj#Io0(S2BJQN(5j|eycIx0#V=R zkMvjyvWM!4tnax+Na(kJsbHrP;;DLx1|9~(b$bV-F@ehx{X;vqc@XS#!}94KPQM^b z1N%QdPxr*Vlg|=xs7Eg{+pNB^WIbw&GY&TllT zs~x0rL$c{UYvdTEC*sdyFCA++1PP-8G+*#h^(rv76g|%az-werO=VBcplcT1v3uMY zL1*%;PLUVI>2SmC)mt-JHdfX0;)}&NrNWXD0}fNRW9~2vle+@0(FmJbCdR0S_5Gn$ zRVpwa$E$7n{S6bmqXsrz18)1j`m(#kn!2?2R+Pe{zPG54lIj8@R(rVadn_hhF}bgo z{hGc}%tt@x95drx*VD&L>1s(e_HvMG|1DAlE#V(>YAn(>7b+oP+!UY0+? zSL2JE*Q}6-wa{@RI1r|BPWINa+KyHGl>f=}7SLHT``IVd>dn_0g)chL@ZtA2gh2^8 zLvi)Oxz1qiKBod2{^`a>x$6X&=%*emp5{Jt3>b*uw@{hij*^k?n8l={%;t(K5)lwO zIEp`qu?&<2YXysmg3BD{o$^iDG&siGjrgaw9G1y!ev~L2Pmh`_0!&Sc{tx=H!R0$z z!K^$fC|1GaD;snmh$X_Dx6Gs_79P(Cv)l|{|43{{=0kA6eN4VuW}Jc}&HG4+VmUbh z-8+^V)}_-fg-YkU8km_?Aa18iPWpDkNTXhq5!dd<=bZn`$pGecaD$( zl2G_%ix3a4$YmP(rH@6bY!juT|L{0Jn?4!t&<*v@0_+<& zowl}wA>z$E=agezETA76TD0*F+RtIPY?I*HO1|5ycH8|>X0ZfJWUK`Rg3YdIT7J*|N2yjkjH;rV!40oz;r(*JsKaz4QU7(`hcWW z-w%Qo9-+kL(RLae8P7tg=4>z3^Z=+Tzz1(l((TL9meBiv5%bj-<99US(5%AhTw`fu zWvK~+M3_F-9ys7~w!l=Y)vz|MZYQ1@vx(^ah?n-Fc2g5{=nw?dpgT( zgr4@vyC6v1-+Xnt5Lv=l9@aFg5jnf)L@t`@6}A}uqFP{1Q2E#)RZDBogtpB{@Mh(1 zz5s%L=gXJNv2XcX<)WNTu;%f`2Jucfjl!+w!u9L@7*Kj4;;DQ#6rj1t zGk5cq({Ly;2qH$mRMjURB@bL|B{=2bYo=KjOg& zZvaL3thAfC+jV|T$-geajkFQm-zx0I*G&>F*WzU$kvq-^29l8vBAvFPw-=L6(rDYQ zx_=?Do1QZ*J z3eB{tZ&v)8Z|0ojZ_(}q@N{?20zxJ)AhWc6-ob7DJ3~xFn#P-TgX0I0v8~Tpwq4*$gCIoA;vOIf9h+oF$KwEfxCK*KbA#x`Kq(Gi1s1Ku-K` zw+Qq?*k&d0V4By(k}~@FUI)=uCHnm<8~q=zQ>cv{FdRIs0<)uwcmm1y(a!8&cRs?x z_=tMa#yJ|N)djgyZGoeaH$P%VVfc6S;Drz%6ZHtxC7i?p<1u`ONuEoITU&-XAQJ6I ztLIgkyy+VMTOHmkHLY)5AEEh z#I=;qnzs`t`9V}}^jT({yPs;(gcm}1n&aQsHdHk%x@2Utd}98Nl&b%Dt@JxtN=JW10t8_2R*%v5_}*j&zE#>*u?192)N#G+_FZW5 zcGbQtP=7u^cTc}N>}42jXiR=m^^vD<^~7CU0>b%3#arI1VcRNor5KpkSQ@OZeuZBT zIMs*-fD$&(e^n#ZIWwoqIEF-C=3{?WYUi*n^hA+MTa2c*axoea(A5y)Q2-*t!#;Xq z=&WhPv04Sn-gooSJtEI}@mN|C3yNb|);^}sl?wz6Cgz9zE6A1M@x{x@7`}5F&lyu9 zkfhz@?JfceWHYTw3zb7>#1S8Yw#KBTs?jeDW{T4n**Ps(!`CEb<0eHJo+;`Bt@w^Z zk_xn=4bg#t;2*_bx9SQCWteg0d(2K&DV(q#$xL#c&Mi+-A&`+0Tx~8NKX`luM_mAZ zZM{mm`NnyV{et-wAq*S$D@=+blB{Jn#e=DLuE}5OX49z!O=4tX)cTzt)BE^auiVa$@{> zyU&n^AoV0i>9;_6hLg$Y?dHC}J4!@u=fR|S1`2sahK6I%uB-)j^on_kwRrgUm=QIhX__LcYX*+n<|s5{BykKGq;~|7K9eJc9jP@%pYCb+g?yn z@d{qZbj)_-zSJ`rLJqStL2D;W4=-0Z5hUC1;Km($ni*e@`D8;~t-hhvC`_ zFCUDab3cMtzoT4K%-m8gs0)bAH(C!&PWwgL-ye3w@0(mYJUNlRn4!Ulvj6AO57V?v zwHH+_Z{D_VY)tfzTfD8UkLvk7*2(Lh)R~C!JbI`UFox{C18LnLeR%$Ja~Dxo3^X@h zIbY1QI_PzseM4bEjWj}VbexzxvBG)dJ7}VVrMo=}$ynBvRbX;cqOW>|c{iiwILDM! zQiR02*0prCu_02O=6@z!wTso<-m}|9f6HcKvDejb{5L56_&1&PoQ->;iEvKMhl9uQ zA^1l?ZRgaSPoT%cNl4won_Kcm_Acd9F?`th`g~e>@~byJJtxu19hZ^H`c5yoc3?cp$?mK9gvp1a26Cd1Uq zux!-Y_-50@OL>Mz@Iol^pYadR_oh~fjr9Q90%E`)E0fjC37cMaUp-nh|tqXA;3^ks*_Vm=g5G>%N5MG zsK(q4e*v{PbW?1(Ukub-Zdz*so~i_<(><~r?xZm$%qoSQ_s3M zDsXPC*>7vFt=O`uTD*7#!$lqmcUjmx6!kXX?TJ6)~s3bp?C^vT{}CCZ>2N_HE9397Ulg z*;K7>L;#Dzw+D^}32T~s!qCJi3sM5p%O1toW{LThpsYB;x|ex_t#$8Sx1S$nyx#g0 zXo4&#vA5pxOC?MwkHE^jtz0nt&S;Zu9WrJi@5;h{cif?)CPxbC9Y80;&3{@T z+Hclh&OwF#AzFL%(nKJ>TEFi}pL13F45p0ikUz0q-Sj;vI>%)gZF`st{&UwEbQfVj zy@@>(%RKT0t-X}8mu*w)GffIEdfrnWKWj_a=8Mg_OL#kowq;ed?$l>QX+`-wIkEbt z&hm}FCx&lWLCcwW=0;ZclK+KkEzv!T&>G9^ik8^5 zjzRyHpKn}e`WaI-&Q}{E-LyVJ7>CvE`Clz@&D6JwF-s=IFBD&KY>7syw&h&CV!pU} zwEIUlo|1U06D%IV@V5_7d+47RwXMGCt7>`eRgKe|Ab~OE5oP~tCsD#a1UsR2^;92( zTcTBrLW7>%y|-SzNmYm5uNvZY Date: Thu, 14 Mar 2024 14:07:22 +0700 Subject: [PATCH 02/17] feat: Nitro-Tensorrt-LLM Extension (#2280) * feat: tensorrt-llm-extension * fix: loading * feat: add download tensorrt llm runner Signed-off-by: James * feat: update to rollupjs instead of webpack for monitoring extension Signed-off-by: James * feat: move update nvidia info to monitor extension Signed-off-by: James * allow download tensorrt Signed-off-by: James * update Signed-off-by: James * allow download tensor rt based on gpu setting Signed-off-by: James * update downloaded models Signed-off-by: James * feat: add extension compatibility * dynamic tensor rt engines Signed-off-by: James * update models Signed-off-by: James * chore: remove ts-ignore * feat: getting installation state from extension Signed-off-by: James * chore: adding type for decompress Signed-off-by: James * feat: update according Louis's comment Signed-off-by: James * feat: add progress for installing extension Signed-off-by: James * chore: remove args from extension installation * fix: model download does not work properly * fix: do not allow user to stop tensorrtllm inference * fix: extension installed style * fix: download tensorrt does not update state Signed-off-by: James * chore: replace int4 by fl16 * feat: modal for installing extension Signed-off-by: James * fix: start download immediately after press install Signed-off-by: James * fix: error switching between engines * feat: rename inference provider to ai engine and refactor to core * fix: missing ulid * fix: core bundler * feat: add cancel extension installing Signed-off-by: James * remove mocking for mac Signed-off-by: James * fix: show models only when extension is ready * add tensorrt badge for model Signed-off-by: James * fix: copy * fix: add compatible check (#2342) * fix: add compatible check Signed-off-by: James * fix: copy * fix: font * fix: copy * fix: broken monitoring extension * chore: bump engine * fix: copy * fix: model copy * fix: copy * fix: model json --------- Signed-off-by: James Co-authored-by: James Co-authored-by: Louis * fix: vulkan support * fix: installation button padding * fix: empty script * fix: remove hard code string --------- Signed-off-by: James Co-authored-by: James Co-authored-by: NamH --- .gitignore | 20 +- core/package.json | 12 +- core/rollup.config.ts | 2 +- core/src/api/index.ts | 3 + core/src/core.ts | 36 +- core/src/extension.ts | 44 +++ core/src/extensions/ai-engines/AIEngine.ts | 60 ++++ .../extensions/ai-engines/LocalOAIEngine.ts | 63 ++++ core/src/extensions/ai-engines/OAIEngine.ts | 116 +++++++ core/src/extensions/ai-engines/helpers/sse.ts | 67 ++++ core/src/extensions/ai-engines/index.ts | 3 + core/src/extensions/index.ts | 5 + core/src/extensions/model.ts | 3 +- core/src/extensions/monitoring.ts | 3 +- core/src/node/api/processors/download.ts | 36 +- core/src/node/api/restful/helper/builder.ts | 50 +-- core/src/types/file/index.ts | 35 +- .../miscellaneous/fileDownloadRequest.ts | 8 + core/src/types/miscellaneous/index.ts | 2 + core/src/types/miscellaneous/networkConfig.ts | 4 + .../types/miscellaneous/systemResourceInfo.ts | 28 ++ core/src/types/model/modelEntity.ts | 1 + core/src/types/model/modelInterface.ts | 7 +- core/src/types/monitoring/index.ts | 1 + core/src/types/monitoring/resourceInfo.ts | 6 + core/tsconfig.json | 4 +- extensions/huggingface-extension/src/index.ts | 7 +- .../inference-nitro-extension/download.bat | 2 +- .../inference-nitro-extension/package.json | 2 +- .../inference-nitro-extension/src/index.ts | 3 - .../src/node/accelerator.ts | 237 ------------- .../src/node/execute.ts | 9 +- .../src/node/index.ts | 20 +- extensions/model-extension/src/index.ts | 87 ++++- extensions/monitoring-extension/bin/.gitkeep | 0 extensions/monitoring-extension/download.bat | 2 + extensions/monitoring-extension/package.json | 29 +- .../monitoring-extension/rollup.config.ts | 68 ++++ .../src/@types/global.d.ts | 19 +- extensions/monitoring-extension/src/index.ts | 19 +- extensions/monitoring-extension/src/module.ts | 92 ----- .../monitoring-extension/src/node/index.ts | 317 ++++++++++++++++++ .../monitoring-extension/webpack.config.js | 35 -- extensions/tensorrt-llm-extension/README.md | 79 +++++ extensions/tensorrt-llm-extension/models.json | 49 +++ .../tensorrt-llm-extension/package.json | 75 +++++ .../tensorrt-llm-extension/rollup.config.ts | 73 ++++ .../src/@types/global.d.ts | 10 + .../tensorrt-llm-extension/src/index.ts | 147 ++++++++ .../tensorrt-llm-extension/src/node/index.ts | 191 +++++++++++ .../tensorrt-llm-extension/tsconfig.json | 20 ++ web/containers/DropdownListSidebar/index.tsx | 3 +- .../InstallingExtensionModal.tsx | 87 +++++ .../BottomBar/InstallingExtension/index.tsx | 52 +++ web/containers/Layout/BottomBar/index.tsx | 2 + web/containers/Layout/index.tsx | 3 + web/containers/Providers/EventListener.tsx | 59 +++- web/extension/ExtensionManager.ts | 4 +- web/helpers/atoms/Extension.atom.ts | 40 +++ web/hooks/useActiveModel.ts | 12 +- web/hooks/useDownloadModel.ts | 28 +- web/hooks/useDownloadState.ts | 202 +++++------ web/hooks/useGpuSetting.ts | 21 ++ web/next.config.js | 1 + .../ExploreModels/ExploreModelItem/index.tsx | 13 +- .../ExploreModelItemHeader/index.tsx | 18 + .../CoreExtensions/TensorRtExtensionItem.tsx | 225 +++++++++++++ web/screens/Settings/CoreExtensions/index.tsx | 110 +++--- web/services/appService.ts | 24 ++ web/services/coreService.ts | 7 +- web/types/index.d.ts | 1 + 71 files changed, 2497 insertions(+), 626 deletions(-) create mode 100644 core/src/extensions/ai-engines/AIEngine.ts create mode 100644 core/src/extensions/ai-engines/LocalOAIEngine.ts create mode 100644 core/src/extensions/ai-engines/OAIEngine.ts create mode 100644 core/src/extensions/ai-engines/helpers/sse.ts create mode 100644 core/src/extensions/ai-engines/index.ts create mode 100644 core/src/types/miscellaneous/fileDownloadRequest.ts create mode 100644 core/src/types/miscellaneous/networkConfig.ts create mode 100644 core/src/types/monitoring/resourceInfo.ts delete mode 100644 extensions/inference-nitro-extension/src/node/accelerator.ts create mode 100644 extensions/monitoring-extension/bin/.gitkeep create mode 100644 extensions/monitoring-extension/download.bat create mode 100644 extensions/monitoring-extension/rollup.config.ts delete mode 100644 extensions/monitoring-extension/src/module.ts create mode 100644 extensions/monitoring-extension/src/node/index.ts delete mode 100644 extensions/monitoring-extension/webpack.config.js create mode 100644 extensions/tensorrt-llm-extension/README.md create mode 100644 extensions/tensorrt-llm-extension/models.json create mode 100644 extensions/tensorrt-llm-extension/package.json create mode 100644 extensions/tensorrt-llm-extension/rollup.config.ts create mode 100644 extensions/tensorrt-llm-extension/src/@types/global.d.ts create mode 100644 extensions/tensorrt-llm-extension/src/index.ts create mode 100644 extensions/tensorrt-llm-extension/src/node/index.ts create mode 100644 extensions/tensorrt-llm-extension/tsconfig.json create mode 100644 web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx create mode 100644 web/containers/Layout/BottomBar/InstallingExtension/index.tsx create mode 100644 web/helpers/atoms/Extension.atom.ts create mode 100644 web/hooks/useGpuSetting.ts create mode 100644 web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx create mode 100644 web/services/appService.ts diff --git a/.gitignore b/.gitignore index ae0691605..d9787d87b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,16 +22,16 @@ package-lock.json core/lib/** # Nitro binary files -extensions/inference-nitro-extension/bin/*/nitro -extensions/inference-nitro-extension/bin/*/*.metal -extensions/inference-nitro-extension/bin/*/*.exe -extensions/inference-nitro-extension/bin/*/*.dll -extensions/inference-nitro-extension/bin/*/*.exp -extensions/inference-nitro-extension/bin/*/*.lib -extensions/inference-nitro-extension/bin/saved-* -extensions/inference-nitro-extension/bin/*.tar.gz -extensions/inference-nitro-extension/bin/vulkaninfoSDK.exe -extensions/inference-nitro-extension/bin/vulkaninfo +extensions/*-extension/bin/*/nitro +extensions/*-extension/bin/*/*.metal +extensions/*-extension/bin/*/*.exe +extensions/*-extension/bin/*/*.dll +extensions/*-extension/bin/*/*.exp +extensions/*-extension/bin/*/*.lib +extensions/*-extension/bin/saved-* +extensions/*-extension/bin/*.tar.gz +extensions/*-extension/bin/vulkaninfoSDK.exe +extensions/*-extension/bin/vulkaninfo # Turborepo diff --git a/core/package.json b/core/package.json index 2bf3e1735..2f4f6b576 100644 --- a/core/package.json +++ b/core/package.json @@ -45,11 +45,12 @@ "start": "rollup -c rollup.config.ts -w" }, "devDependencies": { - "jest": "^29.7.0", "@types/jest": "^29.5.12", "@types/node": "^12.0.2", - "eslint-plugin-jest": "^27.9.0", "eslint": "8.57.0", + "eslint-plugin-jest": "^27.9.0", + "jest": "^29.7.0", + "rimraf": "^3.0.2", "rollup": "^2.38.5", "rollup-plugin-commonjs": "^9.1.8", "rollup-plugin-json": "^3.1.0", @@ -58,7 +59,10 @@ "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", "tslib": "^2.6.2", - "typescript": "^5.3.3", - "rimraf": "^3.0.2" + "typescript": "^5.3.3" + }, + "dependencies": { + "rxjs": "^7.8.1", + "ulid": "^2.3.0" } } diff --git a/core/rollup.config.ts b/core/rollup.config.ts index ebea8e237..95305bf25 100644 --- a/core/rollup.config.ts +++ b/core/rollup.config.ts @@ -64,7 +64,7 @@ export default [ // Allow json resolution json(), // Compile TypeScript files - typescript({ useTsconfigDeclarationDir: true }), + typescript({ useTsconfigDeclarationDir: true, exclude: ['src/*.ts', 'src/extensions/**'] }), // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) commonjs(), // Allow node_modules resolution, so you can use 'external' to control diff --git a/core/src/api/index.ts b/core/src/api/index.ts index e62b49087..f97593934 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -33,6 +33,8 @@ export enum AppRoute { stopServer = 'stopServer', log = 'log', logServer = 'logServer', + systemInformations = 'systemInformations', + showToast = 'showToast', } export enum AppEvent { @@ -56,6 +58,7 @@ export enum DownloadEvent { onFileDownloadUpdate = 'onFileDownloadUpdate', onFileDownloadError = 'onFileDownloadError', onFileDownloadSuccess = 'onFileDownloadSuccess', + onFileUnzipSuccess = 'onFileUnzipSuccess', } export enum LocalImportModelEvent { diff --git a/core/src/core.ts b/core/src/core.ts index 6e2442c2b..b8cbd3162 100644 --- a/core/src/core.ts +++ b/core/src/core.ts @@ -1,4 +1,4 @@ -import { FileStat } from './types' +import { DownloadRequest, FileStat, NetworkConfig } from './types' /** * Execute a extension module function in main process @@ -17,18 +17,16 @@ const executeOnMain: (extension: string, method: string, ...args: any[]) => Prom /** * Downloads a file from a URL and saves it to the local file system. - * @param {string} url - The URL of the file to download. - * @param {string} fileName - The name to use for the downloaded file. - * @param {object} network - Optional object to specify proxy/whether to ignore SSL certificates. + * + * @param {DownloadRequest} downloadRequest - The request to download the file. + * @param {NetworkConfig} network - Optional object to specify proxy/whether to ignore SSL certificates. + * * @returns {Promise} A promise that resolves when the file is downloaded. */ -const downloadFile: ( - url: string, - fileName: string, - network?: { proxy?: string; ignoreSSL?: boolean } -) => Promise = (url, fileName, network) => { - return global.core?.api?.downloadFile(url, fileName, network) -} +const downloadFile: (downloadRequest: DownloadRequest, network?: NetworkConfig) => Promise = ( + downloadRequest, + network +) => global.core?.api?.downloadFile(downloadRequest, network) /** * Aborts the download of a specific file. @@ -108,6 +106,20 @@ const log: (message: string, fileName?: string) => void = (message, fileName) => const isSubdirectory: (from: string, to: string) => Promise = (from: string, to: string) => global.core.api?.isSubdirectory(from, to) +/** + * Get system information + * @returns {Promise} - A promise that resolves with the system information. + */ +const systemInformations: () => Promise = () => global.core.api?.systemInformations() + +/** + * Show toast message from browser processes. + * @param title + * @param message + * @returns + */ +const showToast: (title: string, message: string) => void = (title, message) => + global.core.api?.showToast(title, message) /** * Register extension point function type definition */ @@ -134,5 +146,7 @@ export { log, isSubdirectory, getUserHomePath, + systemInformations, + showToast, FileStat, } diff --git a/core/src/extension.ts b/core/src/extension.ts index 3b3edc7b3..22accb4b4 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -10,6 +10,22 @@ export enum ExtensionTypeEnum { export interface ExtensionType { type(): ExtensionTypeEnum | undefined } + +export interface Compatibility { + platform: string[] + version: string +} + +const ALL_INSTALLATION_STATE = [ + 'NotRequired', // not required. + 'Installed', // require and installed. Good to go. + 'NotInstalled', // require to be installed. + 'Corrupted', // require but corrupted. Need to redownload. +] as const + +export type InstallationStateTuple = typeof ALL_INSTALLATION_STATE +export type InstallationState = InstallationStateTuple[number] + /** * Represents a base extension. * This class should be extended by any class that represents an extension. @@ -33,4 +49,32 @@ export abstract class BaseExtension implements ExtensionType { * Any cleanup logic for the extension should be put here. */ abstract onUnload(): void + + /** + * The compatibility of the extension. + * This is used to check if the extension is compatible with the current environment. + * @property {Array} platform + */ + compatibility(): Compatibility | undefined { + return undefined + } + + /** + * Determine if the prerequisites for the extension are installed. + * + * @returns {boolean} true if the prerequisites are installed, false otherwise. + */ + async installationState(): Promise { + return 'NotRequired' + } + + /** + * Install the prerequisites for the extension. + * + * @returns {Promise} + */ + // @ts-ignore + async install(...args): Promise { + return + } } diff --git a/core/src/extensions/ai-engines/AIEngine.ts b/core/src/extensions/ai-engines/AIEngine.ts new file mode 100644 index 000000000..608b5c193 --- /dev/null +++ b/core/src/extensions/ai-engines/AIEngine.ts @@ -0,0 +1,60 @@ +import { getJanDataFolderPath, joinPath } from '../../core' +import { events } from '../../events' +import { BaseExtension } from '../../extension' +import { fs } from '../../fs' +import { Model, ModelEvent } from '../../types' + +/** + * Base AIEngine + * Applicable to all AI Engines + */ +export abstract class AIEngine extends BaseExtension { + // The inference engine + abstract provider: string + // The model folder + modelFolder: string = 'models' + + abstract models(): Promise + + /** + * On extension load, subscribe to events. + */ + onLoad() { + this.prePopulateModels() + } + + /** + * Pre-populate models to App Data Folder + */ + prePopulateModels(): Promise { + return this.models().then((models) => { + const prePoluateOperations = models.map((model) => + getJanDataFolderPath() + .then((janDataFolder) => + // Attempt to create the model folder + joinPath([janDataFolder, this.modelFolder, model.id]).then((path) => + fs + .mkdirSync(path) + .catch() + .then(() => path) + ) + ) + .then((path) => joinPath([path, 'model.json'])) + .then((path) => { + // Do not overwite existing model.json + return fs.existsSync(path).then((exist: any) => { + if (!exist) return fs.writeFileSync(path, JSON.stringify(model, null, 2)) + }) + }) + .catch((e: Error) => { + console.error('Error', e) + }) + ) + Promise.all(prePoluateOperations).then(() => + // Emit event to update models + // So the UI can update the models list + events.emit(ModelEvent.OnModelsUpdate, {}) + ) + }) + } +} diff --git a/core/src/extensions/ai-engines/LocalOAIEngine.ts b/core/src/extensions/ai-engines/LocalOAIEngine.ts new file mode 100644 index 000000000..79dbcbf5e --- /dev/null +++ b/core/src/extensions/ai-engines/LocalOAIEngine.ts @@ -0,0 +1,63 @@ +import { executeOnMain, getJanDataFolderPath, joinPath } from '../../core' +import { events } from '../../events' +import { Model, ModelEvent } from '../../types' +import { OAIEngine } from './OAIEngine' + +/** + * Base OAI Local Inference Provider + * Added the implementation of loading and unloading model (applicable to local inference providers) + */ +export abstract class LocalOAIEngine extends OAIEngine { + // The inference engine + loadModelFunctionName: string = 'loadModel' + unloadModelFunctionName: string = 'unloadModel' + isRunning: boolean = false + + /** + * On extension load, subscribe to events. + */ + onLoad() { + super.onLoad() + // These events are applicable to local inference providers + events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)) + events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)) + } + + /** + * Load the model. + */ + async onModelInit(model: Model) { + if (model.engine.toString() !== this.provider) return + + const modelFolder = await joinPath([await getJanDataFolderPath(), this.modelFolder, model.id]) + + const res = await executeOnMain(this.nodeModule, this.loadModelFunctionName, { + modelFolder, + model, + }) + + if (res?.error) { + events.emit(ModelEvent.OnModelFail, { + ...model, + error: res.error, + }) + return + } else { + this.loadedModel = model + events.emit(ModelEvent.OnModelReady, model) + this.isRunning = true + } + } + /** + * Stops the model. + */ + onModelStop(model: Model) { + if (model.engine?.toString() !== this.provider) return + + this.isRunning = false + + executeOnMain(this.nodeModule, this.unloadModelFunctionName).then(() => { + events.emit(ModelEvent.OnModelStopped, {}) + }) + } +} diff --git a/core/src/extensions/ai-engines/OAIEngine.ts b/core/src/extensions/ai-engines/OAIEngine.ts new file mode 100644 index 000000000..3e583c9b9 --- /dev/null +++ b/core/src/extensions/ai-engines/OAIEngine.ts @@ -0,0 +1,116 @@ +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { AIEngine } from './AIEngine' +import { + ChatCompletionRole, + ContentType, + InferenceEvent, + MessageEvent, + MessageRequest, + MessageRequestType, + MessageStatus, + Model, + ModelInfo, + ThreadContent, + ThreadMessage, +} from '../../types' +import { events } from '../../events' + +/** + * Base OAI Inference Provider + * Applicable to all OAI compatible inference providers + */ +export abstract class OAIEngine extends AIEngine { + // The inference engine + abstract inferenceUrl: string + abstract nodeModule: string + + // Controller to handle stop requests + controller = new AbortController() + isCancelled = false + + // The loaded model instance + loadedModel: Model | undefined + + /** + * On extension load, subscribe to events. + */ + onLoad() { + super.onLoad() + events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => this.inference(data)) + events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped()) + } + + /** + * On extension unload + */ + onUnload(): void {} + + /* + * Inference request + */ + inference(data: MessageRequest) { + if (data.model?.engine?.toString() !== this.provider) return + + const timestamp = Date.now() + const message: ThreadMessage = { + id: ulid(), + thread_id: data.threadId, + type: data.type, + assistant_id: data.assistantId, + role: ChatCompletionRole.Assistant, + content: [], + status: MessageStatus.Pending, + created: timestamp, + updated: timestamp, + object: 'thread.message', + } + + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } + + this.isCancelled = false + this.controller = new AbortController() + + const model: ModelInfo = { + ...(this.loadedModel ? this.loadedModel : {}), + ...data.model, + } + + requestInference(this.inferenceUrl, data.messages ?? [], model, this.controller).subscribe({ + next: (content: any) => { + const messageContent: ThreadContent = { + type: ContentType.Text, + text: { + value: content.trim(), + annotations: [], + }, + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) + }, + complete: async () => { + message.status = message.content.length ? MessageStatus.Ready : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + }, + error: async (err: any) => { + if (this.isCancelled || message.content.length) { + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return + } + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + }, + }) + } + + /** + * Stops the inference. + */ + onInferenceStopped() { + this.isCancelled = true + this.controller?.abort() + } +} diff --git a/core/src/extensions/ai-engines/helpers/sse.ts b/core/src/extensions/ai-engines/helpers/sse.ts new file mode 100644 index 000000000..3d810d934 --- /dev/null +++ b/core/src/extensions/ai-engines/helpers/sse.ts @@ -0,0 +1,67 @@ +import { Observable } from 'rxjs' +import { ModelRuntimeParams } from '../../../types' +/** + * Sends a request to the inference server to generate a response based on the recent messages. + * @param recentMessages - An array of recent messages to use as context for the inference. + * @returns An Observable that emits the generated response as a string. + */ +export function requestInference( + inferenceUrl: string, + recentMessages: any[], + model: { + id: string + parameters: ModelRuntimeParams + }, + controller?: AbortController +): Observable { + return new Observable((subscriber) => { + const requestBody = JSON.stringify({ + messages: recentMessages, + model: model.id, + stream: true, + ...model.parameters, + }) + fetch(inferenceUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Accept': model.parameters.stream ? 'text/event-stream' : 'application/json', + }, + body: requestBody, + signal: controller?.signal, + }) + .then(async (response) => { + if (model.parameters.stream === false) { + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') + } else { + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' + + while (true && reader) { + const { done, value } = await reader.read() + if (done) { + break + } + const text = decoder.decode(value) + const lines = text.trim().split('\n') + for (const line of lines) { + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') + } + subscriber.next(content) + } + } + } + } + subscriber.complete() + }) + .catch((err) => subscriber.error(err)) + }) +} diff --git a/core/src/extensions/ai-engines/index.ts b/core/src/extensions/ai-engines/index.ts new file mode 100644 index 000000000..f4da62a7c --- /dev/null +++ b/core/src/extensions/ai-engines/index.ts @@ -0,0 +1,3 @@ +export * from './AIEngine' +export * from './OAIEngine' +export * from './LocalOAIEngine' diff --git a/core/src/extensions/index.ts b/core/src/extensions/index.ts index c6834482c..c049f3b3a 100644 --- a/core/src/extensions/index.ts +++ b/core/src/extensions/index.ts @@ -28,3 +28,8 @@ export { ModelExtension } from './model' * Hugging Face extension for converting HF models to GGUF. */ export { HuggingFaceExtension } from './huggingface' + +/** + * Base AI Engines. + */ +export * from './ai-engines' diff --git a/core/src/extensions/model.ts b/core/src/extensions/model.ts index 79202398b..33eec0afc 100644 --- a/core/src/extensions/model.ts +++ b/core/src/extensions/model.ts @@ -1,5 +1,5 @@ import { BaseExtension, ExtensionTypeEnum } from '../extension' -import { ImportingModel, Model, ModelInterface, OptionType } from '../index' +import { GpuSetting, ImportingModel, Model, ModelInterface, OptionType } from '../index' /** * Model extension for managing models. @@ -14,6 +14,7 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter abstract downloadModel( model: Model, + gpuSettings?: GpuSetting, network?: { proxy: string; ignoreSSL?: boolean } ): Promise abstract cancelModelDownload(modelId: string): Promise diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts index ba193f0f4..8d61580fc 100644 --- a/core/src/extensions/monitoring.ts +++ b/core/src/extensions/monitoring.ts @@ -1,5 +1,5 @@ import { BaseExtension, ExtensionTypeEnum } from '../extension' -import { MonitoringInterface } from '../index' +import { GpuSetting, MonitoringInterface } from '../index' /** * Monitoring extension for system monitoring. @@ -13,6 +13,7 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit return ExtensionTypeEnum.SystemMonitoring } + abstract getGpuSetting(): Promise abstract getResourcesInfo(): Promise abstract getCurrentLoad(): Promise } diff --git a/core/src/node/api/processors/download.ts b/core/src/node/api/processors/download.ts index 4ddeff160..8e8e08f2f 100644 --- a/core/src/node/api/processors/download.ts +++ b/core/src/node/api/processors/download.ts @@ -5,7 +5,7 @@ import { getJanDataFolderPath } from '../../helper' import { DownloadManager } from '../../helper/download' import { createWriteStream, renameSync } from 'fs' import { Processor } from './Processor' -import { DownloadState } from '../../../types' +import { DownloadRequest, DownloadState, NetworkConfig } from '../../../types' export class Downloader implements Processor { observer?: Function @@ -20,24 +20,27 @@ export class Downloader implements Processor { return func(this.observer, ...args) } - downloadFile(observer: any, url: string, localPath: string, network: any) { + downloadFile(observer: any, downloadRequest: DownloadRequest, network?: NetworkConfig) { const request = require('request') const progress = require('request-progress') const strictSSL = !network?.ignoreSSL const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined + + const { localPath, url } = downloadRequest + let normalizedPath = localPath if (typeof localPath === 'string') { - localPath = normalizeFilePath(localPath) + normalizedPath = normalizeFilePath(localPath) } - const array = localPath.split(sep) + const array = normalizedPath.split(sep) const fileName = array.pop() ?? '' const modelId = array.pop() ?? '' - const destination = resolve(getJanDataFolderPath(), localPath) + const destination = resolve(getJanDataFolderPath(), normalizedPath) const rq = request({ url, strictSSL, proxy }) // Put request to download manager instance - DownloadManager.instance.setRequest(localPath, rq) + DownloadManager.instance.setRequest(normalizedPath, rq) // Downloading file to a temp file first const downloadingTempFile = `${destination}.download` @@ -56,16 +59,25 @@ export class Downloader implements Processor { total: 0, transferred: 0, }, + children: [], downloadState: 'downloading', + extensionId: downloadRequest.extensionId, + downloadType: downloadRequest.downloadType, + localPath: normalizedPath, } DownloadManager.instance.downloadProgressMap[modelId] = initialDownloadState + if (downloadRequest.downloadType === 'extension') { + observer?.(DownloadEvent.onFileDownloadUpdate, initialDownloadState) + } + progress(rq, {}) .on('progress', (state: any) => { + const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] const downloadState: DownloadState = { + ...currentDownloadState, ...state, - modelId, - fileName, + fileName: fileName, downloadState: 'downloading', } console.debug('progress: ', downloadState) @@ -76,22 +88,22 @@ export class Downloader implements Processor { const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] const downloadState: DownloadState = { ...currentDownloadState, + fileName: fileName, error: error.message, downloadState: 'error', } - if (currentDownloadState) { - DownloadManager.instance.downloadProgressMap[modelId] = downloadState - } observer?.(DownloadEvent.onFileDownloadError, downloadState) + DownloadManager.instance.downloadProgressMap[modelId] = downloadState }) .on('end', () => { const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] - if (currentDownloadState && DownloadManager.instance.networkRequests[localPath]) { + if (currentDownloadState && DownloadManager.instance.networkRequests[normalizedPath]) { // Finished downloading, rename temp file to actual file renameSync(downloadingTempFile, destination) const downloadState: DownloadState = { ...currentDownloadState, + fileName: fileName, downloadState: 'end', } observer?.(DownloadEvent.onFileDownloadSuccess, downloadState) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index 7001c0c76..6b9bbb3a8 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -1,7 +1,16 @@ -import fs from 'fs' +import { + existsSync, + readdirSync, + readFileSync, + writeFileSync, + mkdirSync, + appendFileSync, + createWriteStream, + rmdirSync, +} from 'fs' import { JanApiRouteConfiguration, RouteConfiguration } from './configuration' import { join } from 'path' -import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../index' +import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../types' import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper' import { DEFAULT_CHAT_COMPLETION_URL } from './consts' @@ -9,12 +18,12 @@ import { DEFAULT_CHAT_COMPLETION_URL } from './consts' export const getBuilder = async (configuration: RouteConfiguration) => { const directoryPath = join(getJanDataFolderPath(), configuration.dirName) try { - if (!fs.existsSync(directoryPath)) { + if (!existsSync(directoryPath)) { console.debug('model folder not found') return [] } - const files: string[] = fs.readdirSync(directoryPath) + const files: string[] = readdirSync(directoryPath) const allDirectories: string[] = [] for (const file of files) { @@ -46,8 +55,8 @@ export const getBuilder = async (configuration: RouteConfiguration) => { } const readModelMetadata = (path: string): string | undefined => { - if (fs.existsSync(path)) { - return fs.readFileSync(path, 'utf-8') + if (existsSync(path)) { + return readFileSync(path, 'utf-8') } else { return undefined } @@ -81,7 +90,7 @@ export const deleteBuilder = async (configuration: RouteConfiguration, id: strin } const objectPath = join(directoryPath, id) - fs.rmdirSync(objectPath, { recursive: true }) + rmdirSync(objectPath, { recursive: true }) return { id: id, object: configuration.delete.object, @@ -96,20 +105,19 @@ export const getMessages = async (threadId: string): Promise => const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId) const messageFile = 'messages.jsonl' try { - const files: string[] = fs.readdirSync(threadDirPath) + const files: string[] = readdirSync(threadDirPath) if (!files.includes(messageFile)) { console.error(`${threadDirPath} not contains message file`) return [] } const messageFilePath = join(threadDirPath, messageFile) - if (!fs.existsSync(messageFilePath)) { + if (!existsSync(messageFilePath)) { console.debug('message file not found') return [] } - const lines = fs - .readFileSync(messageFilePath, 'utf-8') + const lines = readFileSync(messageFilePath, 'utf-8') .toString() .split('\n') .filter((line: any) => line !== '') @@ -157,11 +165,11 @@ export const createThread = async (thread: any) => { const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id) const threadJsonPath = join(threadDirPath, threadMetadataFileName) - if (!fs.existsSync(threadDirPath)) { - fs.mkdirSync(threadDirPath) + if (!existsSync(threadDirPath)) { + mkdirSync(threadDirPath) } - await fs.writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) + await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) return updatedThread } catch (err) { return { @@ -191,7 +199,7 @@ export const updateThread = async (threadId: string, thread: any) => { const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id) const threadJsonPath = join(threadDirPath, threadMetadataFileName) - await fs.writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) + await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) return updatedThread } catch (err) { return { @@ -233,10 +241,10 @@ export const createMessage = async (threadId: string, message: any) => { const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId) const threadMessagePath = join(threadDirPath, threadMessagesFileName) - if (!fs.existsSync(threadDirPath)) { - fs.mkdirSync(threadDirPath) + if (!existsSync(threadDirPath)) { + mkdirSync(threadDirPath) } - fs.appendFileSync(threadMessagePath, JSON.stringify(threadMessage) + '\n') + appendFileSync(threadMessagePath, JSON.stringify(threadMessage) + '\n') return threadMessage } catch (err) { return { @@ -259,8 +267,8 @@ export const downloadModel = async ( } const directoryPath = join(getJanDataFolderPath(), 'models', modelId) - if (!fs.existsSync(directoryPath)) { - fs.mkdirSync(directoryPath) + if (!existsSync(directoryPath)) { + mkdirSync(directoryPath) } // path to model binary @@ -281,7 +289,7 @@ export const downloadModel = async ( .on('end', function () { console.debug('end') }) - .pipe(fs.createWriteStream(modelBinaryPath)) + .pipe(createWriteStream(modelBinaryPath)) } return { diff --git a/core/src/types/file/index.ts b/core/src/types/file/index.ts index cc7274a28..d941987ef 100644 --- a/core/src/types/file/index.ts +++ b/core/src/types/file/index.ts @@ -4,16 +4,43 @@ export type FileStat = { } export type DownloadState = { - modelId: string + modelId: string // TODO: change to download id fileName: string time: DownloadTime speed: number - percent: number + percent: number size: DownloadSize - children?: DownloadState[] - error?: string downloadState: 'downloading' | 'error' | 'end' + children?: DownloadState[] + + error?: string + extensionId?: string + downloadType?: DownloadType + localPath?: string +} + +export type DownloadType = 'model' | 'extension' + +export type DownloadRequest = { + /** + * The URL to download the file from. + */ + url: string + + /** + * The local path to save the file to. + */ + localPath: string + + /** + * The extension ID of the extension that initiated the download. + * + * Can be extension name. + */ + extensionId?: string + + downloadType?: DownloadType } type DownloadTime = { diff --git a/core/src/types/miscellaneous/fileDownloadRequest.ts b/core/src/types/miscellaneous/fileDownloadRequest.ts new file mode 100644 index 000000000..83131aa71 --- /dev/null +++ b/core/src/types/miscellaneous/fileDownloadRequest.ts @@ -0,0 +1,8 @@ +export type FileDownloadRequest = { + downloadId: string + url: string + localPath: string + fileName: string + displayName: string + metadata: Record +} diff --git a/core/src/types/miscellaneous/index.ts b/core/src/types/miscellaneous/index.ts index e9c205a73..b4ef68ab6 100644 --- a/core/src/types/miscellaneous/index.ts +++ b/core/src/types/miscellaneous/index.ts @@ -1,3 +1,5 @@ export * from './systemResourceInfo' export * from './promptTemplate' export * from './appUpdate' +export * from './fileDownloadRequest' +export * from './networkConfig' \ No newline at end of file diff --git a/core/src/types/miscellaneous/networkConfig.ts b/core/src/types/miscellaneous/networkConfig.ts new file mode 100644 index 000000000..2d27f4223 --- /dev/null +++ b/core/src/types/miscellaneous/networkConfig.ts @@ -0,0 +1,4 @@ +export type NetworkConfig = { + proxy?: string + ignoreSSL?: boolean +} diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts index 1472cda47..f7dd4a82b 100644 --- a/core/src/types/miscellaneous/systemResourceInfo.ts +++ b/core/src/types/miscellaneous/systemResourceInfo.ts @@ -2,3 +2,31 @@ export type SystemResourceInfo = { numCpuPhysicalCore: number memAvailable: number } + +export type RunMode = 'cpu' | 'gpu' + +export type GpuSetting = { + notify: boolean + run_mode: RunMode + nvidia_driver: { + exist: boolean + version: string + } + cuda: { + exist: boolean + version: string + } + gpus: GpuSettingInfo[] + gpu_highest_vram: string + gpus_in_use: string[] + is_initial: boolean + // TODO: This needs to be set based on user toggle in settings + vulkan: boolean +} + +export type GpuSettingInfo = { + id: string + vram: string + name: string + arch?: string +} diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts index 11d3e0526..74568686b 100644 --- a/core/src/types/model/modelEntity.ts +++ b/core/src/types/model/modelEntity.ts @@ -19,6 +19,7 @@ export enum InferenceEngine { nitro = 'nitro', openai = 'openai', triton_trtllm = 'triton_trtllm', + nitro_tensorrt_llm = 'nitro-tensorrt-llm', tool_retrieval_enabled = 'tool_retrieval_enabled', } diff --git a/core/src/types/model/modelInterface.ts b/core/src/types/model/modelInterface.ts index 93d5867ee..639c7c8d3 100644 --- a/core/src/types/model/modelInterface.ts +++ b/core/src/types/model/modelInterface.ts @@ -1,3 +1,4 @@ +import { GpuSetting } from '../miscellaneous' import { Model } from './modelEntity' /** @@ -10,7 +11,11 @@ export interface ModelInterface { * @param network - Optional object to specify proxy/whether to ignore SSL certificates. * @returns A Promise that resolves when the model has been downloaded. */ - downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise + downloadModel( + model: Model, + gpuSettings?: GpuSetting, + network?: { ignoreSSL?: boolean; proxy?: string } + ): Promise /** * Cancels the download of a specific model. diff --git a/core/src/types/monitoring/index.ts b/core/src/types/monitoring/index.ts index 5828dae8b..b96c518fd 100644 --- a/core/src/types/monitoring/index.ts +++ b/core/src/types/monitoring/index.ts @@ -1 +1,2 @@ export * from './monitoringInterface' +export * from './resourceInfo' diff --git a/core/src/types/monitoring/resourceInfo.ts b/core/src/types/monitoring/resourceInfo.ts new file mode 100644 index 000000000..b19da5462 --- /dev/null +++ b/core/src/types/monitoring/resourceInfo.ts @@ -0,0 +1,6 @@ +export type ResourceInfo = { + mem: { + totalMemory: number + usedMemory: number + } +} diff --git a/core/tsconfig.json b/core/tsconfig.json index b112079d2..daeb7eeff 100644 --- a/core/tsconfig.json +++ b/core/tsconfig.json @@ -13,7 +13,7 @@ "declarationDir": "dist/types", "outDir": "dist/lib", "importHelpers": true, - "types": ["@types/jest"] + "types": ["@types/jest"], }, - "include": ["src"] + "include": ["src"], } diff --git a/extensions/huggingface-extension/src/index.ts b/extensions/huggingface-extension/src/index.ts index d8f755080..88292ce58 100644 --- a/extensions/huggingface-extension/src/index.ts +++ b/extensions/huggingface-extension/src/index.ts @@ -13,6 +13,7 @@ import { events, DownloadEvent, log, + DownloadRequest, } from '@janhq/core' import { ggufMetadata } from 'hyllama' @@ -148,7 +149,11 @@ export default class JanHuggingFaceExtension extends HuggingFaceExtension { if (this.interrupted) return if (!(await fs.existsSync(localPath))) { - downloadFile(url, localPath, network) + const downloadRequest: DownloadRequest = { + url, + localPath, + } + downloadFile(downloadRequest, network) filePaths.push(filePath) } } diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat index 2ef3165c1..bb8c4ffdc 100644 --- a/extensions/inference-nitro-extension/download.bat +++ b/extensions/inference-nitro-extension/download.bat @@ -1,3 +1,3 @@ @echo off set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json index e6365ad92..dd5798764 100644 --- a/extensions/inference-nitro-extension/package.json +++ b/extensions/inference-nitro-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro && download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro", "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", "downloadnitro:win32": "download.bat", "downloadnitro": "run-script-os", diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 979b4cfac..70244a5d9 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -108,9 +108,6 @@ export default class JanInferenceNitroExtension extends InferenceExtension { events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped() ) - - // Attempt to fetch nvidia info - await executeOnMain(NODE, 'updateNvidiaInfo', {}) } /** diff --git a/extensions/inference-nitro-extension/src/node/accelerator.ts b/extensions/inference-nitro-extension/src/node/accelerator.ts deleted file mode 100644 index 1ffdbc5bd..000000000 --- a/extensions/inference-nitro-extension/src/node/accelerator.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { writeFileSync, existsSync, readFileSync } from 'fs' -import { exec, spawn } from 'child_process' -import path from 'path' -import { getJanDataFolderPath, log } from '@janhq/core/node' - -/** - * Default GPU settings - * TODO: This needs to be refactored to support multiple accelerators - **/ -const DEFALT_SETTINGS = { - notify: true, - run_mode: 'cpu', - nvidia_driver: { - exist: false, - version: '', - }, - cuda: { - exist: false, - version: '', - }, - gpus: [], - gpu_highest_vram: '', - gpus_in_use: [], - is_initial: true, - // TODO: This needs to be set based on user toggle in settings - vulkan: false -} - -/** - * Path to the settings file - **/ -export const GPU_INFO_FILE = path.join( - getJanDataFolderPath(), - 'settings', - 'settings.json' -) - -/** - * Current nitro process - */ -let nitroProcessInfo: NitroProcessInfo | undefined = undefined - -/** - * Nitro process info - */ -export interface NitroProcessInfo { - isRunning: boolean -} - -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export async function updateNvidiaInfo() { - if (process.platform !== 'darwin') { - let data - try { - data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - } catch (error) { - data = DEFALT_SETTINGS - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - } - updateNvidiaDriverInfo() - updateGpuInfo() - } -} - -/** - * Retrieve current nitro process - */ -export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { - nitroProcessInfo = { - isRunning: subprocess != null, - } - return nitroProcessInfo -} - -/** - * Validate nvidia and cuda for linux and windows - */ -export async function updateNvidiaDriverInfo(): Promise { - exec( - 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', - (error, stdout) => { - let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - - if (!error) { - const firstLine = stdout.split('\n')[0].trim() - data['nvidia_driver'].exist = true - data['nvidia_driver'].version = firstLine - } else { - data['nvidia_driver'].exist = false - } - - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - Promise.resolve() - } - ) -} - -/** - * Check if file exists in paths - */ -export function checkFileExistenceInPaths( - file: string, - paths: string[] -): boolean { - return paths.some((p) => existsSync(path.join(p, file))) -} - -/** - * Validate cuda for linux and windows - */ -export function updateCudaExistence( - data: Record = DEFALT_SETTINGS -): Record { - let filesCuda12: string[] - let filesCuda11: string[] - let paths: string[] - let cudaVersion: string = '' - - if (process.platform === 'win32') { - filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] - filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] - } else { - filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] - filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : [] - paths.push('/usr/lib/x86_64-linux-gnu/') - } - - let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ) - - if (!cudaExists) { - cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ) - if (cudaExists) { - cudaVersion = '11' - } - } else { - cudaVersion = '12' - } - - data['cuda'].exist = cudaExists - data['cuda'].version = cudaVersion - console.debug(data['is_initial'], data['gpus_in_use']) - if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) { - data.run_mode = 'gpu' - } - data.is_initial = false - return data -} - -/** - * Get GPU information - */ -export async function updateGpuInfo(): Promise { - let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - - // Cuda - if (data['vulkan'] === true) { - // Vulkan - exec( - process.platform === 'win32' - ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` - : `${__dirname}/../bin/vulkaninfo --summary`, - (error, stdout) => { - if (!error) { - const output = stdout.toString() - log(output) - const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g - - let gpus = [] - let match - while ((match = gpuRegex.exec(output)) !== null) { - const id = match[1] - const name = match[2] - gpus.push({ id, vram: 0, name }) - } - data.gpus = gpus - - if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { - data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] - } - - data = updateCudaExistence(data) - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - } - Promise.resolve() - } - ) - } else { - exec( - 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', - (error, stdout) => { - if (!error) { - log(stdout) - // Get GPU info and gpu has higher memory first - let highestVram = 0 - let highestVramId = '0' - let gpus = stdout - .trim() - .split('\n') - .map((line) => { - let [id, vram, name] = line.split(', ') - vram = vram.replace(/\r/g, '') - if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram) - highestVramId = id - } - return { id, vram, name } - }) - - data.gpus = gpus - data.gpu_highest_vram = highestVramId - } else { - data.gpus = [] - data.gpu_highest_vram = '' - } - - if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { - data.gpus_in_use = [data['gpu_highest_vram']] - } - - data = updateCudaExistence(data) - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - Promise.resolve() - } - ) - } -} diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index f9a668507..8bcc75ae4 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -1,12 +1,19 @@ +import { getJanDataFolderPath } from '@janhq/core/node' import { readFileSync } from 'fs' import * as path from 'path' -import { GPU_INFO_FILE } from './accelerator' export interface NitroExecutableOptions { executablePath: string cudaVisibleDevices: string vkVisibleDevices: string } + +export const GPU_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) + /** * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts index 9b2684a6c..c57eb262d 100644 --- a/extensions/inference-nitro-extension/src/node/index.ts +++ b/extensions/inference-nitro-extension/src/node/index.ts @@ -4,7 +4,6 @@ import { ChildProcessWithoutNullStreams, spawn } from 'child_process' import tcpPortUsed from 'tcp-port-used' import fetchRT from 'fetch-retry' import { log, getSystemResourceInfo } from '@janhq/core/node' -import { getNitroProcessInfo, updateNvidiaInfo } from './accelerator' import { Model, InferenceEngine, @@ -385,11 +384,26 @@ function dispose() { killSubprocess() } +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean +} + +/** + * Retrieve current nitro process + */ +const getCurrentNitroProcessInfo = (): NitroProcessInfo => { + return { + isRunning: subprocess != null, + } +} + export default { runModel, stopModel, killSubprocess, dispose, - updateNvidiaInfo, - getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), + getCurrentNitroProcessInfo, } diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index fb1f26885..d05e7d07f 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -17,6 +17,8 @@ import { ImportingModel, LocalImportModelEvent, baseName, + GpuSetting, + DownloadRequest, } from '@janhq/core' import { extractFileName } from './helpers/path' @@ -29,10 +31,14 @@ export default class JanModelExtension extends ModelExtension { private static readonly _modelMetadataFileName = 'model.json' private static readonly _supportedModelFormat = '.gguf' private static readonly _incompletedModelFileName = '.download' - private static readonly _offlineInferenceEngine = InferenceEngine.nitro - + private static readonly _offlineInferenceEngine = [ + InferenceEngine.nitro, + InferenceEngine.nitro_tensorrt_llm, + ] + private static readonly _tensorRtEngineFormat = '.engine' private static readonly _configDirName = 'config' private static readonly _defaultModelFileName = 'default-model.json' + private static readonly _supportedGpuArch = ['turing', 'ampere', 'ada'] /** * Called when the extension is loaded. @@ -89,12 +95,52 @@ export default class JanModelExtension extends ModelExtension { */ async downloadModel( model: Model, + gpuSettings?: GpuSetting, network?: { ignoreSSL?: boolean; proxy?: string } ): Promise { // create corresponding directory const modelDirPath = await joinPath([JanModelExtension._homeDir, model.id]) if (!(await fs.existsSync(modelDirPath))) await fs.mkdirSync(modelDirPath) + if (model.engine === InferenceEngine.nitro_tensorrt_llm) { + if (!gpuSettings || gpuSettings.gpus.length === 0) { + console.error('No GPU found. Please check your GPU setting.') + return + } + const firstGpu = gpuSettings.gpus[0] + if (!firstGpu.name.toLowerCase().includes('nvidia')) { + console.error('No Nvidia GPU found. Please check your GPU setting.') + return + } + const gpuArch = firstGpu.arch + if (gpuArch === undefined) { + console.error( + 'No GPU architecture found. Please check your GPU setting.' + ) + return + } + + if (!JanModelExtension._supportedGpuArch.includes(gpuArch)) { + console.error( + `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` + ) + return + } + + const os = 'windows' // TODO: remove this hard coded value + + const newSources = model.sources.map((source) => { + const newSource = { ...source } + newSource.url = newSource.url + .replace(//g, os) + .replace(//g, gpuArch) + return newSource + }) + model.sources = newSources + } + + console.debug(`Download sources: ${JSON.stringify(model.sources)}`) + if (model.sources.length > 1) { // path to model binaries for (const source of model.sources) { @@ -105,8 +151,11 @@ export default class JanModelExtension extends ModelExtension { if (source.filename) { path = await joinPath([modelDirPath, source.filename]) } - - downloadFile(source.url, path, network) + const downloadRequest: DownloadRequest = { + url: source.url, + localPath: path, + } + downloadFile(downloadRequest, network) } // TODO: handle multiple binaries for web later } else { @@ -115,7 +164,11 @@ export default class JanModelExtension extends ModelExtension { JanModelExtension._supportedModelFormat ) const path = await joinPath([modelDirPath, fileName]) - downloadFile(model.sources[0]?.url, path, network) + const downloadRequest: DownloadRequest = { + url: model.sources[0]?.url, + localPath: path, + } + downloadFile(downloadRequest, network) if (window && window.core?.api && window.core.api.baseApiUrl) { this.startPollingDownloadProgress(model.id) @@ -238,7 +291,7 @@ export default class JanModelExtension extends ModelExtension { async getDownloadedModels(): Promise { return await this.getModelsMetadata( async (modelDir: string, model: Model) => { - if (model.engine !== JanModelExtension._offlineInferenceEngine) + if (!JanModelExtension._offlineInferenceEngine.includes(model.engine)) return true // model binaries (sources) are absolute path & exist @@ -247,22 +300,32 @@ export default class JanModelExtension extends ModelExtension { ) if (existFiles.every((exist) => exist)) return true - return await fs + const result = await fs .readdirSync(await joinPath([JanModelExtension._homeDir, modelDir])) .then((files: string[]) => { // Model binary exists in the directory // Model binary name can match model ID or be a .gguf file and not be an incompleted model file return ( files.includes(modelDir) || - files.filter( - (file) => + files.filter((file) => { + if ( + file.endsWith(JanModelExtension._incompletedModelFileName) + ) { + return false + } + return ( file .toLowerCase() - .includes(JanModelExtension._supportedModelFormat) && - !file.endsWith(JanModelExtension._incompletedModelFileName) - )?.length >= model.sources.length + .includes(JanModelExtension._supportedModelFormat) || + file + .toLowerCase() + .includes(JanModelExtension._tensorRtEngineFormat) + ) + })?.length > 0 // TODO: NamH find better way (can use basename to check the file name with source url) ) }) + + return result } ) } diff --git a/extensions/monitoring-extension/bin/.gitkeep b/extensions/monitoring-extension/bin/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/extensions/monitoring-extension/download.bat b/extensions/monitoring-extension/download.bat new file mode 100644 index 000000000..f1cf8b7ea --- /dev/null +++ b/extensions/monitoring-extension/download.bat @@ -0,0 +1,2 @@ +@echo off +.\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin \ No newline at end of file diff --git a/extensions/monitoring-extension/package.json b/extensions/monitoring-extension/package.json index 582f7cd7b..73d28ab37 100644 --- a/extensions/monitoring-extension/package.json +++ b/extensions/monitoring-extension/package.json @@ -3,21 +3,40 @@ "version": "1.0.10", "description": "This extension provides system health and OS level data", "main": "dist/index.js", - "module": "dist/module.js", + "node": "dist/node/index.cjs.js", "author": "Jan ", "license": "AGPL-3.0", "scripts": { - "build": "tsc -b . && webpack --config webpack.config.js", + "build": "tsc --module commonjs && rollup -c rollup.config.ts && npm run download-artifacts", + "download-artifacts": "run-script-os && cpx \"bin/**\" \"dist/bin\"", + "download-artifacts:darwin": "echo 'No artifacts to download for darwin'", + "download-artifacts:win32": "download.bat", + "download-artifacts:linux": "download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/node/index.cjs.js" + }, "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^15.2.3", + "@types/node": "^20.11.4", + "@types/node-os-utils": "^1.3.4", + "run-script-os": "^1.1.6", + "cpx": "^1.5.0", "rimraf": "^3.0.2", - "webpack": "^5.88.2", - "webpack-cli": "^5.1.4", - "ts-loader": "^9.5.0" + "rollup": "^2.38.5", + "rollup-plugin-define": "^1.0.1", + "rollup-plugin-sourcemaps": "^0.6.3", + "rollup-plugin-typescript2": "^0.36.0", + "typescript": "^5.3.3", + "download-cli": "^1.1.1" }, "dependencies": { "@janhq/core": "file:../../core", + "@rollup/plugin-replace": "^5.0.5", "node-os-utils": "^1.3.7" }, "files": [ diff --git a/extensions/monitoring-extension/rollup.config.ts b/extensions/monitoring-extension/rollup.config.ts new file mode 100644 index 000000000..1b7a40bad --- /dev/null +++ b/extensions/monitoring-extension/rollup.config.ts @@ -0,0 +1,68 @@ +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') + +export default [ + { + input: `src/index.ts`, + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [], + watch: { + include: 'src/**', + }, + plugins: [ + replace({ + NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + }), + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Compile TypeScript files + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: ['.js', '.ts', '.svelte'], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, + { + input: `src/node/index.ts`, + output: [ + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, + ], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: ['@janhq/core/node'], + watch: { + include: 'src/node/**', + }, + plugins: [ + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: ['.ts', '.js', '.json'], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, +] diff --git a/extensions/monitoring-extension/src/@types/global.d.ts b/extensions/monitoring-extension/src/@types/global.d.ts index 8106353cf..dfa96a0b1 100644 --- a/extensions/monitoring-extension/src/@types/global.d.ts +++ b/extensions/monitoring-extension/src/@types/global.d.ts @@ -1 +1,18 @@ -declare const MODULE: string +declare const NODE: string + +type CpuGpuInfo = { + cpu: { + usage: number + } + gpu: GpuInfo[] +} + +type GpuInfo = { + id: string + name: string + temperature: string + utilization: string + memoryTotal: string + memoryFree: string + memoryUtilization: string +} diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index ce9b2fc14..c7f53455d 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,4 +1,4 @@ -import { MonitoringExtension, executeOnMain } from '@janhq/core' +import { GpuSetting, MonitoringExtension, executeOnMain } from '@janhq/core' /** * JanMonitoringExtension is a extension that provides system monitoring functionality. @@ -8,19 +8,30 @@ export default class JanMonitoringExtension extends MonitoringExtension { /** * Called when the extension is loaded. */ - async onLoad() {} + async onLoad() { + // Attempt to fetch nvidia info + await executeOnMain(NODE, 'updateNvidiaInfo') + } /** * Called when the extension is unloaded. */ onUnload(): void {} + /** + * Returns the GPU configuration. + * @returns A Promise that resolves to an object containing the GPU configuration. + */ + async getGpuSetting(): Promise { + return executeOnMain(NODE, 'getGpuConfig') + } + /** * Returns information about the system resources. * @returns A Promise that resolves to an object containing information about the system resources. */ getResourcesInfo(): Promise { - return executeOnMain(MODULE, 'getResourcesInfo') + return executeOnMain(NODE, 'getResourcesInfo') } /** @@ -28,6 +39,6 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the current system load. */ getCurrentLoad(): Promise { - return executeOnMain(MODULE, 'getCurrentLoad') + return executeOnMain(NODE, 'getCurrentLoad') } } diff --git a/extensions/monitoring-extension/src/module.ts b/extensions/monitoring-extension/src/module.ts deleted file mode 100644 index 27781a5d6..000000000 --- a/extensions/monitoring-extension/src/module.ts +++ /dev/null @@ -1,92 +0,0 @@ -const nodeOsUtils = require('node-os-utils') -const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath -const path = require('path') -const { readFileSync } = require('fs') -const exec = require('child_process').exec - -const NVIDIA_INFO_FILE = path.join( - getJanDataFolderPath(), - 'settings', - 'settings.json' -) - -const getResourcesInfo = () => - new Promise((resolve) => { - nodeOsUtils.mem.used().then((ramUsedInfo) => { - const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 - const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 - const response = { - mem: { - totalMemory, - usedMemory, - }, - } - resolve(response) - }) - }) - -const getCurrentLoad = () => - new Promise((resolve, reject) => { - nodeOsUtils.cpu.usage().then((cpuPercentage) => { - let data = { - run_mode: 'cpu', - gpus_in_use: [], - } - if (process.platform !== 'darwin') { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) - } - if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { - const gpuIds = data['gpus_in_use'].join(',') - if (gpuIds !== '' && data['vulkan'] !== true) { - exec( - `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, - (error, stdout, _) => { - if (error) { - console.error(`exec error: ${error}`) - reject(error) - return - } - const gpuInfo = stdout - .trim() - .split('\n') - .map((line) => { - const [ - id, - name, - temperature, - utilization, - memoryTotal, - memoryFree, - memoryUtilization, - ] = line.split(', ').map((item) => item.replace(/\r/g, '')) - return { - id, - name, - temperature, - utilization, - memoryTotal, - memoryFree, - memoryUtilization, - } - }) - resolve({ - cpu: { usage: cpuPercentage }, - gpu: gpuInfo, - }) - } - ) - } else { - // Handle the case where gpuIds is empty - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) - } - } else { - // Handle the case where run_mode is not 'gpu' or no GPUs are in use - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) - } - }) - }) - -module.exports = { - getResourcesInfo, - getCurrentLoad, -} diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts new file mode 100644 index 000000000..1d65704de --- /dev/null +++ b/extensions/monitoring-extension/src/node/index.ts @@ -0,0 +1,317 @@ +import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core' +import { getJanDataFolderPath, log } from '@janhq/core/node' +import { mem, cpu } from 'node-os-utils' +import { exec } from 'child_process' +import { writeFileSync, existsSync, readFileSync } from 'fs' +import path from 'path' + +/** + * Path to the settings file + **/ +export const GPU_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) + +/** + * Default GPU settings + * TODO: This needs to be refactored to support multiple accelerators + **/ +const DEFAULT_SETTINGS: GpuSetting = { + notify: true, + run_mode: 'cpu', + nvidia_driver: { + exist: false, + version: '', + }, + cuda: { + exist: false, + version: '', + }, + gpus: [], + gpu_highest_vram: '', + gpus_in_use: [], + is_initial: true, + // TODO: This needs to be set based on user toggle in settings + vulkan: false, +} + +export const getGpuConfig = async (): Promise => { + if (process.platform === 'darwin') return undefined + return JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) +} + +export const getResourcesInfo = async (): Promise => { + const ramUsedInfo = await mem.used() + const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 + const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 + + const resourceInfo: ResourceInfo = { + mem: { + totalMemory, + usedMemory, + }, + } + + return resourceInfo +} + +export const getCurrentLoad = () => + new Promise(async (resolve, reject) => { + const cpuPercentage = await cpu.usage() + let data = { + run_mode: 'cpu', + gpus_in_use: [], + } + + if (process.platform !== 'darwin') { + data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + } + + if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { + const gpuIds = data.gpus_in_use.join(',') + if (gpuIds !== '' && data['vulkan'] !== true) { + exec( + `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, + (error, stdout, _) => { + if (error) { + console.error(`exec error: ${error}`) + throw new Error(error.message) + } + const gpuInfo: GpuInfo[] = stdout + .trim() + .split('\n') + .map((line) => { + const [ + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + ] = line.split(', ').map((item) => item.replace(/\r/g, '')) + return { + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + } + }) + + resolve({ + cpu: { usage: cpuPercentage }, + gpu: gpuInfo, + }) + } + ) + } else { + // Handle the case where gpuIds is empty + resolve({ + cpu: { usage: cpuPercentage }, + gpu: [], + }) + } + } else { + // Handle the case where run_mode is not 'gpu' or no GPUs are in use + resolve({ + cpu: { usage: cpuPercentage }, + gpu: [], + }) + } + }) + +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export const updateNvidiaInfo = async () => { + // ignore if macos + if (process.platform === 'darwin') return + + try { + JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + } catch (error) { + writeFileSync(GPU_INFO_FILE, JSON.stringify(DEFAULT_SETTINGS, null, 2)) + } + + await updateNvidiaDriverInfo() + await updateGpuInfo() +} + +const updateNvidiaDriverInfo = async () => + new Promise((resolve, reject) => { + exec( + 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', + (error, stdout) => { + const data: GpuSetting = JSON.parse( + readFileSync(GPU_INFO_FILE, 'utf-8') + ) + + if (!error) { + const firstLine = stdout.split('\n')[0].trim() + data.nvidia_driver.exist = true + data.nvidia_driver.version = firstLine + } else { + data.nvidia_driver.exist = false + } + + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + resolve({}) + } + ) + }) + +const getGpuArch = (gpuName: string): string => { + if (!gpuName.toLowerCase().includes('nvidia')) return 'unknown' + + if (gpuName.includes('20')) return 'turing' + else if (gpuName.includes('30')) return 'ampere' + else if (gpuName.includes('40')) return 'ada' + else return 'unknown' +} + +const updateGpuInfo = async () => + new Promise((resolve, reject) => { + let data: GpuSetting = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + + // Cuda + if (data.vulkan === true) { + // Vulkan + exec( + process.platform === 'win32' + ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` + : `${__dirname}/../bin/vulkaninfo --summary`, + (error, stdout) => { + if (!error) { + const output = stdout.toString() + + log(output) + const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g + + const gpus: GpuSettingInfo[] = [] + let match + while ((match = gpuRegex.exec(output)) !== null) { + const id = match[1] + const name = match[2] + const arch = getGpuArch(name) + gpus.push({ id, vram: '0', name, arch }) + } + data.gpus = gpus + + if (!data.gpus_in_use || data.gpus_in_use.length === 0) { + data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + resolve({}) + } else { + reject(error) + } + } + ) + } else { + exec( + 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', + (error, stdout) => { + if (!error) { + log(stdout) + // Get GPU info and gpu has higher memory first + let highestVram = 0 + let highestVramId = '0' + const gpus: GpuSettingInfo[] = stdout + .trim() + .split('\n') + .map((line) => { + let [id, vram, name] = line.split(', ') + const arch = getGpuArch(name) + vram = vram.replace(/\r/g, '') + if (parseFloat(vram) > highestVram) { + highestVram = parseFloat(vram) + highestVramId = id + } + return { id, vram, name, arch } + }) + + data.gpus = gpus + data.gpu_highest_vram = highestVramId + } else { + data.gpus = [] + data.gpu_highest_vram = '' + } + + if (!data.gpus_in_use || data.gpus_in_use.length === 0) { + data.gpus_in_use = [data.gpu_highest_vram] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + resolve({}) + } + ) + } + }) + +/** + * Check if file exists in paths + */ +const checkFileExistenceInPaths = (file: string, paths: string[]): boolean => { + return paths.some((p) => existsSync(path.join(p, file))) +} + +/** + * Validate cuda for linux and windows + */ +const updateCudaExistence = ( + data: GpuSetting = DEFAULT_SETTINGS +): GpuSetting => { + let filesCuda12: string[] + let filesCuda11: string[] + let paths: string[] + let cudaVersion: string = '' + + if (process.platform === 'win32') { + filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] + filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] + } else { + filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] + filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path.delimiter) + : [] + paths.push('/usr/lib/x86_64-linux-gnu/') + } + + let cudaExists = filesCuda12.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + if (cudaExists) { + cudaVersion = '11' + } + } else { + cudaVersion = '12' + } + + data.cuda.exist = cudaExists + data.cuda.version = cudaVersion + + console.debug(data.is_initial, data.gpus_in_use) + + if (cudaExists && data.is_initial && data.gpus_in_use.length > 0) { + data.run_mode = 'gpu' + } + + data.is_initial = false + return data +} diff --git a/extensions/monitoring-extension/webpack.config.js b/extensions/monitoring-extension/webpack.config.js deleted file mode 100644 index c8c3a34f7..000000000 --- a/extensions/monitoring-extension/webpack.config.js +++ /dev/null @@ -1,35 +0,0 @@ -const path = require('path') -const webpack = require('webpack') -const packageJson = require('./package.json') - -module.exports = { - experiments: { outputModule: true }, - entry: './src/index.ts', // Adjust the entry point to match your project's main file - mode: 'production', - module: { - rules: [ - { - test: /\.tsx?$/, - use: 'ts-loader', - exclude: /node_modules/, - }, - ], - }, - output: { - filename: 'index.js', // Adjust the output file name as needed - path: path.resolve(__dirname, 'dist'), - library: { type: 'module' }, // Specify ESM output format - }, - plugins: [ - new webpack.DefinePlugin({ - MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - }), - ], - resolve: { - extensions: ['.ts', '.js'], - }, - optimization: { - minimize: false, - }, - // Add loaders and other configuration as needed for your project -} diff --git a/extensions/tensorrt-llm-extension/README.md b/extensions/tensorrt-llm-extension/README.md new file mode 100644 index 000000000..34a670516 --- /dev/null +++ b/extensions/tensorrt-llm-extension/README.md @@ -0,0 +1,79 @@ +# Tensorrt-LLM Extension + +Created using Jan extension example + +# Create a Jan Extension using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 + +## Create Your Own Extension + +To create your own extension, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your extension directory now + +## Update the Extension Metadata + +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your extension. + +## Update the Extension Code + +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your extension code: + +- Most Jan Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { events, MessageEvent, MessageRequest } from '@janhq/core' + + function onStart(): Promise { + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) + } + ``` + + For more information about the Jan Extension Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/tensorrt-llm-extension/models.json b/extensions/tensorrt-llm-extension/models.json new file mode 100644 index 000000000..bc6a78256 --- /dev/null +++ b/extensions/tensorrt-llm-extension/models.json @@ -0,0 +1,49 @@ +[ + { + "sources": [ + { + "filename": "config.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/config.json" + }, + { + "filename": "rank0.engine", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/rank0.engine" + }, + { + "filename": "tokenizer.model", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer.model" + }, + { + "filename": "special_tokens_map.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/special_tokens_map.json" + }, + { + "filename": "tokenizer.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer.json" + }, + { + "filename": "tokenizer_config.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer_config.json" + } + ], + "id": "llamacorn-1.1b-chat-fp16", + "object": "model", + "name": "LlamaCorn 1.1B Chat FP16", + "version": "1.0", + "description": "LlamaCorn is a refined version of TinyLlama-1.1B, optimized for conversational quality, running on consumer devices through TensorRT-LLM", + "format": "TensorRT-LLM", + "settings": { + "ctx_len": 2048 + }, + "parameters": { + "stream": true, + "max_tokens": 4096 + }, + "metadata": { + "author": "LLama", + "tags": ["TensorRT-LLM", "1B", "Finetuned"], + "size": 2151000000 + }, + "engine": "nitro-tensorrt-llm" + } +] diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json new file mode 100644 index 000000000..01ff3e2c6 --- /dev/null +++ b/extensions/tensorrt-llm-extension/package.json @@ -0,0 +1,75 @@ +{ + "name": "@janhq/tensorrt-llm-extension", + "version": "0.0.2", + "description": "Enables accelerated inference leveraging Nvidia's TensorRT-LLM for optimal GPU hardware optimizations. Compatible with models in TensorRT-LLM format. Requires Nvidia GPU driver and CUDA Toolkit installation.", + "main": "dist/index.js", + "node": "dist/node/index.cjs.js", + "author": "Jan ", + "license": "AGPL-3.0", + "config": { + "host": "127.0.0.1", + "port": "3928" + }, + "compatibility": { + "platform": [ + "win32", + "linux" + ], + "app": [ + "0.1.0" + ] + }, + "scripts": { + "build": "tsc --module commonjs && rollup -c rollup.config.ts", + "build:publish:win32": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:linux": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:darwin": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish": "run-script-os" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/node/index.cjs.js" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-replace": "^5.0.5", + "@types/node": "^20.11.4", + "@types/os-utils": "^0.0.4", + "@types/tcp-port-used": "^1.0.4", + "@types/decompress": "4.2.7", + "cpx": "^1.5.0", + "download-cli": "^1.1.1", + "rimraf": "^3.0.2", + "rollup": "^2.38.5", + "rollup-plugin-define": "^1.0.1", + "rollup-plugin-sourcemaps": "^0.6.3", + "rollup-plugin-typescript2": "^0.36.0", + "run-script-os": "^1.1.6", + "typescript": "^5.2.2" + }, + "dependencies": { + "@janhq/core": "file:../../core", + "decompress": "^4.2.1", + "fetch-retry": "^5.0.6", + "path-browserify": "^1.0.1", + "rxjs": "^7.8.1", + "tcp-port-used": "^1.0.2", + "ulid": "^2.3.0" + }, + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ], + "bundleDependencies": [ + "tcp-port-used", + "fetch-retry", + "decompress", + "@janhq/core" + ] +} diff --git a/extensions/tensorrt-llm-extension/rollup.config.ts b/extensions/tensorrt-llm-extension/rollup.config.ts new file mode 100644 index 000000000..33e45823b --- /dev/null +++ b/extensions/tensorrt-llm-extension/rollup.config.ts @@ -0,0 +1,73 @@ +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') + +export default [ + { + input: `src/index.ts`, + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], + watch: { + include: 'src/**', + }, + plugins: [ + replace({ + EXTENSION_NAME: JSON.stringify(packageJson.name), + TENSORRT_VERSION: JSON.stringify('0.1.5'), + DOWNLOAD_RUNNER_URL: + process.platform === 'darwin' || process.platform === 'win32' + ? JSON.stringify( + 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/windows-v/nitro-windows-v-amd64-tensorrt-llm-.tar.gz' + ) + : JSON.stringify( + 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/linux-v/nitro-linux-v-amd64-tensorrt-llm-.tar.gz' + ), + NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + INFERENCE_URL: JSON.stringify( + process.env.INFERENCE_URL || + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/v1/chat/completions` + ), + COMPATIBILITY: JSON.stringify(packageJson.compatibility), + }), + json(), + typescript({ useTsconfigDeclarationDir: true }), + commonjs(), + resolve({ + extensions: ['.js', '.ts', '.svelte'], + }), + sourceMaps(), + ], + }, + { + input: `src/node/index.ts`, + output: [ + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, + ], + external: ['@janhq/core/node'], + watch: { + include: 'src/node/**', + }, + plugins: [ + replace({ + LOAD_MODEL_URL: JSON.stringify( + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/tensorrtllm/loadmodel` + ), + TERMINATE_ENGINE_URL: JSON.stringify( + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/processmanager/destroy` + ), + ENGINE_HOST: JSON.stringify(packageJson.config?.host ?? '127.0.0.1'), + ENGINE_PORT: JSON.stringify(packageJson.config?.port ?? '3928'), + }), + json(), + typescript({ useTsconfigDeclarationDir: true }), + commonjs(), + resolve({ + extensions: ['.ts', '.js', '.json'], + }), + sourceMaps(), + ], + }, +] diff --git a/extensions/tensorrt-llm-extension/src/@types/global.d.ts b/extensions/tensorrt-llm-extension/src/@types/global.d.ts new file mode 100644 index 000000000..905e86380 --- /dev/null +++ b/extensions/tensorrt-llm-extension/src/@types/global.d.ts @@ -0,0 +1,10 @@ +declare const NODE: string +declare const INFERENCE_URL: string +declare const LOAD_MODEL_URL: string +declare const TERMINATE_ENGINE_URL: string +declare const ENGINE_HOST: string +declare const ENGINE_PORT: string +declare const DOWNLOAD_RUNNER_URL: string +declare const TENSORRT_VERSION: string +declare const COMPATIBILITY: object +declare const EXTENSION_NAME: string diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts new file mode 100644 index 000000000..076951c3f --- /dev/null +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -0,0 +1,147 @@ +/** + * @module tensorrt-llm-extension/src/index + */ + +import { + Compatibility, + DownloadEvent, + DownloadRequest, + DownloadState, + GpuSetting, + InstallationState, + Model, + baseName, + downloadFile, + events, + executeOnMain, + joinPath, + showToast, + systemInformations, + LocalOAIEngine, + fs, +} from '@janhq/core' +import models from '../models.json' + +/** + * TensorRTLLMExtension - Implementation of LocalOAIEngine + * @extends BaseOAILocalInferenceProvider + * Provide pre-populated models for TensorRTLLM + */ +export default class TensorRTLLMExtension extends LocalOAIEngine { + /** + * Override custom function name for loading and unloading model + * Which are implemented from node module + */ + override provider = 'nitro-tensorrt-llm' + override inferenceUrl = INFERENCE_URL + override nodeModule = NODE + + private supportedGpuArch = ['turing', 'ampere', 'ada'] + + compatibility() { + return COMPATIBILITY as unknown as Compatibility + } + /** + * models implemented by the extension + * define pre-populated models + */ + async models(): Promise { + if ((await this.installationState()) === 'Installed') + return models as unknown as Model[] + return [] + } + + override async install(): Promise { + const info = await systemInformations() + console.debug( + `TensorRTLLMExtension installing pre-requisites... ${JSON.stringify(info)}` + ) + const gpuSetting: GpuSetting | undefined = info.gpuSetting + if (gpuSetting === undefined || gpuSetting.gpus.length === 0) { + console.error('No GPU setting found. Please check your GPU setting.') + return + } + + // TODO: we only check for the first graphics card. Need to refactor this later. + const firstGpu = gpuSetting.gpus[0] + if (!firstGpu.name.toLowerCase().includes('nvidia')) { + console.error('No Nvidia GPU found. Please check your GPU setting.') + return + } + + if (firstGpu.arch === undefined) { + console.error('No GPU architecture found. Please check your GPU setting.') + return + } + + if (!this.supportedGpuArch.includes(firstGpu.arch)) { + console.error( + `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` + ) + return + } + + const binaryFolderPath = await executeOnMain( + this.nodeModule, + 'binaryFolder' + ) + if (!(await fs.existsSync(binaryFolderPath))) { + await fs.mkdirSync(binaryFolderPath) + } + + const placeholderUrl = DOWNLOAD_RUNNER_URL + const tensorrtVersion = TENSORRT_VERSION + + const url = placeholderUrl + .replace(//g, tensorrtVersion) + .replace(//g, firstGpu.arch) + + const tarball = await baseName(url) + + const tarballFullPath = await joinPath([binaryFolderPath, tarball]) + const downloadRequest: DownloadRequest = { + url, + localPath: tarballFullPath, + extensionId: EXTENSION_NAME, + downloadType: 'extension', + } + downloadFile(downloadRequest) + + // TODO: wrap this into a Promise + const onFileDownloadSuccess = async (state: DownloadState) => { + // if other download, ignore + if (state.fileName !== tarball) return + events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + await executeOnMain(this.nodeModule, 'decompressRunner', tarballFullPath) + events.emit(DownloadEvent.onFileUnzipSuccess, state) + + // Prepopulate models as soon as it's ready + this.prePopulateModels().then(() => { + showToast( + 'Extension installed successfully.', + 'New models are added to Model Hub.' + ) + }) + } + events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + } + + override async installationState(): Promise { + // For now, we just check the executable of nitro x tensor rt + const isNitroExecutableAvailable = await executeOnMain( + this.nodeModule, + 'isNitroExecutableAvailable' + ) + + return isNitroExecutableAvailable ? 'Installed' : 'NotInstalled' + } + + override onInferenceStopped() { + if (!this.isRunning) return + showToast( + 'Unable to Stop Inference', + 'The model does not support stopping inference.' + ) + return Promise.resolve() + } +} diff --git a/extensions/tensorrt-llm-extension/src/node/index.ts b/extensions/tensorrt-llm-extension/src/node/index.ts new file mode 100644 index 000000000..252468fc1 --- /dev/null +++ b/extensions/tensorrt-llm-extension/src/node/index.ts @@ -0,0 +1,191 @@ +import path from 'path' +import { ChildProcessWithoutNullStreams, spawn } from 'child_process' +import tcpPortUsed from 'tcp-port-used' +import fetchRT from 'fetch-retry' +import { log } from '@janhq/core/node' +import { existsSync } from 'fs' +import decompress from 'decompress' + +// Polyfill fetch with retry +const fetchRetry = fetchRT(fetch) + +/** + * The response object for model init operation. + */ +interface ModelLoadParams { + engine_path: string + ctx_len: number +} + +// The subprocess instance for Engine +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined + +/** + * Initializes a engine subprocess to load a machine learning model. + * @param params - The model load settings. + */ +async function loadModel(params: any): Promise<{ error: Error | undefined }> { + // modelFolder is the absolute path to the running model folder + // e.g. ~/jan/models/llama-2 + let modelFolder = params.modelFolder + + const settings: ModelLoadParams = { + engine_path: modelFolder, + ctx_len: params.model.settings.ctx_len ?? 2048, + } + return runEngineAndLoadModel(settings) +} + +/** + * Stops a Engine subprocess. + */ +function unloadModel(): Promise { + const controller = new AbortController() + setTimeout(() => controller.abort(), 5000) + debugLog(`Request to kill engine`) + + subprocess?.kill() + return fetch(TERMINATE_ENGINE_URL, { + method: 'DELETE', + signal: controller.signal, + }) + .then(() => { + subprocess = undefined + }) + .catch(() => {}) // Do nothing with this attempt + .then(() => tcpPortUsed.waitUntilFree(parseInt(ENGINE_PORT), 300, 5000)) // Wait for port available + .then(() => debugLog(`Engine process is terminated`)) + .catch((err) => { + debugLog( + `Could not kill running process on port ${ENGINE_PORT}. Might be another process running on the same port? ${err}` + ) + throw 'PORT_NOT_AVAILABLE' + }) +} +/** + * 1. Spawn engine process + * 2. Load model into engine subprocess + * @returns + */ +async function runEngineAndLoadModel(settings: ModelLoadParams) { + return unloadModel() + .then(runEngine) + .then(() => loadModelRequest(settings)) + .catch((err) => { + // TODO: Broadcast error so app could display proper error message + debugLog(`${err}`, 'Error') + return { error: err } + }) +} + +/** + * Loads a LLM model into the Engine subprocess by sending a HTTP POST request. + */ +function loadModelRequest( + settings: ModelLoadParams +): Promise<{ error: Error | undefined }> { + debugLog(`Loading model with params ${JSON.stringify(settings)}`) + return fetchRetry(LOAD_MODEL_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + }) + .then((res) => { + debugLog(`Load model success with response ${JSON.stringify(res)}`) + return Promise.resolve({ error: undefined }) + }) + .catch((err) => { + debugLog(`Load model failed with error ${err}`, 'Error') + return Promise.resolve({ error: err }) + }) +} + +/** + * Spawns engine subprocess. + */ +function runEngine(): Promise { + debugLog(`Spawning engine subprocess...`) + + return new Promise((resolve, reject) => { + // Current directory by default + let binaryFolder = path.join(__dirname, '..', 'bin') + // Binary path + const binary = path.join( + binaryFolder, + process.platform === 'win32' ? 'nitro.exe' : 'nitro' + ) + + const args: string[] = ['1', ENGINE_HOST, ENGINE_PORT] + // Execute the binary + debugLog(`Spawn nitro at path: ${binary}, and args: ${args}`) + subprocess = spawn(binary, args, { + cwd: binaryFolder, + env: { + ...process.env, + }, + }) + + // Handle subprocess output + subprocess.stdout.on('data', (data: any) => { + debugLog(`${data}`) + }) + + subprocess.stderr.on('data', (data: any) => { + debugLog(`${data}`) + }) + + subprocess.on('close', (code: any) => { + debugLog(`Engine exited with code: ${code}`) + subprocess = undefined + reject(`child process exited with code ${code}`) + }) + + tcpPortUsed.waitUntilUsed(parseInt(ENGINE_PORT), 300, 30000).then(() => { + debugLog(`Engine is ready`) + resolve() + }) + }) +} + +function debugLog(message: string, level: string = 'Debug') { + log(`[TENSORRT_LLM_NITRO]::${level}:${message}`) +} + +const binaryFolder = async (): Promise => { + return path.join(__dirname, '..', 'bin') +} + +const decompressRunner = async (zipPath: string) => { + const output = path.join(__dirname, '..', 'bin') + console.debug(`Decompressing ${zipPath} to ${output}...`) + try { + const files = await decompress(zipPath, output) + console.debug('Decompress finished!', files) + } catch (err) { + console.error(`Decompress ${zipPath} failed: ${err}`) + } +} + +const isNitroExecutableAvailable = async (): Promise => { + const binary = path.join( + __dirname, + '..', + 'bin', + process.platform === 'win32' ? 'nitro.exe' : 'nitro' + ) + + return existsSync(binary) +} + +export default { + binaryFolder, + decompressRunner, + loadModel, + unloadModel, + dispose: unloadModel, + isNitroExecutableAvailable, +} diff --git a/extensions/tensorrt-llm-extension/tsconfig.json b/extensions/tensorrt-llm-extension/tsconfig.json new file mode 100644 index 000000000..478a05728 --- /dev/null +++ b/extensions/tensorrt-llm-extension/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "moduleResolution": "node", + "target": "es5", + "module": "ES2020", + "lib": ["es2015", "es2016", "es2017", "dom"], + "strict": true, + "sourceMap": true, + "declaration": true, + "allowSyntheticDefaultImports": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declarationDir": "dist/types", + "outDir": "dist", + "importHelpers": true, + "resolveJsonModule": true, + "typeRoots": ["node_modules/@types"] + }, + "include": ["src"] +} diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index c05d26e51..ef7532063 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -73,8 +73,9 @@ const DropdownListSidebar = ({ const [copyId, setCopyId] = useState('') + // TODO: Update filter condition for the local model const localModel = downloadedModels.filter( - (model) => model.engine === InferenceEngine.nitro + (model) => model.engine !== InferenceEngine.openai ) const remoteModel = downloadedModels.filter( (model) => model.engine === InferenceEngine.openai diff --git a/web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx b/web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx new file mode 100644 index 000000000..d46764e88 --- /dev/null +++ b/web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx @@ -0,0 +1,87 @@ +import { useCallback, useEffect } from 'react' + +import { abortDownload } from '@janhq/core' +import { + Button, + Modal, + ModalContent, + ModalHeader, + ModalTitle, + Progress, +} from '@janhq/uikit' +import { atom, useAtom, useAtomValue } from 'jotai' + +import { + formatDownloadPercentage, + formatExtensionsName, +} from '@/utils/converter' + +import { + InstallingExtensionState, + installingExtensionAtom, +} from '@/helpers/atoms/Extension.atom' + +export const showInstallingExtensionModalAtom = atom(false) + +const InstallingExtensionModal: React.FC = () => { + const [showInstallingExtensionModal, setShowInstallingExtensionModal] = + useAtom(showInstallingExtensionModalAtom) + const installingExtensions = useAtomValue(installingExtensionAtom) + + useEffect(() => { + if (installingExtensions.length === 0) { + setShowInstallingExtensionModal(false) + } + }, [installingExtensions, setShowInstallingExtensionModal]) + + const onAbortInstallingExtensionClick = useCallback( + (item: InstallingExtensionState) => { + if (item.localPath) { + abortDownload(item.localPath) + } + }, + [] + ) + + return ( + setShowInstallingExtensionModal(false)} + > + + + Installing Extension + + {Object.values(installingExtensions).map((item) => ( +
+ +
+
+

+ {formatExtensionsName(item.extensionId)} +

+ {formatDownloadPercentage(item.percentage)} +
+ +
+
+ ))} + + + ) +} + +export default InstallingExtensionModal diff --git a/web/containers/Layout/BottomBar/InstallingExtension/index.tsx b/web/containers/Layout/BottomBar/InstallingExtension/index.tsx new file mode 100644 index 000000000..05e803881 --- /dev/null +++ b/web/containers/Layout/BottomBar/InstallingExtension/index.tsx @@ -0,0 +1,52 @@ +import { Fragment, useCallback } from 'react' + +import { Progress } from '@janhq/uikit' +import { useAtomValue, useSetAtom } from 'jotai' + +import { showInstallingExtensionModalAtom } from './InstallingExtensionModal' + +import { installingExtensionAtom } from '@/helpers/atoms/Extension.atom' + +const InstallingExtension: React.FC = () => { + const installingExtensions = useAtomValue(installingExtensionAtom) + const setShowInstallingExtensionModal = useSetAtom( + showInstallingExtensionModalAtom + ) + const shouldShowInstalling = installingExtensions.length > 0 + + let totalPercentage = 0 + let totalExtensions = 0 + for (const installation of installingExtensions) { + totalPercentage += installation.percentage + totalExtensions++ + } + const progress = (totalPercentage / totalExtensions) * 100 + + const onClick = useCallback(() => { + setShowInstallingExtensionModal(true) + }, [setShowInstallingExtensionModal]) + + return ( + + {shouldShowInstalling ? ( +
+

+ Installing Extension +

+ +
+ + + {progress.toFixed(2)}% + +
+
+ ) : null} + + ) +} + +export default InstallingExtension diff --git a/web/containers/Layout/BottomBar/index.tsx b/web/containers/Layout/BottomBar/index.tsx index 2373ac3d4..3683d23db 100644 --- a/web/containers/Layout/BottomBar/index.tsx +++ b/web/containers/Layout/BottomBar/index.tsx @@ -16,6 +16,7 @@ import ProgressBar from '@/containers/ProgressBar' import { appDownloadProgress } from '@/containers/Providers/Jotai' import ImportingModelState from './ImportingModelState' +import InstallingExtension from './InstallingExtension' import SystemMonitor from './SystemMonitor' import UpdatedFailedModal from './UpdateFailedModal' @@ -46,6 +47,7 @@ const BottomBar = () => { +
diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index 7e3ad38ab..fb08bc6ac 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -22,6 +22,8 @@ import ImportModelOptionModal from '@/screens/Settings/ImportModelOptionModal' import ImportingModelModal from '@/screens/Settings/ImportingModelModal' import SelectingModelModal from '@/screens/Settings/SelectingModelModal' +import InstallingExtensionModal from './BottomBar/InstallingExtension/InstallingExtensionModal' + import { mainViewStateAtom } from '@/helpers/atoms/App.atom' const BaseLayout = (props: PropsWithChildren) => { @@ -68,6 +70,7 @@ const BaseLayout = (props: PropsWithChildren) => { {importModelStage === 'IMPORTING_MODEL' && } {importModelStage === 'EDIT_MODEL_INFO' && } {importModelStage === 'CONFIRM_CANCEL' && } +
) } diff --git a/web/containers/Providers/EventListener.tsx b/web/containers/Providers/EventListener.tsx index bfc87917b..20fc6dde2 100644 --- a/web/containers/Providers/EventListener.tsx +++ b/web/containers/Providers/EventListener.tsx @@ -7,6 +7,10 @@ import { useSetAtom } from 'jotai' import { setDownloadStateAtom } from '@/hooks/useDownloadState' +import { formatExtensionsName } from '@/utils/converter' + +import { toaster } from '../Toast' + import AppUpdateListener from './AppUpdateListener' import ClipboardListener from './ClipboardListener' import EventHandler from './EventHandler' @@ -14,46 +18,89 @@ import EventHandler from './EventHandler' import ModelImportListener from './ModelImportListener' import QuickAskListener from './QuickAskListener' +import { + InstallingExtensionState, + removeInstallingExtensionAtom, + setInstallingExtensionAtom, +} from '@/helpers/atoms/Extension.atom' + const EventListenerWrapper = ({ children }: PropsWithChildren) => { const setDownloadState = useSetAtom(setDownloadStateAtom) + const setInstallingExtension = useSetAtom(setInstallingExtensionAtom) + const removeInstallingExtension = useSetAtom(removeInstallingExtensionAtom) const onFileDownloadUpdate = useCallback( async (state: DownloadState) => { console.debug('onFileDownloadUpdate', state) - setDownloadState(state) + if (state.downloadType === 'extension') { + const installingExtensionState: InstallingExtensionState = { + extensionId: state.extensionId!, + percentage: state.percent, + localPath: state.localPath, + } + setInstallingExtension(state.extensionId!, installingExtensionState) + } else { + setDownloadState(state) + } }, - [setDownloadState] + [setDownloadState, setInstallingExtension] ) const onFileDownloadError = useCallback( (state: DownloadState) => { console.debug('onFileDownloadError', state) - setDownloadState(state) + if (state.downloadType === 'extension') { + removeInstallingExtension(state.extensionId!) + } else { + setDownloadState(state) + } }, - [setDownloadState] + [setDownloadState, removeInstallingExtension] ) const onFileDownloadSuccess = useCallback( (state: DownloadState) => { console.debug('onFileDownloadSuccess', state) - setDownloadState(state) + if (state.downloadType !== 'extension') { + setDownloadState(state) + } }, [setDownloadState] ) + const onFileUnzipSuccess = useCallback( + (state: DownloadState) => { + console.debug('onFileUnzipSuccess', state) + toaster({ + title: 'Success', + description: `Install ${formatExtensionsName(state.extensionId!)} successfully.`, + type: 'success', + }) + removeInstallingExtension(state.extensionId!) + }, + [removeInstallingExtension] + ) + useEffect(() => { console.debug('EventListenerWrapper: registering event listeners...') events.on(DownloadEvent.onFileDownloadUpdate, onFileDownloadUpdate) events.on(DownloadEvent.onFileDownloadError, onFileDownloadError) events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + events.on(DownloadEvent.onFileUnzipSuccess, onFileUnzipSuccess) return () => { console.debug('EventListenerWrapper: unregistering event listeners...') events.off(DownloadEvent.onFileDownloadUpdate, onFileDownloadUpdate) events.off(DownloadEvent.onFileDownloadError, onFileDownloadError) events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + events.off(DownloadEvent.onFileUnzipSuccess, onFileUnzipSuccess) } - }, [onFileDownloadUpdate, onFileDownloadError, onFileDownloadSuccess]) + }, [ + onFileDownloadUpdate, + onFileDownloadError, + onFileDownloadSuccess, + onFileUnzipSuccess, + ]) return ( diff --git a/web/extension/ExtensionManager.ts b/web/extension/ExtensionManager.ts index 1259021f7..c976010c6 100644 --- a/web/extension/ExtensionManager.ts +++ b/web/extension/ExtensionManager.ts @@ -23,7 +23,9 @@ export class ExtensionManager { * @param type - The type of the extension to retrieve. * @returns The extension, if found. */ - get(type: ExtensionTypeEnum): T | undefined { + get( + type: ExtensionTypeEnum | string + ): T | undefined { return this.extensions.get(type) as T | undefined } diff --git a/web/helpers/atoms/Extension.atom.ts b/web/helpers/atoms/Extension.atom.ts new file mode 100644 index 000000000..7af755e35 --- /dev/null +++ b/web/helpers/atoms/Extension.atom.ts @@ -0,0 +1,40 @@ +import { atom } from 'jotai' + +type ExtensionId = string + +export type InstallingExtensionState = { + extensionId: ExtensionId + percentage: number + localPath?: string +} + +export const installingExtensionAtom = atom([]) + +export const setInstallingExtensionAtom = atom( + null, + (get, set, extensionId: string, state: InstallingExtensionState) => { + const current = get(installingExtensionAtom) + + const isExists = current.some((e) => e.extensionId === extensionId) + if (isExists) { + const newCurrent = current.map((e) => { + if (e.extensionId === extensionId) { + return state + } + return e + }) + set(installingExtensionAtom, newCurrent) + } else { + set(installingExtensionAtom, [...current, state]) + } + } +) + +export const removeInstallingExtensionAtom = atom( + null, + (get, set, extensionId: string) => { + const current = get(installingExtensionAtom) + const newCurrent = current.filter((e) => e.extensionId !== extensionId) + set(installingExtensionAtom, newCurrent) + } +) diff --git a/web/hooks/useActiveModel.ts b/web/hooks/useActiveModel.ts index 600e10783..e6c519f9f 100644 --- a/web/hooks/useActiveModel.ts +++ b/web/hooks/useActiveModel.ts @@ -40,6 +40,16 @@ export function useActiveModel() { console.debug(`Model ${modelId} is already initialized. Ignore..`) return } + + let model = downloadedModelsRef?.current.find((e) => e.id === modelId) + + // Switch between engines + if (model && activeModel && activeModel.engine !== model.engine) { + stopModel() + // TODO: Refactor inference provider would address this + await new Promise((res) => setTimeout(res, 1000)) + } + // TODO: incase we have multiple assistants, the configuration will be from assistant setLoadModelError(undefined) @@ -47,8 +57,6 @@ export function useActiveModel() { setStateModel({ state: 'start', loading: true, model: modelId }) - let model = downloadedModelsRef?.current.find((e) => e.id === modelId) - if (!model) { toaster({ title: `Model ${modelId} not found!`, diff --git a/web/hooks/useDownloadModel.ts b/web/hooks/useDownloadModel.ts index 9f6334c71..d0d13d93b 100644 --- a/web/hooks/useDownloadModel.ts +++ b/web/hooks/useDownloadModel.ts @@ -8,12 +8,15 @@ import { joinPath, ModelArtifact, DownloadState, + GpuSetting, } from '@janhq/core' import { useAtomValue, useSetAtom } from 'jotai' import { setDownloadStateAtom } from './useDownloadState' +import useGpuSetting from './useGpuSetting' + import { extensionManager } from '@/extension/ExtensionManager' import { ignoreSslAtom, @@ -29,6 +32,8 @@ export default function useDownloadModel() { const setDownloadState = useSetAtom(setDownloadStateAtom) const addDownloadingModel = useSetAtom(addDownloadingModelAtom) + const { getGpuSettings } = useGpuSetting() + const downloadModel = useCallback( async (model: Model) => { const childProgresses: DownloadState[] = model.sources.map( @@ -68,10 +73,22 @@ export default function useDownloadModel() { }) addDownloadingModel(model) - - await localDownloadModel(model, ignoreSSL, proxyEnabled ? proxy : '') + const gpuSettings = await getGpuSettings() + await localDownloadModel( + model, + ignoreSSL, + proxyEnabled ? proxy : '', + gpuSettings + ) }, - [ignoreSSL, proxy, proxyEnabled, addDownloadingModel, setDownloadState] + [ + ignoreSSL, + proxy, + proxyEnabled, + getGpuSettings, + addDownloadingModel, + setDownloadState, + ] ) const abortModelDownload = useCallback(async (model: Model) => { @@ -90,8 +107,9 @@ export default function useDownloadModel() { const localDownloadModel = async ( model: Model, ignoreSSL: boolean, - proxy: string + proxy: string, + gpuSettings?: GpuSetting ) => extensionManager .get(ExtensionTypeEnum.Model) - ?.downloadModel(model, { ignoreSSL, proxy }) + ?.downloadModel(model, gpuSettings, { ignoreSSL, proxy }) diff --git a/web/hooks/useDownloadState.ts b/web/hooks/useDownloadState.ts index 06de9bef6..03a8883cb 100644 --- a/web/hooks/useDownloadState.ts +++ b/web/hooks/useDownloadState.ts @@ -18,123 +18,129 @@ export const modelDownloadStateAtom = atom>({}) export const setDownloadStateAtom = atom( null, (get, set, state: DownloadState) => { - const currentState = { ...get(modelDownloadStateAtom) } + try { + const currentState = { ...get(modelDownloadStateAtom) } - if (state.downloadState === 'end') { - const modelDownloadState = currentState[state.modelId] + if (state.downloadState === 'end') { + const modelDownloadState = currentState[state.modelId] - const updatedChildren: DownloadState[] = - modelDownloadState.children!.filter( - (m) => m.fileName !== state.fileName + const updatedChildren: DownloadState[] = ( + modelDownloadState.children ?? [] + ).filter((m) => m.fileName !== state.fileName) + updatedChildren.push(state) + modelDownloadState.children = updatedChildren + currentState[state.modelId] = modelDownloadState + + const isAllChildrenDownloadEnd = modelDownloadState.children?.every( + (m) => m.downloadState === 'end' ) - updatedChildren.push(state) - modelDownloadState.children = updatedChildren - currentState[state.modelId] = modelDownloadState - const isAllChildrenDownloadEnd = modelDownloadState.children?.every( - (m) => m.downloadState === 'end' - ) + if (isAllChildrenDownloadEnd) { + // download successfully + delete currentState[state.modelId] + set(removeDownloadingModelAtom, state.modelId) - if (isAllChildrenDownloadEnd) { - // download successfully + const model = get(configuredModelsAtom).find( + (e) => e.id === state.modelId + ) + if (model) set(downloadedModelsAtom, (prev) => [...prev, model]) + toaster({ + title: 'Download Completed', + description: `Download ${state.modelId} completed`, + type: 'success', + }) + } + } else if (state.downloadState === 'error') { + // download error delete currentState[state.modelId] set(removeDownloadingModelAtom, state.modelId) - - const model = get(configuredModelsAtom).find( - (e) => e.id === state.modelId - ) - if (model) set(downloadedModelsAtom, (prev) => [...prev, model]) - toaster({ - title: 'Download Completed', - description: `Download ${state.modelId} completed`, - type: 'success', - }) - } - } else if (state.downloadState === 'error') { - // download error - delete currentState[state.modelId] - set(removeDownloadingModelAtom, state.modelId) - if (state.error === 'aborted') { - toaster({ - title: 'Cancel Download', - description: `Model ${state.modelId} download cancelled`, - type: 'warning', - }) - } else { - let error = state.error - if ( - typeof error?.includes === 'function' && - state.error?.includes('certificate') - ) { - error += - '. To fix enable "Ignore SSL Certificates" in Advanced settings.' + if (state.error === 'aborted') { + toaster({ + title: 'Cancel Download', + description: `Model ${state.modelId} download cancelled`, + type: 'warning', + }) + } else { + let error = state.error + if ( + typeof error?.includes === 'function' && + state.error?.includes('certificate') + ) { + error += + '. To fix enable "Ignore SSL Certificates" in Advanced settings.' + } + toaster({ + title: 'Download Failed', + description: `Model ${state.modelId} download failed: ${error}`, + type: 'error', + }) + } + } else { + // download in progress + if (state.size.total === 0) { + // this is initial state, just set the state + currentState[state.modelId] = state + set(modelDownloadStateAtom, currentState) + return } - toaster({ - title: 'Download Failed', - description: `Model ${state.modelId} download failed: ${error}`, - type: 'error', - }) - } - } else { - // download in progress - if (state.size.total === 0) { - // this is initial state, just set the state - currentState[state.modelId] = state - set(modelDownloadStateAtom, currentState) - return - } - const modelDownloadState = currentState[state.modelId] - if (!modelDownloadState) { - console.debug('setDownloadStateAtom: modelDownloadState not found') - return - } + const modelDownloadState = currentState[state.modelId] + if (!modelDownloadState) { + console.debug('setDownloadStateAtom: modelDownloadState not found') + return + } - // delete the children if the filename is matched and replace the new state - const updatedChildren: DownloadState[] = - modelDownloadState.children!.filter( - (m) => m.fileName !== state.fileName + // delete the children if the filename is matched and replace the new state + const updatedChildren: DownloadState[] = ( + modelDownloadState.children ?? [] + ).filter((m) => m.fileName !== state.fileName) + + updatedChildren.push(state) + + // re-calculate the overall progress if we have all the children download data + const isAnyChildDownloadNotReady = updatedChildren.some( + (m) => + m.size.total === 0 && + !modelDownloadState.children?.some( + (e) => e.fileName === m.fileName && e.downloadState === 'end' + ) && + modelDownloadState.children?.some((e) => e.fileName === m.fileName) ) - updatedChildren.push(state) + modelDownloadState.children = updatedChildren + if (isAnyChildDownloadNotReady) { + // just update the children + currentState[state.modelId] = modelDownloadState + set(modelDownloadStateAtom, currentState) + return + } - // re-calculate the overall progress if we have all the children download data - const isAnyChildDownloadNotReady = updatedChildren.some( - (m) => m.size.total === 0 - ) + const parentTotalSize = modelDownloadState.size.total + if (parentTotalSize === 0) { + // calculate the total size of the parent by sum all children total size + const totalSize = updatedChildren.reduce( + (acc, m) => acc + m.size.total, + 0 + ) - modelDownloadState.children = updatedChildren + modelDownloadState.size.total = totalSize + } - if (isAnyChildDownloadNotReady) { - // just update the children - currentState[state.modelId] = modelDownloadState - set(modelDownloadStateAtom, currentState) - - return - } - - const parentTotalSize = modelDownloadState.size.total - if (parentTotalSize === 0) { - // calculate the total size of the parent by sum all children total size - const totalSize = updatedChildren.reduce( - (acc, m) => acc + m.size.total, + // calculate the total transferred size by sum all children transferred size + const transferredSize = updatedChildren.reduce( + (acc, m) => acc + m.size.transferred, 0 ) - - modelDownloadState.size.total = totalSize + modelDownloadState.size.transferred = transferredSize + modelDownloadState.percent = + parentTotalSize === 0 ? 0 : transferredSize / parentTotalSize + currentState[state.modelId] = modelDownloadState } - // calculate the total transferred size by sum all children transferred size - const transferredSize = updatedChildren.reduce( - (acc, m) => acc + m.size.transferred, - 0 - ) - modelDownloadState.size.transferred = transferredSize - modelDownloadState.percent = - parentTotalSize === 0 ? 0 : transferredSize / parentTotalSize - currentState[state.modelId] = modelDownloadState + set(modelDownloadStateAtom, currentState) + } catch (e) { + console.debug('setDownloadStateAtom: state', state) + console.debug('setDownloadStateAtom: error', e) } - - set(modelDownloadStateAtom, currentState) } ) diff --git a/web/hooks/useGpuSetting.ts b/web/hooks/useGpuSetting.ts new file mode 100644 index 000000000..36f51ed57 --- /dev/null +++ b/web/hooks/useGpuSetting.ts @@ -0,0 +1,21 @@ +import { useCallback } from 'react' + +import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' + +import { extensionManager } from '@/extension' + +export default function useGpuSetting() { + const getGpuSettings = useCallback(async () => { + const gpuSetting = await extensionManager + ?.get(ExtensionTypeEnum.SystemMonitoring) + ?.getGpuSetting() + + if (!gpuSetting) { + console.debug('No GPU setting found') + return undefined + } + return gpuSetting + }, []) + + return { getGpuSettings } +} diff --git a/web/next.config.js b/web/next.config.js index a4b3e6d43..48ea0703e 100644 --- a/web/next.config.js +++ b/web/next.config.js @@ -38,6 +38,7 @@ const nextConfig = { isMac: process.platform === 'darwin', isWindows: process.platform === 'win32', isLinux: process.platform === 'linux', + PLATFORM: JSON.stringify(process.platform), }), ] return config diff --git a/web/screens/ExploreModels/ExploreModelItem/index.tsx b/web/screens/ExploreModels/ExploreModelItem/index.tsx index 9cdfbc01a..e8887e70f 100644 --- a/web/screens/ExploreModels/ExploreModelItem/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItem/index.tsx @@ -3,6 +3,8 @@ import { useState } from 'react' import { Model } from '@janhq/core' import { Badge } from '@janhq/uikit' +import { twMerge } from 'tailwind-merge' + import ExploreModelItemHeader from '@/screens/ExploreModels/ExploreModelItemHeader' type Props = { @@ -75,7 +77,16 @@ const ExploreModelItem: React.FC = ({ model }) => { Format -

{model.format}

+

+ {model.format} +

diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx index 38e7f65a6..581a628ba 100644 --- a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx @@ -152,6 +152,7 @@ const ExploreModelItemHeader: React.FC = ({ model, onClick, open }) => {
{model.name} +
@@ -172,4 +173,21 @@ const ExploreModelItemHeader: React.FC = ({ model, onClick, open }) => { ) } +type EngineBadgeProps = { + engine: string +} + +const EngineBadge: React.FC = ({ engine }) => { + switch (engine) { + case 'nitro-tensorrt-llm': + return ( +
+ TensorRT-LLM +
+ ) + default: + return null + } +} + export default ExploreModelItemHeader diff --git a/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx new file mode 100644 index 000000000..e4c8ae04f --- /dev/null +++ b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx @@ -0,0 +1,225 @@ +import { useCallback, useEffect, useState } from 'react' + +import { + Compatibility, + GpuSetting, + InstallationState, + abortDownload, + systemInformations, +} from '@janhq/core' +import { + Button, + Progress, + Tooltip, + TooltipArrow, + TooltipContent, + TooltipPortal, + TooltipTrigger, +} from '@janhq/uikit' + +import { InfoCircledIcon } from '@radix-ui/react-icons' +import { useAtomValue } from 'jotai' + +import { extensionManager } from '@/extension' +import Extension from '@/extension/Extension' +import { installingExtensionAtom } from '@/helpers/atoms/Extension.atom' + +type Props = { + item: Extension +} + +const TensorRtExtensionItem: React.FC = ({ item }) => { + const [compatibility, setCompatibility] = useState( + undefined + ) + const [installState, setInstallState] = + useState('NotRequired') + const installingExtensions = useAtomValue(installingExtensionAtom) + const [isGpuSupported, setIsGpuSupported] = useState(false) + + const isInstalling = installingExtensions.some( + (e) => e.extensionId === item.name + ) + + const progress = isInstalling + ? installingExtensions.find((e) => e.extensionId === item.name) + ?.percentage ?? -1 + : -1 + + useEffect(() => { + const getSystemInfos = async () => { + const info = await systemInformations() + if (!info) { + setIsGpuSupported(false) + return + } + + const gpuSettings: GpuSetting | undefined = info.gpuSetting + if (!gpuSettings || gpuSettings.gpus.length === 0) { + setIsGpuSupported(false) + return + } + + const arch = gpuSettings.gpus[0].arch + if (!arch) { + setIsGpuSupported(false) + return + } + + const supportedGpuArch = ['turing', 'ampere', 'ada'] + setIsGpuSupported(supportedGpuArch.includes(arch)) + } + getSystemInfos() + }, []) + + useEffect(() => { + const getExtensionInstallationState = async () => { + const extension = extensionManager.get(item.name ?? '') + if (!extension) return + + if (typeof extension?.installationState === 'function') { + const installState = await extension.installationState() + setInstallState(installState) + } + } + + getExtensionInstallationState() + }, [item.name, isInstalling]) + + useEffect(() => { + const extension = extensionManager.get(item.name ?? '') + if (!extension) return + setCompatibility(extension.compatibility()) + }, [setCompatibility, item.name]) + + const onInstallClick = useCallback(async () => { + const extension = extensionManager.get(item.name ?? '') + if (!extension) return + + await extension.install() + }, [item.name]) + + const onCancelInstallingClick = () => { + const extension = installingExtensions.find( + (e) => e.extensionId === item.name + ) + if (extension?.localPath) { + abortDownload(extension.localPath) + } + } + + return ( +
+
+
+
+ TensorRT-LLM Extension +
+

+ v{item.version} +

+
+

+ {item.description} +

+
+ {(!compatibility || compatibility['platform']?.includes(PLATFORM)) && + isGpuSupported ? ( +
+ +
+ ) : ( +
+
+ Incompatible{' '} + + + + + + + {compatibility ? ( + + Only available on{' '} + {compatibility?.platform + ?.map((e: string) => + e === 'win32' + ? 'Windows' + : e === 'linux' + ? 'Linux' + : 'MacOS' + ) + .join(', ')} + + ) : ( + + Your GPUs are not compatible with this extension + + )} + + + + +
+
+ )} +
+ ) +} + +type InstallStateProps = { + installProgress: number + installState: InstallationState + onInstallClick: () => void + onCancelClick: () => void +} + +const InstallStateIndicator: React.FC = ({ + installProgress, + installState, + onInstallClick, + onCancelClick, +}) => { + // TODO: NamH support dark mode for this + if (installProgress !== -1) { + const progress = installProgress * 100 + return ( +
+ +
+ + + {progress.toFixed(0)}% + +
+
+ ) + } + + // TODO: NamH check for dark mode here + switch (installState) { + case 'Installed': + return ( +
+ Installed +
+ ) + case 'NotInstalled': + return ( + + ) + default: + return
+ } +} + +export default TensorRtExtensionItem diff --git a/web/screens/Settings/CoreExtensions/index.tsx b/web/screens/Settings/CoreExtensions/index.tsx index 8c9f92d7a..f5b66abeb 100644 --- a/web/screens/Settings/CoreExtensions/index.tsx +++ b/web/screens/Settings/CoreExtensions/index.tsx @@ -4,13 +4,18 @@ import React, { useState, useEffect, useRef } from 'react' import { Button, ScrollArea } from '@janhq/uikit' +import Loader from '@/containers/Loader' + import { formatExtensionsName } from '@/utils/converter' +import TensorRtExtensionItem from './TensorRtExtensionItem' + import { extensionManager } from '@/extension' import Extension from '@/extension/Extension' const ExtensionCatalog = () => { const [activeExtensions, setActiveExtensions] = useState([]) + const [showLoading, setShowLoading] = useState(false) const fileInputRef = useRef(null) /** * Fetches the active extensions and their preferences from the `extensions` and `preferences` modules. @@ -63,65 +68,76 @@ const ExtensionCatalog = () => { const handleFileChange = (event: React.ChangeEvent) => { const file = event.target.files?.[0] if (file) { + setShowLoading(true) install(event) } } return ( - -
- {activeExtensions.map((item, i) => { - return ( -
-
-
-
- {formatExtensionsName(item.name ?? item.description ?? '')} -
-

- v{item.version} + <> + +

+ {activeExtensions.map((item, i) => { + // TODO: this is bad code, rewrite it + if (item.name === '@janhq/tensorrt-llm-extension') { + return + } + + return ( +
+
+
+
+ {formatExtensionsName( + item.name ?? item.description ?? '' + )} +
+

+ v{item.version} +

+
+

+ {item.description}

-

- {item.description} -

+ ) + })} + {/* Manual Installation */} +
+
+
+
+ Manual Installation +
+
+

+ Select a extension file to install (.tgz) +

- ) - })} - {/* Manual Installation */} -
-
-
-
- Manual Installation -
+
+ +
-

- Select a extension file to install (.tgz) -

-
-
- -
-
- + + {showLoading && } + ) } diff --git a/web/services/appService.ts b/web/services/appService.ts new file mode 100644 index 000000000..9327d55c3 --- /dev/null +++ b/web/services/appService.ts @@ -0,0 +1,24 @@ +import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' + +import { toaster } from '@/containers/Toast' + +import { extensionManager } from '@/extension' + +export const appService = { + systemInformations: async () => { + const gpuSetting = await extensionManager + ?.get(ExtensionTypeEnum.SystemMonitoring) + ?.getGpuSetting() + + return { + gpuSetting, + // TODO: Other system information + } + }, + showToast: (title: string, description: string) => { + toaster({ + title, + description: description, + }) + }, +} diff --git a/web/services/coreService.ts b/web/services/coreService.ts index c010c6cec..a483cc452 100644 --- a/web/services/coreService.ts +++ b/web/services/coreService.ts @@ -1,5 +1,7 @@ +import { appService } from './appService' import { EventEmitter } from './eventsService' import { restAPI } from './restService' + export const setupCoreServices = () => { if (typeof window === 'undefined') { console.debug('undefine', window) @@ -10,7 +12,10 @@ export const setupCoreServices = () => { if (!window.core) { window.core = { events: new EventEmitter(), - api: window.electronAPI ?? restAPI, + api: { + ...(window.electronAPI ? window.electronAPI : restAPI), + ...appService, + }, } } } diff --git a/web/types/index.d.ts b/web/types/index.d.ts index 833c3e2bd..ed83e0d14 100644 --- a/web/types/index.d.ts +++ b/web/types/index.d.ts @@ -11,6 +11,7 @@ declare global { declare const isMac: boolean declare const isWindows: boolean declare const isLinux: boolean + declare const PLATFORM: string interface Core { api: APIFunctions events: EventEmitter From 561b1dd94a07aec5289236410f75f6be2e055d14 Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 14 Mar 2024 07:27:24 +0000 Subject: [PATCH 03/17] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bc206f4eb..b25a917d7 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 03a0978f5345e394522f21f1638d0d62f89d2796 Mon Sep 17 00:00:00 2001 From: NamH Date: Thu, 14 Mar 2024 15:58:42 +0700 Subject: [PATCH 04/17] fix: some costmetic issues: badges corner, recommended for tensorrt models(#2346) Signed-off-by: hiro Co-authored-by: hiro --- uikit/src/badge/styles.scss | 2 +- web/containers/DropdownListSidebar/index.tsx | 2 +- web/screens/ExploreModels/ExploreModelItemHeader/index.tsx | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/uikit/src/badge/styles.scss b/uikit/src/badge/styles.scss index b777892d7..4788f65be 100644 --- a/uikit/src/badge/styles.scss +++ b/uikit/src/badge/styles.scss @@ -1,5 +1,5 @@ .badge { - @apply focus:ring-ring border-border inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-offset-2; + @apply focus:ring-ring border-border inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-offset-2; &-primary { @apply border-transparent bg-blue-100 text-blue-600; diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index ef7532063..70651a4d4 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -294,7 +294,7 @@ const DropdownListSidebar = ({ {toGibibytes(x.metadata.size)} - {x.engine == InferenceEngine.nitro && ( + {x.metadata.size && ( )}
diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx index 581a628ba..ef75f57f5 100644 --- a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx @@ -180,10 +180,11 @@ type EngineBadgeProps = { const EngineBadge: React.FC = ({ engine }) => { switch (engine) { case 'nitro-tensorrt-llm': + const title = 'TensorRT-LLM' return ( -
- TensorRT-LLM -
+ + {title} + ) default: return null From d9c3852997eb42159701fefa5265f7ba3dd74ea3 Mon Sep 17 00:00:00 2001 From: NamH Date: Thu, 14 Mar 2024 17:16:14 +0700 Subject: [PATCH 05/17] fix: ts error when declar var in case (#2348) Signed-off-by: James Co-authored-by: James --- web/screens/ExploreModels/ExploreModelItemHeader/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx index ef75f57f5..465e69fa6 100644 --- a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx @@ -178,9 +178,10 @@ type EngineBadgeProps = { } const EngineBadge: React.FC = ({ engine }) => { + const title = 'TensorRT-LLM' + switch (engine) { case 'nitro-tensorrt-llm': - const title = 'TensorRT-LLM' return ( {title} From 8120ad2a038e0b88333e5044db96b43dd73d4c4e Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Thu, 14 Mar 2024 18:37:54 +0700 Subject: [PATCH 06/17] fix: badge or progress tensorRtExtensionItem (#2349) --- .../Settings/CoreExtensions/TensorRtExtensionItem.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx index e4c8ae04f..b363f79be 100644 --- a/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx +++ b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx @@ -123,6 +123,7 @@ const TensorRtExtensionItem: React.FC = ({ item }) => { {item.description}

+ {(!compatibility || compatibility['platform']?.includes(PLATFORM)) && isGpuSupported ? (
@@ -185,15 +186,14 @@ const InstallStateIndicator: React.FC = ({ onInstallClick, onCancelClick, }) => { - // TODO: NamH support dark mode for this if (installProgress !== -1) { const progress = installProgress * 100 return ( -
+
-
+
{progress.toFixed(0)}% From aab8ee84afd02f6e3da3ead0d9d5c27bcd1fe50f Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 14 Mar 2024 19:04:31 +0700 Subject: [PATCH 07/17] fix: disable rag & stream settings from tensorrt model.json (#2351) --- extensions/tensorrt-llm-extension/models.json | 4 ++-- extensions/tensorrt-llm-extension/src/index.ts | 7 +++++++ web/screens/Chat/ChatInput/index.tsx | 7 ++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/extensions/tensorrt-llm-extension/models.json b/extensions/tensorrt-llm-extension/models.json index bc6a78256..31bb11a9e 100644 --- a/extensions/tensorrt-llm-extension/models.json +++ b/extensions/tensorrt-llm-extension/models.json @@ -33,10 +33,10 @@ "description": "LlamaCorn is a refined version of TinyLlama-1.1B, optimized for conversational quality, running on consumer devices through TensorRT-LLM", "format": "TensorRT-LLM", "settings": { - "ctx_len": 2048 + "ctx_len": 2048, + "text_model": false }, "parameters": { - "stream": true, "max_tokens": 4096 }, "metadata": { diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index 076951c3f..e3014b447 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -19,6 +19,7 @@ import { systemInformations, LocalOAIEngine, fs, + MessageRequest, } from '@janhq/core' import models from '../models.json' @@ -144,4 +145,10 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { ) return Promise.resolve() } + + inference(data: MessageRequest): void { + // TensorRT LLM Extension supports streaming only + if (data.model) data.model.parameters.stream = true + super.inference(data) + } } diff --git a/web/screens/Chat/ChatInput/index.tsx b/web/screens/Chat/ChatInput/index.tsx index c90a12cd2..8707e8bcd 100644 --- a/web/screens/Chat/ChatInput/index.tsx +++ b/web/screens/Chat/ChatInput/index.tsx @@ -244,16 +244,13 @@ const ChatInput: React.FC = () => {
  • { if ( - !activeThread?.assistants[0].model.settings - .vision_model || activeThread?.assistants[0].model.settings .text_model !== false ) { From 0415786e2063bbffa2c625be6ea261803b0c5dd7 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 14 Mar 2024 19:28:38 +0700 Subject: [PATCH 08/17] fix: app does not recognize GPU first launch (#2350) --- .../monitoring-extension/src/node/index.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts index 1d65704de..25f151112 100644 --- a/extensions/monitoring-extension/src/node/index.ts +++ b/extensions/monitoring-extension/src/node/index.ts @@ -2,17 +2,17 @@ import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core' import { getJanDataFolderPath, log } from '@janhq/core/node' import { mem, cpu } from 'node-os-utils' import { exec } from 'child_process' -import { writeFileSync, existsSync, readFileSync } from 'fs' +import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs' import path from 'path' +/** + * Path to the settings directory + **/ +export const SETTINGS_DIR = path.join(getJanDataFolderPath(), 'settings') /** * Path to the settings file **/ -export const GPU_INFO_FILE = path.join( - getJanDataFolderPath(), - 'settings', - 'settings.json' -) +export const GPU_INFO_FILE = path.join(SETTINGS_DIR, 'settings.json') /** * Default GPU settings @@ -136,6 +136,11 @@ export const updateNvidiaInfo = async () => { try { JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) } catch (error) { + if (!existsSync(SETTINGS_DIR)) { + mkdirSync(SETTINGS_DIR, { + recursive: true, + }) + } writeFileSync(GPU_INFO_FILE, JSON.stringify(DEFAULT_SETTINGS, null, 2)) } From f878555598563e10ecda4498b08f5cad249a01e5 Mon Sep 17 00:00:00 2001 From: Nicole Zhu Date: Thu, 14 Mar 2024 20:30:37 +0800 Subject: [PATCH 09/17] docs: trt-llm extension guides --- docs/docs/guides/inference/README.mdx | 8 ++ docs/docs/guides/inference/image.png | Bin 0 -> 27275 bytes docs/docs/guides/inference/llama-cpp.md | 11 +++ docs/docs/guides/inference/tensorrt-llm.md | 83 +++++++++++++++++++++ docs/sidebars.js | 13 ++++ 5 files changed, 115 insertions(+) create mode 100644 docs/docs/guides/inference/README.mdx create mode 100644 docs/docs/guides/inference/image.png create mode 100644 docs/docs/guides/inference/llama-cpp.md create mode 100644 docs/docs/guides/inference/tensorrt-llm.md diff --git a/docs/docs/guides/inference/README.mdx b/docs/docs/guides/inference/README.mdx new file mode 100644 index 000000000..289fd8241 --- /dev/null +++ b/docs/docs/guides/inference/README.mdx @@ -0,0 +1,8 @@ +--- +title: Extensions +slug: /guides/inference/ +--- + +import DocCardList from "@theme/DocCardList"; + + diff --git a/docs/docs/guides/inference/image.png b/docs/docs/guides/inference/image.png new file mode 100644 index 0000000000000000000000000000000000000000..5f1f7104eb85703538d73fff97bdeee0019e4d33 GIT binary patch literal 27275 zcmc$`XH-*N*EUL1P^2g#(vc!aFG{Z>y-F_zP(*r_0HF%pqEe+xM`_ZF^eQSXp-2fu z2ti1IP$eNkLV&ZmpZER7c)#Z_(oE+eS=~4X2KYebs$rx-M%I`@OL}?<_iZT<+y z2*t?9#?V7wZ&N6itbF6)TP3dWt@4Njt6N_er%ec8=1}E}c?DfrqNR0ESC1ywM&C(U zhyRQZl)I0x?=wHBS7qk3&CJQPWVSs$c^IhuLR z$+(V-$}T)lf41CB7w?=bl&zxCumrS3-xE>p5K!YYE#i|jemxJCMgh8F8P`#v z7uUMS3m=F?7<(yr%ZgQeFPTPV)`wd*V#@Q%{bMFs&PW@pWznxJfQz=uY$?gNL}ne7 zE>%xo=7mf1axOn3{t|P{8_X63*>(TbSH#2<`zLGdZNx&BoFh3h1TXkG>o;Z5fT(S@<#yA{3`o@&j$DaiCepU<7_@?Nhr5 zGOy^OZ8Gjs#db}XaVT2qF)+l`BlcVQ<2fo7D;;JMeO`XTdTFKaTSUv>}2FkP$~+NTYp7&qiU`)J9JLDpjZ4<J-kujSc z?T;-Lcg%Z*cYY68>Y_`zB|ptj72;;ew5LxtC`hl~-n+#zbd1yQD>(*l-AA+x1*I2& zXKe=>hq&xTL7mOVgo&uaIG}$Yp>yITLGn_qG=BH{(=R4+6%LOK+7=^E4m;f@zhJ3fkvF^6i60ZQZ7Q7y&w<{c z9Ytz1QJ2K(;9x4;N0;1vXT@b|)#?*Yv7$-hJLR6z*n`c!JQJ8}xK5;uKm?5H-!OVDy=75MM}YY zKbwKTv2O$%dP|M;n_>EX+fsmm-4QUQ-PT3%p1m&k+#xbLIH)Shbj*1o2LxOYcTgFd-4YU$a2( zDF$ywdev^C`N!A7t+9 z85A1Z{c~lQclcQ@OuWm>ZJC=etmEN(3)W~jn-JM`_+22R+_H)76^>KRKuT>rf^;xc zasNiK_Hto3b_9d(*^ltem6$$&h6W}$$!FQVVPfocW|x%Yn!eF~BV>a-7=`XTUF^#& z(oHLUFxGy#g2#=I=Th{|Q=zEC67I62$%kxW5!P*+cyFt9fcmqTQ1fw>}5vs46D+s80NrjnEJ@NAYsX?6y z*tbFzUwLTzKn!@K$E5c9G4)ZW+c%G|zrU`F7Axt#k@)kOeuI_)+jLtjVdg1Uht)LP zw%P5mLTjhny(DmQZ}7XI?d8MpyFmCqS>wtdq?4s8 zKO93%>UOl3sM(p+cWzZ3DzeHh`F#)Y0z(q`Y8$D{sA_7IPW1_e&5X@yw|CcPADf&D*eLs(BZFzW5 z+INY0(vIsB=!lqpE1K|*`x7O(Hc_*$!5dAgeM-1dndv~PPO5K$1%YO{aRiw^>{B<4 z0awmYK+~x|$W1q>9()#ge`~e~d7v`gs2oFTTscBr!MqsOas&|tWW)hKc`@Ap{A5aX z51!TNW_#&WGVI;gB!f-T1L3o4LgXAN#!&9Yn>0T@WDTiEiACQ*Ft#^-CbHD2i*QH# zmeNgfXHM7#B?nF^j)v!`MaA7Olr(*V|2=84gdPVSDGSH*B%`OIG6|RV+FezyWaX$) zlKU^~k9DR-X2JN{KE&Z|T?@d_dza5=L~(aaQDGFDxCymBo9+;Q1+8FXwbR4ZcY=kw zm=_@2>|(oAe4rK1Fpa+!GPpaDt-s4-S~ieT9MY`g32e`mz7Oq;%dFh)td|S<{sU~k zlJfA<6>Sj;^G?y99$QJKY5Z3lX6H&*{GSvT9oU@d1Cf(!cg+~c5@v2Y%mR)xbH1H_ zk^V^iSL6TK#N+xeatF5W|8DR}Pyfvb1|{WzPX7P6th1G{p2q@gjn5kweY!G>+^U@N zn!FbVryfbmXU;Hf6a+h5WE-WwiQDxOYM|E>t6}Bv<1Nd`l>6K~Sv!#=9I^MPM8XB8zL{R{-ZL|5br6J_Ni5ZexNUh|&y zJ@FC|7=wW%xn&zsVDr!SV9XYD%SjuICre8&bl609myQJd>8Gwx3;|NDdv)DdDApX` zW$)8bHVT!uP;gf$^09K|=Fgd~iAlmBle`df%LpV^9TrYn_eJ^G-EQts8(DgX$rD#M~GmFAuFBpzWvCf zuJ2rV-~6VHVZV=Pry&?w6zB|5lEy;6bMeaFKsuF|MutzmaDV8G# zs*$jt$&I6)qAtZC#JYcdVBTO5)Jh56jEE?!g751Z^!k*>p5}$urSHe0G(;XJYv+mU zgxUrb){}xn(RbIsOPZz4*7)*JPkri&C+idTCKowlfS3GL<%!J_b8rg-Pu~J$qriMcUF+u+;tmQyyU90w zRmrJ)Oxtr{%+1m!WMfm-$L=(h45XP8~mcx7%W))^rVpZKoaA|?26^4f|HEeImr8Mj) zJC3+wGr*knyU-@ci@PVcV6apAed z+=c&7;In=Giyqkfpgtn-Ywg4}`iW2G1 zxQ`4GpE=ljnPQQCa={M${;s0iHoNw)WVynAO_9K|VFuLBA4_TuKXt&mhZb!*XNC8* zRsH;6ZA_3uT+z8Q9fZqtB!G--jMX(iHEUw*C3rD6e3X)U%36+d-&qN@EtTf8{@}Vx z@f>8Sva$T&1bH5*VCC3q8thaRim!GAB{L65L=j*>&Qs&aamYspmh+sgb*=XX(js@; zx5~9yYO{D?!H_`W=>?TVyL`IVxBvXqWGG`I1~L9vcX>_lml1V&P{>8rtA?L3hXvPy z?YOC0K`o*)Gi_(nkv+(cm!*x8>_h}O~P^3(I(r`V^tC26AP2Xg>Qn2-VhJ=;o zP{|Oqh@)*XorBr12%T`cCWA^duzIPO6mwN`pjTwsYA?k>^Pum-Q@o8t48)^~N{~A_Kb%c=OTsBJ+d{u`ynng51&}E+#H*=@Kt#WdQC8|#&y{K$%1N) z=MMMDe&4w%=zHsEiU2idps9kajgD66%Zp_A=f*Y2+Bow36vUmx1@(_fz+O>?9KC$60w0 z%mMjEP3h?VClV7gi(sH!%5cVuS*Hfs={|g*hCR9{w1LpRZ}xkxV1YY|bO5XolZCAM zXdQG0Up-F0@fTFq_HNU)L+KQy$?E&-i}iM`*giidmj*jy))f4Tx>YvA?lToz$5|z7 zBg$vB>&!Cy!BoU8o(QVA$K#8L%Z}&7xi?}Jb)%?gaSb2Y7h&oO^&&b|&O`RgmL=w zLTle@ya=>$Z>D9ml@F=Cb5v_ATLf9p3GnjjZwS3`JiB-EV+C0uaUBbW%lYq;OYJ%W zo4zz?qFfK)27-iSkhj>VNcJRs7W`^!cE4s<`?NGFlPTLn-4xp2P=^-wZGnpP3YV|d z84#N$-xWl6iND;g`C_bKCDA9ESt4OJZK%z@%HXqbGEugC*>Z!#Zk6*x(9mUQl)RUz`Ck|2Q$QNl)SFqW?MdVlq3f z%0zmkYYohc)XsQA5 zcRn4Qva>O=mxz56gw_aO4qzq^p(-FG-SzKy;M-C~Yh3x~gIWQmV>|8Y>)kxxwIHxP zeXdW9OK~_rS`;#F$+Y1xyGl^a9Rzx>Rz|*+bdS7ym|LM^0&g2OG%`hgG;f8G_F0{eN_#&-FO@!IKVm-&*4SSVPzpPAxH4ZH6a)(s=lG79cwkvgoUj-ci7F-xB5q3iWz(xu2vOh z?6(||(Bv3AwP?JMEE32!Vxm5how}9jkZFV;%(!A)S+nr8*ZQRkf}MyJ8wAB1}C?kA|dLu;x3lH_nj(u5d``IM<0zA zisDRBh}`-5GP_kclkg+%`qj)i5o#*n=n5cPv4&X&!TYTQeTN zLL4J3tR?;!&Ie^#BI*Y?d?&y_#YM@+|3650~ zuZHME4Jz}jy|VW%bI)X%JCCh}a|?W}Ru#(2vwo&h-KQlnJ^W4>EjFYljJO%Fbmxi6z2ZK5 zJZswS=PXXBG-ObOda9FHDqduYxb^l`S1!YCvA&mlySOLMY72*xTM_Mx3WpIKCoaK;geI=-K|L!_gsvpi$SWFYx(q2h(FBRS1a(wpMJppQcNw>~V&(-Oi{&}1 z3|7mPdli{q6&Cwk!dXn{{bi={`eF1K1OLr+hnseh6WswjANYRD9n6_hFkj!cK{VkwjP#^>h{7esZ`n2WMy z(Si<_Gzz5l3wSa^EiY?JFF=-H_h(zq;GlvPG6zfV>)TcwuSS$+i*|EMcG?tk2&4@) zHBs*O8n^fZ~-s;I^I$0*5u_VNVrRJ9TR@Z+|HCi)V7NaF%U$ic?q{`(_F=K4fMQk^?=@_e&OeONbou zo%iulqcDd?D9JtYzx1W9jt7aWj+wax4;A>U7so-+CaWKjD~mA~<_ofmF{9txbtwFV z<@Z}_gRdk(n)oI^$;^JeZxK5gB;M2DH5(1CwCb#p**0dgbEG@; z@n!9)f0XxJ*w$XLZqX~kof5l!;pbn>O9i$v!~1wa9HQ7OMJz9r(@on!A6%zzB>wT7Z3HFS89A@8(OdRS= zY#sI4e&u6n(zGxLAAHLZ{StgyB5Jm`*l{PGVD1P3T0e*lgwcXTB%2Aw_LPWANs z7J$H6UNaVCENI3l#bVf<|GQ;~mHb|c<*L7MAX!QmtNR1=iz@&pR{P9Pas1fx_Q7VB zx)T%>*P06eZ*;CeqXWLm8c2mS+?9o$ONYF4f!|h!XJq_~y=DDNr~OYPPAUT6xWr~{ z!wT+?=BNnL0l?qX^9J+<4flUW>B?C|-W2vr=y^=9=lMEtKv1azK3%(3k-U0am+Xz< zSpd`33ou61Q+~^kjm#qb(>GXlmz)3u9mVwj2{+SC8-xOwpyDIRPJoum*pFZJ_6@UF zC^vqeV(l0K_McEWx>g{$1gyRT!F?wn`_i%pANt+1-4q&IN)_mp|9lO6XPNsp0yn9{ zEjDyn44~;a%$4RHuRKhVUw!+mKjzYaohJaHnBUCvYSR0c>tO)}rh;pl;2T+PrCQO^&&wFF4?P4k$O=zOQj1` z`LOf(NZP978QL)(sAKP=1?n$w4Wc;5%TX!xhtdbH0gTlB;q}p%Eo(sD`to3{S3DN2+pOm87A-vAjgF8=WM&u?0qVm z#j#V$YjyVHvzWePlk-CpO;>Dmm#z+OC6=Ll2*we%(})(-Vym}jAr1}*=!!c}+L>Br zuL~1JSF&KHll2mHu;V! zO(Lp7zOye&?nuj^OArJZ260!P7PyrrA$by;kQY5K>!mrxLNHu-L9l+}ujO6HgDvZf z@aZM2U!PO(QU3+RY}^$4$?63#atN}N>vJzdM7-SZIWk^@5Z+CU?5CTkLTn-!Ph>PK z3c@@1G9R>R_0lQbqXutn)dgYe#1UR^qDVtgh9{8G1nP;ePxrq;23efsKR%+SkA zzKH;XsVP=*`d8=-Cef#SNfT0}=>VBP^9uj53NgIWcaaRf5jK-lo$Y!Q^(U4)!#VaZ z5NZKl%)&@zw>!5#r*r>|1Q;^lVXg}9TF3BkAMZK>z*||Y$aHIL_-{F>`k;OFyDAsd z1;OW!Is~XD>6Ccj-V*N_Pzuz1K5ncgYpJf59PN6v?$9==XJ`MCoc-O*P%fAEWIPg- zZ%M0RSd}5gFMFjaNHoFF?%uvvtcU6s@%->dy%uc~x{z-GjJ+^1)wzyw;TlbVq-}Kg zM9LJ}aAUvNale0)|D?%CNvlxXw%X^r5H(5z%%1!YB!JQhgvi|BJh4-Hres=(g)|z? z%Z6LGja7#}DDoR*bp_^0UKd^2ypDmvh!!yx1lnujkx zeX2;C5V0tC9zJ<^N7a?*w)3#k;mfa|6<1zSLK`(i>+K*L`?fEZiBR&StIBE(>1Q@6 z$`bN6lDlry)I>0v5JQ@>6XB2qIKG{X%MD%>@NUbLSOYH!zib&o#9ERZJz8+F(UR7Z z8N%T{rnT*K%|z`qkT7!U&9rG+Q+Nw!8MS<11 z5aRc~LPD~hL>0fQ(~wVXO8EOO{Aq7RVW>y0>X2(=-Kf%3-aaJ&b=xY)e$GQC@=pzS z2OKD9R%dkqSecr)=d2)BL4wvPFX+aj81L&YNrcqw($wxG!a2YZc$|2Ue{gjudheJT z4EeP%r%k(w4V=ql`U679PT)OMSHfcUg%U<3eDc$pU$A!8OsG9RcyQXJDsO7;ul4g5 z^@8&8X9tgJhtN9DI|3m|>hb!r;+Nt`X}8+HH^&>SH7CBhDl48uE$oEN-T&O)&sn{6 zn-L4~Q(qfjF&#_T@=ZPG0=rj^Q*&$%GtsZ3+&O0KL9-J$S`PJUjTdK&o%^=SwQb9a z9@dG9gfe(|PgQ89YsnooSw!G;V1Nakausf49OdyUC))&i3{VA1HP^lv4cE_WAF$S% zKh6@e21m8LMFRp7@OQul6QpFO1k%Y?Wafr{Z*R>uSJ+PKASF5C=^IX2(V{MbQP-~p zxyJ>`fkjbggzr$7mM3$W*{R~&8%UroaQGhMj6M9pWU_PI6zP>P#2uyFvS}5?j=xG{ z1&LR$%k{CCwnQv{HlD5~psJ?lGPU$PlY4va2s0RJXn|BYfCs>b*<~GueRqE)QZN+b?Tl; z`g|%S0(rhuUmZTZH-+!1QsRb&n=-B@KAnc5uJcyomh=VY7U0Er>Q71SO5CwJk!mu+ zdQ(vp&=_hMSb=~w3jS3FsYBydy%>#t0!T#AU7s%W6R^fl^ZrAeu*Oi7#)B~g=|y*H zuSbW#u`~NWTW1PslhA7reKwuNzN!y}OA)cW^LS(j!;nH1!>g7Q9YMH z_*o`IVQy^Ko`C#nVfPa6;-{Zr1sZr@=aNM&v~=67g3ZE$@@~bObfX$sL!k4!gjtw!bB5|NdN(tgqo+p@FnX zv@T4td|(QrwirlSHGh;J( zaQ1rwwGg7TJZI%9l%Z51rjjH3R>z^097@|0geI+J%eH9| zM*IHT^=5E zX>vqd(?GY7eMyUtzL^?kQMJOEIha317A3}0z{8W^BxH1F)7;cGSR-V2A zG4NcDYGGe8Vh8MESwKZX>2W>zDSY2jtOkhFS8p$V zK&F{I&T4lrAHfEwP5A*VUUuJY<;vfzJxfai!Eh0ljEKFNaWF0~@)6R>C+?Gk9J;tO zr4bo86$8D(LGGS^duodve;ekssiS;1CE|5x*&@+uYNlzBs)9hy{6rhIBK7O1&v}m` zfzG5kE`TMV_!1T5e8vrw!2VYL*y7~tx>$Y*!tDoovOC48SJhbyIY~+r$|fwj@AAG{ zi=*$9V;n}~R-U5Qo_@k6I*Bk!zSoXv2!fimgj-2;r0_EouTfQS!e;|ojU4{k)4`>$ zsIDm$`Y@8)@~CYaJ9H9^`tGD)ru7w8P&J{KJ|g z8IIRzo|ZB-JB+?j?c5P*mQ5+^&Abn;k?MEnqKe7W(4MM7E&^=7V{(#s-*XdBx25)n z>RHs6A?Fqp|Ij2K8>m}Cg30YnH%sR9TLc1XY8w=0iz@g~wcmG@VUj}{?{ox`w3Nhp znNnS;7$!fx-U^pA6R~XxzmeaXmkojO3CizY21BY<+P9`G`BhotyETS zsQXd-S4aB7XWyk#M3Ywj1!Q8Wqv)lxsZAi;cWQktIhL?IajK!<{#!-p;L4Y_jdcT_ zqBP$k4qS_wyVy9skhYTKOT`#2+oNf>ztYKp7qcLK zM1EY%+o2$L1Uo+ChS~2e_QO*=xDE<+Qqjle9ek|Y;lm~EPv6gVDyKKUz`)-8VQRKy zID6WRksr45G^?eHu;qm}Hst_3-*<|4Om*#{SN6ExafD%dI41JpAl445wVA zS$a>R2#y4XkAA=#fzm>$xjZ=kPLbh??dkNn-fV>vK(;o2c%0R*I3+zOom1GvzMZ?T zgCLcIcK7li4wg8+Y~O@V3{CmNGsZJz6Fqj9!T>Vymm2`Vz#!FZX2gyw>prkUq&&nk zP@*u&$~qvzRj^C5HSS6YoEyk}J(uVnQ*rpO&~b>$)1aA!nY}>#Ei9 zld7X_M2<%k#aoL0>YlHW(~5)L6x1}cv3KZiIrSD?wk`NnZ_|55mB8`!s|I*HU%{P& zvx6pGqH92~^rk)xR!3jFC(m`y+98=4? z!?BX2?$_@qqwsMbbRD^_UH4kAeUh=^(OYcUi>hrV@V&B;qqsH&^Q;lUodV@i(ct8m z?=Lu}SHWSKINJv7Qj0uLa!oH<=oR$Gg0D9XbzZT2@I`T+T*qyr;)IOtc!W!SI9m{ZW(skci-lS!%xnfK#Zsp;&z?ZD$3HHDN~XZCi#K50n~6@2cn zI<1R5SwsS3Xoo2?_0NVOPnMhOsOTW3KSERKr*!RxWF+aREFqD8Kl$fT#rh6Sg>X_b zzV!7QWH4Wh@jBBpC99<1!b$0&PQQj^$WZ6|!Jru5ECa%`&~?oYXyfu@)#`2CA|tz1 zn#&)=hr)%vP`SecRP1^ixM~;(Y1HOa9JN_PVOc}rIZ>PIas`{uksJ~kl-L}n#@wPB zTEa*Hv7j(9HTbdb>v`4D`gHbB>JFA0+1YbJM}qf4|7<-3#4jhGk1g6o!x*v494Z-#x&v zNWuAq)XVjn)iK|Nl6jvuoC)4-$d4odZvk#%y27ZakeGMofADf3jm1GX_CnUoS}x&u zX4V*hj$b_sJMTG0_@0*>pYBK6M*>3{bnVqGH?3Er+9 zI6Xqxi$-C}ZkAr$TKdypak%x`=l?|A620y9AR_$tw?jaL&2LnC*ao;Mt@Q4HX|HA+ zX3y7(&N3V>8s9yK{U20Xy8Z53f&X4CZT}y+*Z)V#uJY3jKGjZaS+;{{VI;Bg*$8cv zxPwD)_c3t87Lj5X)5*squ&7zrJZWsQ2`&I9VToqxU*V}{S!mNmq2a9y!Em(oPb^ZNX;$k5PG0xZvvR$zPnY|9gl^nGW3TFS2mXS5hRX; z0LJMQTYJ#}Dfex{ln|J4(*Ke`jExE;kZsonHmqw>=@E_U|CehmJ@{ zSpQmS`*n{!JJ0{+RKS;JPMyyNpw9p`{MF?PSEoNEMunaz#RE#D`eOhqvk{Hl>v%>? zWc}~BHIABo!yQ_D>?nu)P#6s5xtM?4ACz_4ftvZ!T1l7s@kpl@LHy&vH(zKevl4IZ!7fpe3R@KBz@S`s2pr zDt_mB*4^vlWKVHr)BDfp%K*KK$%HnK11Mjs|~D$hk{T^CFHeBzV0CNxm7#LFdiKDW7LOE_zq-s~%V&Sv zD&{!IMVMp7!V7Ex<)@;bCAD3X0A9ivwa_OB{k!N}l+{uGFM+lFh$~i}9J0oa1jKSg zleO6CA+9h{Q0qz>P4=Ja)};X0D&RQ7HL#L1Rm~)Y2(TusZzZsQ+5_tN!_bQ}6?5v< zLawMIWK@)*NG+}L+84Lz0XMmR$#EB<<(Jdy)dqb27=9no6}cq^g&Vyostr<6c6_+6 zdB&sR*1^yR#HUxH%2}^|&a!9f-WRu~?D^%l5$E zcYy=STeB=+ObJI~tK*MW7Z9%LYv($Q+g&N>i<(vAPB^3++nehO#ym%CXAQ}&h0W&e zugL*`&5dm?9Pfl3S_)!`D6cIa8#tpe-da0?5->=$SILS-H+qfAg#R3-o;Nb}hsbYN0QC zLH!|1WZ#=0MNvf?VlE&2r;0oUBX&2RtA=x)rSDkl0EoD| zN!D`W{rAM`o4>ZMm2jWBZ3(My1Q2beaz1c&+qEyex{yfbo&(XGT`r)$K0b1x)XMje zze<7h1x^f9?%(Z$9RGfkz!|*M*bb;h;;!0LO&v0iG#bYEciyw0Wuzv{GEvN7;QV82 ze8>!H^|IBUd!(@6jhpfm<=&|Qc4L32$!zAth|(0WDP_~|j-nz!+^fSWTY5ARos6)j z_Nfy9RdDtbtEUDlo4U=gGo!TM{n8&0XqI`yWLN2bk(?e9=f?rqeA<)<^gBKyn7f;4 zEnDMSXd;E!>)LlPmg+p}-cZ5(;{%+qa_7qbks6$_7$hu9KkHfudC`cUwP(mYuwhW0 zn7+I|67-z2Be(zX8tM|x?R7*}_cJvW(x2~d{=(;nj(34crmhd5-LS7oY%&vBQY-UsvcOJedyXLx;x_ z0|Cxq7Dw4t?7rPnrjBTg;E`+ZtxsoBtO z`V%rp?%eK0u-Rg84+sw!yE<3`BX zp;;+$u7XdM*4?QH8pLYCH$7VF-1)tVqR7x+zpLazP05*VC$sqI)1;OMcdv13Zfh*R zT-9*O^nD|5(}Z{y)>vpUN{Iek09DW#z;#SS_}Fj-ynmsl@cSR8xUsJIriWzekx^Ub zMQm2dG%@qsm6EvThKPG_kigox*gFyj;RIA^?kDKzdq6ah0O=#=Uj=kpEZsKuV)yiE zBmMS6LN_YSAih2vErPCyg``%fWoaoSpksHxlfP@qtG9P5g0@k^_=cUs7$2cd3`$d# zlfmy4Q5W2gDlTWDbb=9l^TfFDZ`B8eTK>rj>ijM4PEk%4o&>x{N+a)W6B_9YN`3he<8K1Y#E^{=9Dvm7yJ|fU9p`re zI?Rq!t4BBdpb?7kVuY4;O*Aj|v4w*_qjN7jmOS&a(sO2J-V1{3DUSMecA#nWfHci#+ZTlnz&@~( z_%*ZnjhjO0C00Ns9uzueANEu44u5?mW0hvt7R4e>`xZ7a_L3Oy+js#JJ|!MP|KG}W zTEZYJgePh{ret_Ji2h;0v#xbsqx^U4HuYP7Q@{~po(|~_{h>{C0i-(OVydZz6)qQA zfdBN3T`zy-JrW-9UD^Wg!g_k51i&po1FylW@TVTg9|OYTuYjT#+ecLfRktCCx(c4+7mLb1$V?VK=S zZJ=2{HMbkS1r6TWhn&t|h2?H`o|mQ@+leJL?p9f=bxCCbPpF0}q(3~5?ZYSE7<2l# zw+4j#q&uqA%=f92KL^qTVQ;krJ!;6a2hJUZCOJvW-Vt7_(&3J1=CBu_z*xPGmU;3+ zdUfR{jMr#qlw)On68kkNOY^G7O{bt2Q)9NW?i)L`cGpwlz=#XJmi?C{B8Rsf<)ug* zLs3TGUKIBugWBsVH2CqP#6HXSH(&T;PlTa#B{1a=-gK8*D$pCli+fnMT1CXT;bcry z@>F~l^fl}l9aXXDTZ>9WgM&O{c&3ZrydK(Xl~57nAcr9p6C7Q!(CHqbJ-CQyYf%3JK2_w>=1u|bCW|DeLf*1M&vPlY-BS~aS(j= zhK86|bzyhZ+r*^1#A6PBoyaFwf3Fm_ea7oI8zu|V8{gd1^=&rwx4IpzaepIyyqoNc zv)@<2uWQ9xXhW@0dZh&$lbE-icNn~Ef`hu{=gle~9bH?$URyKiG5S4@vhU+QA7))$ zPfJ%)!*lL=$v2dlcti$`m5rRZmgRxyBP^9g4Jj_Zbviv!(Z309&}gCzr43xf3vv6vk>5jA@p9Cx!lBU-VYzI zncc#~f6J@_IFaslbe8=TdVa*JZCUCE(frL6{vY+Xv%ymuWYgYG_3LhC2uBvTBeOPq z2-PTjcDWprw~~>_FZ9kS$>}wKuEp}He~faY^4WEm^5nIP;p=Hm*0ysixZ_v$EGg@| zM*=f-c+4SienbNQ(K$3|-uR^Akd0cJsNTZ%%_SR@<=|c!e?LH-ikg)o;JrjfENLfG zHJWFaR7jYjkmu5{6?FUfG|!*0B1(8M2J8uMng}-3QPUT;p5lpbiPbrK-^TY2=2I&U(N>6$c-kEC ziQYYarT$kHH_mh83KOt`?ee_(O{TMua4J)0*KBvookH+w&!DXHVvPB(0Rb*G?(1~9Cl$Bpv_x8%hr=B_}&Okpi-N-yl}t@l||OWS}SL|YYoiQ4w6k)^Wg zKa>tFk~sFD=P@ba74fgqyJRZmaRzf#RiwCET*5J1# zDovezFm+pAsL6YTW!>7k-@E;H9D1H-oxApDx;G%}6dJnBd7+J%7w(ylj5?i*>anvu z0Zuz-LCwxKKPXOeV>G#mKRvb|I^4M5L?<+{l6gdxTXJc!qa;OD(Xyd=YlGZlG{xs0 z7zb0^ei7y6(i~oa7n@?GE>?HI2`Pqe+avYsdu}f7Tu-V`e9IKyT+ZLp(6rrsud>JT!Rl4P>JtiM#vS#^{BEmO$U=S!i%VXQB{c+?YD1B z4+-enk!@c7I!5VP+hXO>$UkxS3)6iJ-|itAsDTl2cD?QkyvWvH9@Q2(uK_bRb}pl6 z#ZpTvLv59@Md8KQOmE4%rE;gU7N=cqr6s7_VT$x#fqEXtU5#~}Pi^&3z?2#?ssEh& zkhl3k2PFfM`z7EHWkYBPIoHtErf2vIpSh7IEA3M5vu<)5d^K7~z)UD^*ZkU}&G7%r zJ)7=wXEsScW5^JVOK372Fg(|v{gMY{QT=!2@D257mvrL##SxXz6kS*4SwTmMaslMv zTIq-I-_J7w)VW7Gn;98_GW2avOgbJ9em%Nt4|bo`<9 z!rjeDF>9#x8GB%W+2jau`avATy5eaL*yaA5k%}toXQhQ; z`FEvUrQxmSvKPLCR*ZZ;fn?k~S=((xs@8Bhv7_+nZRiM5dV%H+J8pl>c01duBKAs>&=bqtuANt$7yiqnoor);n5K_2; zW$>n)zMX=pRhwK2>`G~h`g?R!)Fg!_t#P&B?e~HxW@_{Y*SG9HX?Ty5DOI*jYxjRm z?h(!BOv?%4MR8zv9Xb5*BWKR}^Mwiy>wqn4|dMLeO;iTz2MbAMNsC`QdNw^TUb@IJ5M(+bRF>4(fV> zha~6O{>Om%5&(cPaq&m~{Zb@$AwpY4rJIPWJ?_h`kp>3ula}``2D05& zo6bP@1E8F^IQ-6E%X-b2eY17ZM`W3_F2{0l;EOPF{84Kkz|FQ{St^~Xz_D&Rv=V?a z{$JIdXH=70*RCx{Q7KB3jv^{jl_p445$P&Y1R)3;F`|GVLNJ83#VrxA06|(%O11(< zdMB_ow1AXI5rjk_U_bJ{+vJOJLkXVDOp)-&U@bXh4`VY4?i9P zC%-pmbDdVtDFdggBy+l3l>(mPUGu*(w0-(5^y8-M9TN8Ugs#aI4>u#0}S5UUfzkE0=o2 zD_m*D@qFceHw)A?rY3B)eGY{FPB$jg*Rs}6dv*tow>7+PKc0s1L3F8B*=*{;lFf3K zjOkDBGYH~^Xx59t%iT(?%mrmDH&K>Fkah3UQgT#-Yt+^<8ps1Cf}}Kcn;54eMJl9W z020R|F~~EQkt6dgJqJPK*ZX3;`h6lX5b1LtysiRx?SZGXd*$7Fv}^N^dLE5CZu=Ae z0T#H1-vlm4Cot`bVEQ5m*EF43qsE~@wnxd13#7PIN^x+v?vNOy9zwza2D`r+=y>A5 zefa%ya~M(FMGUkzj@EokAVi!6a^LXVH3o)YR3Y?2^FCReyvS*l3A9}>_qxzqx6~?w zTR1cg6az+lbdAT965*-Q&DY>kKKI9gBhs<>oU{ zzEo)d5(OzBJwgUk&sowF#>~za3J(NpfFjW8z)S9Cv@e_xeenc2e)3E*>odFJp9c>L zYW~xi(zU0pY$k4tC@FO#HM36#-!> ze6xc75ujZw>Sc!kCxl>$pDN^2%&^qeI^v~NQwN2KAH|OD(`?WxFm@2eE^YAl*U%Fd zP}~g)ZWXLqhyD%zH2x9Suvq_CSd))e7Bv`refPdGu3MN)uagozyW%Qlz=4&Ro4V;O zU8oGZv~M(NRWaHZximqf8&B>>%B$yPiA;q%XxYe5k&r|6e>$ah%JC}Y05eWh!p2xJUE^kl(km<&ak51 z5sQykhK^W$Oszh5+FsSw#MpdqjA+TV3j1mPBm7;47D3mthx|ZCFBbl_5?S`lImY&P z8A-ziKR{*gR#-EOP+r?Vt^t6t*{cnL9Vwas}%Jjpinuw=ais& zInp88&FNk$e&SJVAR}&vPFsfXHs7>woMuc|YR83m)>WZnLtwTDkORk#N!d z2h=Q0FI7AWI~(@TB#5(1&+eAmjTt*Yes+vuEV*6*p@&>V%*s#$v{NEq4+k#arGXTU zUFTf7o?E5>^?E?lI8Osl)8QtPyKYEUyCQ7ivS4$_gVo%IgX#Ty-9tf=92Z(>!|U7N zG_=#|Wnmpyr2?ll8d?2fVh6-P2brcW#PTU!8S%&XOyV`n=EU@8M4B&9&ozm3(Y7v_ z2c4(1XNQf~x|aQ6)u~yh@<560Feu0q(|C~tA1nKJALPD4)I`1ToScdqycK)Z&8Y)@ zJ0_7+7)FTem$807b!N ztdUz)#DPV-l)AJk`#uI{HIw$zWP#~yvdbk_hJCN!?-bj%iX56NWUfxY%hYAhT&n3S z3Alc$BII?Zgl+8i+k2D`C7Kzs(_U)!QFIm#p7I2mMA2qp#mQaQ*jKkZ5s{e-f6_r1iy5cVjm%CT{L}WuIWp>VhZ1E}YW~&|ciuRcOg@M(9_@h>i<`kB}7&w$jrRF?x z182+55?iolj}`IK4Sj@Evv-dFD#@u!mH0^YJyHXH@P|qP?LYdB@g*9f3xL`Kf}TD} z28FWzJ=IOiB`j-L$DN393VKJ`NyfI78gG!Mf7Q%|m{)672e;^UzagN8_){_l$>CSI zpgQ}+g!N{i|B}?3fsFNmv0Qy=w+sYQjFvE;hI_K~u-|0}<^)c|e9c%#cG|(eg5!kw z{B~K&&yl;wSO#ugZ$lGEI38iOI2(eI{okV|CHEcoF^^E+5)MboyN~@r7Kl9(tK_yA zUNZitXp6YV;oDnEgiA`14s)LcO^hVdmJ??%2fOa8WH?*)iy}U$F2J|ubQw?XbygJ; ze5Vf2^(0=0?4Aa++}pINM};K!vr6GpA0pkZgs@I2oBdJW=F2gw6Hv6$? zJdD!`?r4(EW&9%FF>&ODfAf`{;_JyyTm>u6B9c6lqL|>UD)fa#QZoBF9l$;r`?%)7 zW5t&?f3qGPU7Ig-nA;Rmh9&C}=c|*L#391csL8Fhfwk(p+ z-z7JgY#P+eo(?NxUOLW*-=V^YAH3)$?_fyJ4}L#>MA0X5n3spDrp@6uTYinr*S5dm zJr)coD7l+@VGzQo2d;~+|CDpOr9lC;_G6&GyWGLDPL&{cqs*^S;4<8aZ|5nJkHI=a z$aHg&Wqjli7o_>0gZXhrLSc7~ZV0WDVaIcfl-8K;Ibh;e&wrBt&fRZd9yx0dRuibr z`oH@9cl3{j!_eG1@#cRwf&L%kg#6!wnb?AmJ17#ZUUmPsDUExB#lQk&Urm7^V|PKr zRjE?NMwg&H#y2sv`djt$X)p=GR`Lc!qT8-E-+6#tOrEX(M3PT|@rE(>d1LyqsWl#Z zh7m~*&0od9R<iSn`4=i;@7w=n`bysQ)=&$`Fkd&oW^S zFOVEE;CbDi*jO@J(}<9+>z@V@U9LW1pFA_RqUq^~>bDK!wkd40Us}BqlXILuWI5Yr z>P1p3PdFF58cZ}la2x{}W;f*8Cj||?(YxggQBci3EpO!m5V#Uqep{FQ4MKmkvfxTW zw)&|DQUE)zc6*F`<33TG2#^rJ88Xm+=)+)^@lDh5-e0m1aHLwA`lAoS+_6WgAUXmv zVIBs~fkh?bmjUyj&u=qdR_4LD)dGEk#CGurbt(lySQ_|u5Y;8gcx9N84VL5D9Er+; z;R!{6?-Ew%+~Ymc%q7=OnP`dZ6WUMT>eX7zp)-1J?QcSPKQTt%qp;q4bB-~|$g?lN zjr*0H_+rdyZW6X9tTrx)Lsj1ThZ7LrWI`LeZo3|EMuWQm%2uL281`;IaOoWWr;4`! zAS>qP#dSliuB4kIi#G?GE6UN{4^8$X^&Sn*_9zz69^}6filKMal1O1&^AO<+>LUv7 zdqRG8ot(2XSs@9~?7QaYTXX^lhY&VFV&|PcK(b_Oi~RK^N4qgyJIw3Qq2!wZdO7h1 z+WlH*c@1G(zb@(h_C<4NUeu5|V3Aqx-a6Nk_eO)T$(Vrr1z2Y1hI8^?-t2U~ z^X;ZR3)6l^Sd+AUt&<^xf?YG90h3`j1Q@5s%ak+d-tsfpC{*16@HloJkz>Jz!`6o3 zwW!u-Z`USYDif8^yu12lxUQS`zwINURXt@d{YZFI&n!Cc^qaU|eK>Ayy$=C4md1b`Wwvbj*KA)!2ZB8*_@|v?iuLSfeGoc?mR;U@CM=`iFotY znb-;8$e-tF^-eWARc|1dh$=x}h)68>5i(x2AmjjY4%#N{u0$fwe|UHbfr zI8mUvzbdIpx@@rj>r930`~R#4?OS?tSFZ8|@f3i|dqcfP{@Dd#Fi&J%rzUMYba%kl zPG{%{$0OyRwax4Y&S#g-!1z4z6xvu6cMIb8y~yGH2zI+viqONI?e5ml1eg`s6^3tI zISFyCZpJJD2gWa~YwA(>-q}$q>zgUJHk{glOGqV}JwGE7-FjK_xsXBxIXHu_knE?k zV1R=G>G0}hsc6mEH>{3mh>=CE$dXW?gfo>`)-Dt4%ZpB)-TIQr+at2 zRHnIVPnBf*q~g>A?}J0u@jSOGD{beDSFV9p#eFH$*#5NuwG8QWy(`OBJAF!j#LJ}l z5I@$hfd%$eOIKqK7abDhD*jA}Hy@EJeV*t2O6E7?Cbyk??PdmMD`Le1B$FayV zS%7A8v{0zd5)!s{5kx2&u9>*mDxI*yI=CC3iTwHI33+kxQ`99;Ni1HHmPbtWO@Kh3 z++Hs}+>CDgKs~>$6A%Ja+alul7G{zFJrDTgzZ0>m+B8j1ND?VX-pIO`Yk+Kbw^KSH zr`K(ES(@^g-yX)yaKZf;z>hZ!BSYf5`}2~(xkL5_9k$e7TbJTPCL*~XRQUUM$4t0l zbvu>v!au;N=9}MBVcD87^rbchSd&aceiZQBqyYm||FynVbzD(wUrY1@S4B4=L~cf* z6#wP*dw)?YMy>*aOmYF04z_wt`6%Q~gRw;kGQabX*!rs-HHYa5svj;zt>!p)1?(Px zI_ysY854N-Ef&tyj_C_TZ<&6qZlX-?Pv<3IKSEUcy|I3o&(YrlVlFX1sT)Qjb}c8c zxabE1Is*WEl|GPmm8~C8*0BlXp+q!bZq@X=)!jLi{)zf*a7Tsnbj;D6O5kck0SXBY z&mk8FekEOXn$!%#?5#*g{SLuqe)X#YS6Cvqwa~pjzOK%=2;P^4h#^aElDP_S>k~p0 zHamK0>u~a0-ISr2W^HcP5`+0>DfrBTd8LHy({`MfS*94WsJp9YfFhja0s}mmxCwUV z!pVTOp(6CZtC2yo9p9@<3E*qW|H({qjQw(3N^a{kN@f&NAY1N8Ualk4tj;(ZtD7x<~_fFurjoxPoxV>(~5r4v4S3vzs02(CDHLx+1FMc@zq{{H=`XUXd z?`ny3wVKV{E;5I1bI1j=yb%K#b6?gCNw&~0$~2Y@`w@V%Nwbc6_TA%$K&IMu3Chi) zsVPl4nuN{ZI`}*KmPzf81Zqli%6(umti*#BYfAr^u7{Dh8Tiy}Fdu~S!H(GMBjAI- z9{NA^8UDY=2Xq4^Kz%>Nk4?co`vvVny@7{>fg5|Z->>1}+kwrm(g+}tx}8#Kr5Ax?KY4vtL>D@L~FRu*@spCRszNn7_g-HU6 z7L5ih$p(f2CGexm_Pbq(B?JCd<1iS?_b<)|M%9M{WMlsf@Fs?pf}1K_>bO8UJU3l{ z0PE<>awv%oRg#eT{RBmv_`*{DdJ_1UcM8i}f>%QI9Ictr$vCRd z5Ir+@nJ1?|AzKVj$uX4^*BP@_d0?S4(0)}hG z%zT-DKqUHFSEjyZsKqcydz0QdBY;{6gXqI>>E7xzcu~v&a~AL+W(G950bM;qGdyaE zC^^Kdw_UD&5;_aewZI1F9&zvBV73tfs8k667Hv>~MG^C8rj%EDjJ3s2!kQppSPIyy z3R^!d(Vzv$s{wlQw$x0!?$Puzkmu>$-CF9O%6v)vtOKsGp`Rhdi0AnRiZ@!l-(T^z z@&bfEWV|v54UC=jBqS}D4X5301F|swUxFfdRs?!vGTGEKDKvQp1C(>Bz=BEm8pZ-! zqm}u7{(!ntE@uNbq>66BfBsG#?8TQ-ST<+inkkYYsc9pbhMt^MdmB1#hm%p zm78P#)Q>NJV7cA9?w13VZ+kypml%DyBXvHN>j_TV5BNC8UF4sB){2XOMXsEfyl3am zo#%M;yR_Zf_!T2UrLxvTzEug4&=n#L!BU5K3}Y(DyDaON5*E`kCBYS~PDEB{i@Rgi z!x1=`_A;I>kB3htk{sgm3SJEch2JtJ@^`{9U+da`2aEE&sr zu=h6f<6-3>F5h~up3j3*6Ap~p7$vJhJWJMw|A*Q_fWkHpTrP)Pc#V)eO9wKMt~q57 zu7dX>)<`4npli^_Wa=NqkQ$BgDGGucsJVX3mGxJM5vsAZ%|$>Hk9{YK7_=XdtZAS( zeC{tO@7f5M$EeTE1%4v7E)W@j<{C0kd5oMP?QtOZZdBdQ=;jJ(D(&s)`GCH+7IfVz z)zQa@s?OuFSKJDjp*J@cGl%v1GN59j&^LgvwL&+(EugIUAz`_l$Q}&cf+XD8IdE{9 z>@v_8ND@o2)pHx2sKxQt^-nW;ypPfHlcmxI#{c~WaNcWuktjwx&p0$60;o;o4h_i?|Eg}~3R!lwr z?$GV|BvFy7D3*SK1vst-<~C&B&Q4*7MUfXQ_hwvPjr^{4Ue&h<^>&^byu2-$>s++C zvwSh*A+a0Q_=|6JAJ3gB#|nwTr)R&-X#|GFH76^kO2FlnbtOt8=h$to?qTX6F96#fxpohxE*KudXJd9u&iV1&{z^QdrA zfB$+Q1Rh|GsIV2C;|d|HYUALt?p?l;Kpx{Wlh3r{`}t!BI__BTm>=Ki=Ar1YSOxm7 z!nBK=zs})hfkv*fIC+wwla^N6v~rqF&P$*&A%li` z__~L%etE6zIOIK$QW|dE)cBX8cpcwS2e)~dk2(bjS*qyNxWFpP&Bgd6Uy`RS9@G;3 z7c$DWE2~-4|BtGPX9IEMRND66GHChV2gijEf+B`OGsU@EnRqoYwLSWd&IY-O@H{CZ zhJ*54%G;-B-*Ws@4|ij8LIyVe@yu<4OA3T%0VlZ=SW6M(LwYC2xj8OknrhdtY%(1@ zFMT?ntfBdE?B>JLp@5K16{zw`F6_?G4e|Z{hOi|ximPA7G_Flh`2?BtWW0;){dPm5 z1;!YLoF9!+0}Ny8a)+O%21O7Ae!3eqC8C&5aU=(jY5Dp`Sz0i!rG!8({7d?uw`7*#5(2_Pa^PZ+B9wfb|OTT!*-EN{&GbUta>!MObG zX5lzRt~6k26YBz=qp8xd=dlcDsBEGG2a{jh`ZaN{zDFg1Gf;MusrY=Oy=6{J+s{^& z)24|LCva9V3BH;i)K*6B1>9eJr--5R-I(AY&u=s)Z;2ALhI0GuerZu$Sns| zR0zdpH(6bippqb()hAIJrtjJe@v^Q3yH~Ckt&N>0p+`|n@6sP^GJ0O3 zOQQ<8+*KieHbA?6jZRQkmzPbaIO8*~|0t^;kggLecQXl-za`P`e5iel>E6{3 z9!7ORh)&D2t!X*F87Dii4zE@NIac}d$aG@))gVOsPJdixt{&Dj^pZC8d!JQ|VVo}M$O{?O>W4oE2e)DvqHri7@ zLtnj-OY03y@c#fi&AP5ytW7ZGzMMfja_1&0goZY~;Y%N?OHy}zbt%>@38zFM-q+05 zDHD4*GyP&}3*|%KV+60~SH#e^u-rB_vAn0`1m|~(1z6Hsh!gXxToht&mjENm=(%2h zEWiD0?qP*ThTWzNT}FIU(~D%aOp2%)muMDk+XPhgL??>JAa`m;LL9r*HglCkcYm1U zh@5B-wQQ(pTR2aDgt<(g;A>Chl+8T7#qhTRSNTK-22&0^YO1VC34?M8|Y4#&%sss7pS{TKiC5Q{f($ZX`xuN zRPHVRjni*JeTwsQ7}_V#5C=-r#reyDlUCw&fFRBfYV4okbT1k00CkrDg^m9$(C`f_0; z^3Q9U2QD+()$1gKEF*JVECurm$PH|Y1M(Q{yvKOfw~3w(@J2)Lj1*tjb}2N)>UeF) zRcRiAtxSC8bA*oZl=|&4-WKV9fb%bSMT~1|-tKV>PySN{`~j&QRu=XbKAiK4{VySR Be Extensions +2. Click install next to the TensorRT-LLM Extension +3. Check that files are correctly downloaded + +```sh +ls ~\jan\extensions\@janhq\tensorrt-llm-extension\dist\bin +# Your Extension Folder should now include `nitro.exe`, among other artifacts needed to run TRT-LLM +``` + +## Download a Compatible Model +TensorRT-LLM can only run models in `TensorRT` format. These models, aka "TensorRT Engines", are prebuilt specifically for each target OS+GPU architecture. + +We offer a handful of precompiled models for Ampere and Ada cards that you can immediately download and play with: + +1. Restart the application and go to the Hub +2. Look for models with the `TensorRT-LLM` label in the recommended models list. Click download. This step might take some time. 🙏 + +![image](https://hackmd.io/_uploads/rJewrEgRp.png) + +3. Click use and start chatting! +4. You may need to allow Nitro in your network + +![alt text](image.png) + +:::info +Due to our limited resources, we only prebuilt a few demo models. You can always build your desired models directly on your machine. [Read here](##Build-your-own-TensorRT-models). +::: + +## Configure Settings + +You can customize the default parameters for how Jan runs TensorRT-LLM. + +:::info +coming soon +::: + +## Troubleshooting + +### Incompatible Extension vs Engine versions + +For now, the model versions are pinned to the extension versions. + +### Uninstall Extension + +1. Quit the app +2. Go to Settings > Extensions +3. Delete the entire Extensions folder. +4. Reopen the app, only the default extensions should be restored. + +### Install Nitro-TensorRT-LLM manually + +To manually build the artifacts needed to run the server and TensorRT-LLM, you can reference the source code. [Read here](https://github.com/janhq/nitro-tensorrt-llm?tab=readme-ov-file#quickstart). + +### Build your own TensorRT models + +:::info +coming soon +::: diff --git a/docs/sidebars.js b/docs/sidebars.js index 4c45cadbe..26cb09eb2 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -199,6 +199,19 @@ const sidebars = { "guides/models/integrate-remote", ] }, + { + type: "category", + label: "Inference Providers", + className: "head_SubMenu", + link: { + type: 'doc', + id: "guides/inference/README", + }, + items: [ + "guides/inference/llama-cpp", + "guides/inference/tensorrt-llm", + ] + }, { type: "category", label: "Extensions", From b4eff9a1083b0811bfba99b4283421377df6842f Mon Sep 17 00:00:00 2001 From: Nicole Zhu Date: Thu, 14 Mar 2024 20:55:03 +0800 Subject: [PATCH 10/17] docs: fix slugs --- docs/docs/guides/{inference => engines}/README.mdx | 2 +- docs/docs/guides/{inference => engines}/image.png | Bin .../docs/guides/{inference => engines}/llama-cpp.md | 1 + .../guides/{inference => engines}/tensorrt-llm.md | 3 ++- docs/sidebars.js | 8 ++++---- 5 files changed, 8 insertions(+), 6 deletions(-) rename docs/docs/guides/{inference => engines}/README.mdx (73%) rename docs/docs/guides/{inference => engines}/image.png (100%) rename docs/docs/guides/{inference => engines}/llama-cpp.md (89%) rename docs/docs/guides/{inference => engines}/tensorrt-llm.md (95%) diff --git a/docs/docs/guides/inference/README.mdx b/docs/docs/guides/engines/README.mdx similarity index 73% rename from docs/docs/guides/inference/README.mdx rename to docs/docs/guides/engines/README.mdx index 289fd8241..3a7cdcc44 100644 --- a/docs/docs/guides/inference/README.mdx +++ b/docs/docs/guides/engines/README.mdx @@ -1,6 +1,6 @@ --- title: Extensions -slug: /guides/inference/ +slug: /guides/engines --- import DocCardList from "@theme/DocCardList"; diff --git a/docs/docs/guides/inference/image.png b/docs/docs/guides/engines/image.png similarity index 100% rename from docs/docs/guides/inference/image.png rename to docs/docs/guides/engines/image.png diff --git a/docs/docs/guides/inference/llama-cpp.md b/docs/docs/guides/engines/llama-cpp.md similarity index 89% rename from docs/docs/guides/inference/llama-cpp.md rename to docs/docs/guides/engines/llama-cpp.md index 470424dbb..bc485df6c 100644 --- a/docs/docs/guides/inference/llama-cpp.md +++ b/docs/docs/guides/engines/llama-cpp.md @@ -1,5 +1,6 @@ --- title: Llama-CPP Extension +slug: /guides/engines/llama-cpp --- ## Overview diff --git a/docs/docs/guides/inference/tensorrt-llm.md b/docs/docs/guides/engines/tensorrt-llm.md similarity index 95% rename from docs/docs/guides/inference/tensorrt-llm.md rename to docs/docs/guides/engines/tensorrt-llm.md index d795d55e6..177cc0cf4 100644 --- a/docs/docs/guides/inference/tensorrt-llm.md +++ b/docs/docs/guides/engines/tensorrt-llm.md @@ -1,5 +1,6 @@ --- title: TensorRT-LLM Extension +slug: /guides/engines/tensorrt-llm --- Users with Nvidia GPUs can get 20-40% faster* token speeds on their laptop or desktops by using [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). @@ -48,7 +49,7 @@ We offer a handful of precompiled models for Ampere and Ada cards that you can i ![alt text](image.png) :::info -Due to our limited resources, we only prebuilt a few demo models. You can always build your desired models directly on your machine. [Read here](##Build-your-own-TensorRT-models). +Due to our limited resources, we only prebuilt a few demo models. You can always build your desired models directly on your machine. [Read here](#build-your-own-tensorrt-models). ::: ## Configure Settings diff --git a/docs/sidebars.js b/docs/sidebars.js index 26cb09eb2..8deafeaa1 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -201,15 +201,15 @@ const sidebars = { }, { type: "category", - label: "Inference Providers", + label: "AI Engines", className: "head_SubMenu", link: { type: 'doc', - id: "guides/inference/README", + id: "guides/engines/README", }, items: [ - "guides/inference/llama-cpp", - "guides/inference/tensorrt-llm", + "guides/engines/llama-cpp", + "guides/engines/tensorrt-llm", ] }, { From ab73941d4256549b027c28dc0bfd1a27f1c7b1bf Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 14 Mar 2024 12:55:07 +0000 Subject: [PATCH 11/17] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b25a917d7..6f6044b30 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 1fe3dff875de61b53420058200741d5eca7fab7b Mon Sep 17 00:00:00 2001 From: Nicole Zhu Date: Thu, 14 Mar 2024 21:08:50 +0800 Subject: [PATCH 12/17] docs: update slugs again --- docs/docs/guides/engines/llama-cpp.md | 12 ------------ docs/docs/guides/{engines => providers}/README.mdx | 4 ++-- docs/docs/guides/{engines => providers}/image.png | Bin docs/docs/guides/providers/llama-cpp.md | 10 ++++++++++ .../guides/{engines => providers}/tensorrt-llm.md | 4 ++-- docs/sidebars.js | 8 ++++---- 6 files changed, 18 insertions(+), 20 deletions(-) delete mode 100644 docs/docs/guides/engines/llama-cpp.md rename docs/docs/guides/{engines => providers}/README.mdx (54%) rename docs/docs/guides/{engines => providers}/image.png (100%) create mode 100644 docs/docs/guides/providers/llama-cpp.md rename docs/docs/guides/{engines => providers}/tensorrt-llm.md (95%) diff --git a/docs/docs/guides/engines/llama-cpp.md b/docs/docs/guides/engines/llama-cpp.md deleted file mode 100644 index bc485df6c..000000000 --- a/docs/docs/guides/engines/llama-cpp.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -title: Llama-CPP Extension -slug: /guides/engines/llama-cpp ---- - -## Overview - -[LlamaCPP](https://github.com/ggerganov/llama.cpp) is the default AI engine downloaded with Jan. It is served through Nitro, a C++ inference server, that handles additional UX and hardware optimizations. - -The source code for Nitro-llama-cpp is [here](https://github.com/janhq/nitro). - -There is no additional setup needed. \ No newline at end of file diff --git a/docs/docs/guides/engines/README.mdx b/docs/docs/guides/providers/README.mdx similarity index 54% rename from docs/docs/guides/engines/README.mdx rename to docs/docs/guides/providers/README.mdx index 3a7cdcc44..aa3bfea1f 100644 --- a/docs/docs/guides/engines/README.mdx +++ b/docs/docs/guides/providers/README.mdx @@ -1,6 +1,6 @@ --- -title: Extensions -slug: /guides/engines +title: Inference Providers +slug: /guides/providers --- import DocCardList from "@theme/DocCardList"; diff --git a/docs/docs/guides/engines/image.png b/docs/docs/guides/providers/image.png similarity index 100% rename from docs/docs/guides/engines/image.png rename to docs/docs/guides/providers/image.png diff --git a/docs/docs/guides/providers/llama-cpp.md b/docs/docs/guides/providers/llama-cpp.md new file mode 100644 index 000000000..3a21e80a7 --- /dev/null +++ b/docs/docs/guides/providers/llama-cpp.md @@ -0,0 +1,10 @@ +--- +title: llama.cpp +slug: /guides/providers/llama-cpp +--- + +## Overview + +[Nitro](https://github.com/janhq/nitro) is an inference server on top of [llama.cpp](https://github.com/ggerganov/llama.cpp). OpenAI-compatible API, queue, & scaling. + +Nitro is the default AI engine downloaded with Jan. There is no additional setup needed. \ No newline at end of file diff --git a/docs/docs/guides/engines/tensorrt-llm.md b/docs/docs/guides/providers/tensorrt-llm.md similarity index 95% rename from docs/docs/guides/engines/tensorrt-llm.md rename to docs/docs/guides/providers/tensorrt-llm.md index 177cc0cf4..4b0edec2a 100644 --- a/docs/docs/guides/engines/tensorrt-llm.md +++ b/docs/docs/guides/providers/tensorrt-llm.md @@ -1,6 +1,6 @@ --- -title: TensorRT-LLM Extension -slug: /guides/engines/tensorrt-llm +title: TensorRT-LLM +slug: /guides/providers/tensorrt-llm --- Users with Nvidia GPUs can get 20-40% faster* token speeds on their laptop or desktops by using [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). diff --git a/docs/sidebars.js b/docs/sidebars.js index 8deafeaa1..b95e4044f 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -201,15 +201,15 @@ const sidebars = { }, { type: "category", - label: "AI Engines", + label: "Inference Providers", className: "head_SubMenu", link: { type: 'doc', - id: "guides/engines/README", + id: "guides/providers/README", }, items: [ - "guides/engines/llama-cpp", - "guides/engines/tensorrt-llm", + "guides/providers/llama-cpp", + "guides/providers/tensorrt-llm", ] }, { From 70fc24f6f1857f19d0d837fe95086a373443f210 Mon Sep 17 00:00:00 2001 From: Nicole Zhu Date: Thu, 14 Mar 2024 21:12:16 +0800 Subject: [PATCH 13/17] docs: nits --- docs/docs/guides/providers/llama-cpp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/guides/providers/llama-cpp.md b/docs/docs/guides/providers/llama-cpp.md index 3a21e80a7..d2b0daa2a 100644 --- a/docs/docs/guides/providers/llama-cpp.md +++ b/docs/docs/guides/providers/llama-cpp.md @@ -5,6 +5,6 @@ slug: /guides/providers/llama-cpp ## Overview -[Nitro](https://github.com/janhq/nitro) is an inference server on top of [llama.cpp](https://github.com/ggerganov/llama.cpp). OpenAI-compatible API, queue, & scaling. +[Nitro](https://github.com/janhq/nitro) is an inference server on top of [llama.cpp](https://github.com/ggerganov/llama.cpp). It provides an OpenAI-compatible API, queue, & scaling. Nitro is the default AI engine downloaded with Jan. There is no additional setup needed. \ No newline at end of file From 4b6f218639b5c3f5dc0f231ea88af386c0b1c9c0 Mon Sep 17 00:00:00 2001 From: Nicole Zhu Date: Thu, 14 Mar 2024 21:45:31 +0800 Subject: [PATCH 14/17] docs: nits --- docs/docs/guides/providers/tensorrt-llm.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/docs/guides/providers/tensorrt-llm.md b/docs/docs/guides/providers/tensorrt-llm.md index 4b0edec2a..52da83b36 100644 --- a/docs/docs/guides/providers/tensorrt-llm.md +++ b/docs/docs/guides/providers/tensorrt-llm.md @@ -3,14 +3,17 @@ title: TensorRT-LLM slug: /guides/providers/tensorrt-llm --- -Users with Nvidia GPUs can get 20-40% faster* token speeds on their laptop or desktops by using [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). +Users with Nvidia GPUs can get **20-40% faster\* token speeds** on their laptop or desktops by using [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). The greater implication is that you are running FP16, which is also more accurate than quantized models. This guide walks you through how to install Jan's official [TensorRT-LLM Extension](https://github.com/janhq/nitro-tensorrt-llm). This extension uses [Nitro-TensorRT-LLM](https://github.com/janhq/nitro-tensorrt-llm) as the AI engine, instead of the default [Nitro-Llama-CPP](https://github.com/janhq/nitro). It includes an efficient C++ server to natively execute the [TRT-LLM C++ runtime](https://nvidia.github.io/TensorRT-LLM/gpt_runtime.html). It also comes with additional feature and performance improvements like OpenAI compatibility, tokenizer improvements, and queues. *Compared to using LlamaCPP engine. -:::info +:::warning This feature is only available for Windows users. Linux is coming soon. + +Additionally, we only prebuilt a few demo models. You can always build your desired models directly on your machine. [Read here](#build-your-own-tensorrt-models). + ::: ## Requirements @@ -48,8 +51,8 @@ We offer a handful of precompiled models for Ampere and Ada cards that you can i ![alt text](image.png) -:::info -Due to our limited resources, we only prebuilt a few demo models. You can always build your desired models directly on your machine. [Read here](#build-your-own-tensorrt-models). +:::warning +If you are our nightly builds, you may have to reinstall the TensorRT-LLM extension each time you update the app. We're working on better extension lifecyles - stay tuned. ::: ## Configure Settings From 758afdbeb44589076201aabdbdcf618bca3a581d Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 14 Mar 2024 22:11:55 +0700 Subject: [PATCH 15/17] fix: incompatible GPU error message (#2357) * fix: incompatible GPU error message * fix: change port --- extensions/tensorrt-llm-extension/models.json | 47 +++++++++++++++++++ .../tensorrt-llm-extension/package.json | 4 +- .../tensorrt-llm-extension/src/index.ts | 15 ++++++ web/screens/Chat/ErrorMessage/index.tsx | 21 +++++++++ .../CoreExtensions/TensorRtExtensionItem.tsx | 3 +- 5 files changed, 87 insertions(+), 3 deletions(-) diff --git a/extensions/tensorrt-llm-extension/models.json b/extensions/tensorrt-llm-extension/models.json index 31bb11a9e..30f345f47 100644 --- a/extensions/tensorrt-llm-extension/models.json +++ b/extensions/tensorrt-llm-extension/models.json @@ -45,5 +45,52 @@ "size": 2151000000 }, "engine": "nitro-tensorrt-llm" + }, + { + "sources": [ + { + "filename": "config.json", + "url": "https://delta.jan.ai/dist/models/turing/windows/TinyJensen-1.1B-Chat-fp16/config.json" + }, + { + "filename": "rank0.engine", + "url": "https://delta.jan.ai/dist/models/turing/windows/TinyJensen-1.1B-Chat-fp16/rank0.engine" + }, + { + "filename": "tokenizer.model", + "url": "https://delta.jan.ai/dist/models/turing/windows/TinyJensen-1.1B-Chat-fp16/tokenizer.model" + }, + { + "filename": "special_tokens_map.json", + "url": "https://delta.jan.ai/dist/models/turing/windows/TinyJensen-1.1B-Chat-fp16/special_tokens_map.json" + }, + { + "filename": "tokenizer.json", + "url": "https://delta.jan.ai/dist/models/turing/windows/TinyJensen-1.1B-Chat-fp16/tokenizer.json" + }, + { + "filename": "tokenizer_config.json", + "url": "https://delta.jan.ai/dist/models/turing/windows/TinyJensen-1.1B-Chat-fp16/tokenizer_config.json" + } + ], + "id": "tinyjensen-1.1b-chat-fp16", + "object": "model", + "name": "TinyJensen 1.1B Chat FP16", + "version": "1.0", + "description": "Do you want to chat with Jensen Huan? Here you are", + "format": "TensorRT-LLM", + "settings": { + "ctx_len": 2048, + "text_model": false + }, + "parameters": { + "max_tokens": 4096 + }, + "metadata": { + "author": "LLama", + "tags": ["TensorRT-LLM", "1B", "Finetuned"], + "size": 2151000000 + }, + "engine": "nitro-tensorrt-llm" } ] diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json index 01ff3e2c6..96ede4a56 100644 --- a/extensions/tensorrt-llm-extension/package.json +++ b/extensions/tensorrt-llm-extension/package.json @@ -1,6 +1,6 @@ { "name": "@janhq/tensorrt-llm-extension", - "version": "0.0.2", + "version": "0.0.3", "description": "Enables accelerated inference leveraging Nvidia's TensorRT-LLM for optimal GPU hardware optimizations. Compatible with models in TensorRT-LLM format. Requires Nvidia GPU driver and CUDA Toolkit installation.", "main": "dist/index.js", "node": "dist/node/index.cjs.js", @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "config": { "host": "127.0.0.1", - "port": "3928" + "port": "3929" }, "compatibility": { "platform": [ diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index e3014b447..cd85601dd 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -20,6 +20,7 @@ import { LocalOAIEngine, fs, MessageRequest, + ModelEvent, } from '@janhq/core' import models from '../models.json' @@ -127,6 +128,20 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) } + async onModelInit(model: Model): Promise { + if ((await this.installationState()) === 'Installed') + return super.onModelInit(model) + else { + events.emit(ModelEvent.OnModelFail, { + ...model, + error: { + message: 'EXTENSION_IS_NOT_INSTALLED::TensorRT-LLM extension', + }, + }) + return + } + } + override async installationState(): Promise { // For now, we just check the executable of nitro x tensor rt const isNitroExecutableAvailable = await executeOnMain( diff --git a/web/screens/Chat/ErrorMessage/index.tsx b/web/screens/Chat/ErrorMessage/index.tsx index 25cec1cb9..5be87a59d 100644 --- a/web/screens/Chat/ErrorMessage/index.tsx +++ b/web/screens/Chat/ErrorMessage/index.tsx @@ -7,11 +7,14 @@ import ModalTroubleShooting, { modalTroubleShootingAtom, } from '@/containers/ModalTroubleShoot' +import { MainViewState } from '@/constants/screens' + import { loadModelErrorAtom } from '@/hooks/useActiveModel' import useSendChatMessage from '@/hooks/useSendChatMessage' import { getErrorTitle } from '@/utils/errorMessage' +import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { getCurrentChatMessagesAtom } from '@/helpers/atoms/ChatMessage.atom' const ErrorMessage = ({ message }: { message: ThreadMessage }) => { @@ -19,6 +22,7 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => { const { resendChatMessage } = useSendChatMessage() const setModalTroubleShooting = useSetAtom(modalTroubleShootingAtom) const loadModelError = useAtomValue(loadModelErrorAtom) + const setMainState = useSetAtom(mainViewStateAtom) const PORT_NOT_AVAILABLE = 'PORT_NOT_AVAILABLE' const regenerateMessage = async () => { @@ -70,6 +74,23 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => {

    • + ) : loadModelError?.includes('EXTENSION_IS_NOT_INSTALLED') ? ( +
    +

    + Model is currently unavailable. Please switch to a different + model or install the{' '} + {' '} + to continue using it. +

    +
    ) : (
    = ({ item }) => { - {compatibility ? ( + {compatibility && + !compatibility['platform']?.includes(PLATFORM) ? ( Only available on{' '} {compatibility?.platform From e40d0481b747bcfac8c34a95e65536e4a11809b0 Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 14 Mar 2024 16:28:47 +0000 Subject: [PATCH 16/17] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6f6044b30..adebb8ea1 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 58e12f35c978cc7ce40839f277f4ae3cbf7bc234 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 14 Mar 2024 23:59:42 +0700 Subject: [PATCH 17/17] fix: wrong engine handling (#2363) --- extensions/tensorrt-llm-extension/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index cd85601dd..02c676841 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -129,6 +129,8 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { } async onModelInit(model: Model): Promise { + if (model.engine !== this.provider) return + if ((await this.installationState()) === 'Installed') return super.onModelInit(model) else { @@ -138,7 +140,6 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { message: 'EXTENSION_IS_NOT_INSTALLED::TensorRT-LLM extension', }, }) - return } } @@ -162,6 +163,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { } inference(data: MessageRequest): void { + if (!this.isRunning) return // TensorRT LLM Extension supports streaming only if (data.model) data.model.parameters.stream = true super.inference(data)