From 32b76499d973e67586b1aad460b4577a0a9c50fb Mon Sep 17 00:00:00 2001 From: hieu-jan <150573299+hieu-jan@users.noreply.github.com> Date: Tue, 12 Mar 2024 20:54:59 +0900 Subject: [PATCH 01/13] fix: existing-changelog --- docs/plugins/changelog-plugin/fetchData.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/plugins/changelog-plugin/fetchData.js b/docs/plugins/changelog-plugin/fetchData.js index 351ab3932..a9b970b3a 100644 --- a/docs/plugins/changelog-plugin/fetchData.js +++ b/docs/plugins/changelog-plugin/fetchData.js @@ -1,5 +1,6 @@ const fs = require('fs'); const path = require('path'); +const fetch = require('node-fetch'); async function fetchData(siteConfig) { const owner = siteConfig.organizationName; @@ -70,6 +71,14 @@ async function fetchData(siteConfig) { // Process the GitHub releases data here for (const release of releases) { const version = release.tag_name; + + // Check if the changelog file already exists for the current version + const existingChangelogPath = path.join(outputDirectory, `changelog-${version}.mdx`); + if (fs.existsSync(existingChangelogPath)) { + console.log(`Changelog for version ${version} already exists. Skipping...`); + continue; + } + const releaseUrl = release.html_url; const issueNumberMatch = release.body.match(/#(\d+)/); const issueNumber = issueNumberMatch ? parseInt(issueNumberMatch[1], 10) : null; @@ -94,4 +103,4 @@ async function fetchData(siteConfig) { } } -module.exports = fetchData; \ No newline at end of file +module.exports = fetchData; From af5bcea773e0f0e574663fcf1f399f3e961f6a63 Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 12 Mar 2024 20:10:59 +0700 Subject: [PATCH 02/13] fix: gate quick ask with feature toggle (#2331) --- core/src/node/helper/config.ts | 2 +- core/src/types/config/appConfigEntity.ts | 1 + electron/main.ts | 13 ++++++- electron/managers/tray.ts | 4 +++ electron/managers/window.ts | 4 +++ web/containers/Providers/DataLoader.tsx | 9 +++-- web/helpers/atoms/AppConfig.atom.ts | 2 ++ web/hooks/useFactoryReset.ts | 1 + web/screens/Settings/Advanced/index.tsx | 45 ++++++++++++++++++++++-- 9 files changed, 74 insertions(+), 7 deletions(-) diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index 71e721578..06f2b03cd 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -4,13 +4,13 @@ import fs from 'fs' import os from 'os' import childProcess from 'child_process' -// TODO: move this to core const configurationFileName = 'settings.json' // TODO: do no specify app name in framework module const defaultJanDataFolder = join(os.homedir(), 'jan') const defaultAppConfig: AppConfiguration = { data_folder: defaultJanDataFolder, + quick_ask: false, } /** diff --git a/core/src/types/config/appConfigEntity.ts b/core/src/types/config/appConfigEntity.ts index 81ea0b30f..1402aeca1 100644 --- a/core/src/types/config/appConfigEntity.ts +++ b/core/src/types/config/appConfigEntity.ts @@ -1,3 +1,4 @@ export type AppConfiguration = { data_folder: string + quick_ask: boolean } diff --git a/electron/main.ts b/electron/main.ts index b7ac0a74c..70314b169 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -5,7 +5,7 @@ import { join } from 'path' * Managers **/ import { windowManager } from './managers/window' -import { log } from '@janhq/core/node' +import { getAppConfigurations, log } from '@janhq/core/node' /** * IPC Handlers @@ -95,7 +95,15 @@ app.once('quit', () => { cleanUpAndQuit() }) +app.once('window-all-closed', () => { + // Feature Toggle for Quick Ask + if (getAppConfigurations().quick_ask) return + cleanUpAndQuit() +}) + function createQuickAskWindow() { + // Feature Toggle for Quick Ask + if (!getAppConfigurations().quick_ask) return const startUrl = app.isPackaged ? `file://${quickAskPath}` : quickAskUrl windowManager.createQuickAskWindow(preloadPath, startUrl) } @@ -107,6 +115,9 @@ function createMainWindow() { function registerGlobalShortcuts() { const ret = registerShortcut(quickAskHotKey, (selectedText: string) => { + // Feature Toggle for Quick Ask + if (!getAppConfigurations().quick_ask) return + if (!windowManager.isQuickAskWindowVisible()) { windowManager.showQuickAskWindow() windowManager.sendQuickAskSelectedText(selectedText) diff --git a/electron/managers/tray.ts b/electron/managers/tray.ts index e7e07a0a1..b81b1e556 100644 --- a/electron/managers/tray.ts +++ b/electron/managers/tray.ts @@ -1,11 +1,15 @@ import { join } from 'path' import { Tray, app, Menu } from 'electron' import { windowManager } from '../managers/window' +import { getAppConfigurations } from '@janhq/core/node' class TrayManager { currentTray: Tray | undefined createSystemTray = () => { + // Feature Toggle for Quick Ask + if (!getAppConfigurations().quick_ask) return + if (this.currentTray) { return } diff --git a/electron/managers/window.ts b/electron/managers/window.ts index 7b97f8349..da8dd4b17 100644 --- a/electron/managers/window.ts +++ b/electron/managers/window.ts @@ -2,6 +2,7 @@ import { BrowserWindow, app, shell } from 'electron' import { quickAskWindowConfig } from './quickAskWindowConfig' import { AppEvent } from '@janhq/core' import { mainWindowConfig } from './mainWindowConfig' +import { getAppConfigurations } from '@janhq/core/node' /** * Manages the current window instance. @@ -43,6 +44,9 @@ class WindowManager { }) windowManager.mainWindow?.on('close', function (evt) { + // Feature Toggle for Quick Ask + if (!getAppConfigurations().quick_ask) return + if (!isAppQuitting) { evt.preventDefault() windowManager.hideMainWindow() diff --git a/web/containers/Providers/DataLoader.tsx b/web/containers/Providers/DataLoader.tsx index bc1461d5b..7d38a29d6 100644 --- a/web/containers/Providers/DataLoader.tsx +++ b/web/containers/Providers/DataLoader.tsx @@ -10,7 +10,10 @@ import useGetSystemResources from '@/hooks/useGetSystemResources' import useModels from '@/hooks/useModels' import useThreads from '@/hooks/useThreads' -import { janDataFolderPathAtom } from '@/helpers/atoms/AppConfig.atom' +import { + janDataFolderPathAtom, + quickAskEnabledAtom, +} from '@/helpers/atoms/AppConfig.atom' type Props = { children: ReactNode @@ -18,6 +21,7 @@ type Props = { const DataLoader: React.FC = ({ children }) => { const setJanDataFolderPath = useSetAtom(janDataFolderPathAtom) + const setQuickAskEnabled = useSetAtom(quickAskEnabledAtom) useModels() useThreads() @@ -29,8 +33,9 @@ const DataLoader: React.FC = ({ children }) => { ?.getAppConfigurations() ?.then((appConfig: AppConfiguration) => { setJanDataFolderPath(appConfig.data_folder) + setQuickAskEnabled(appConfig.quick_ask) }) - }, [setJanDataFolderPath]) + }, [setJanDataFolderPath, setQuickAskEnabled]) console.debug('Load Data...') diff --git a/web/helpers/atoms/AppConfig.atom.ts b/web/helpers/atoms/AppConfig.atom.ts index 75343d722..2c678b77d 100644 --- a/web/helpers/atoms/AppConfig.atom.ts +++ b/web/helpers/atoms/AppConfig.atom.ts @@ -6,6 +6,7 @@ const PROXY_FEATURE_ENABLED = 'proxyFeatureEnabled' const VULKAN_ENABLED = 'vulkanEnabled' const IGNORE_SSL = 'ignoreSSLFeature' const HTTPS_PROXY_FEATURE = 'httpsProxyFeature' +const QUICK_ASK_ENABLED = 'quickAskEnabled' export const janDataFolderPathAtom = atom('') @@ -19,3 +20,4 @@ export const proxyAtom = atomWithStorage(HTTPS_PROXY_FEATURE, '') export const ignoreSslAtom = atomWithStorage(IGNORE_SSL, false) export const vulkanEnabledAtom = atomWithStorage(VULKAN_ENABLED, false) +export const quickAskEnabledAtom = atomWithStorage(QUICK_ASK_ENABLED, false) diff --git a/web/hooks/useFactoryReset.ts b/web/hooks/useFactoryReset.ts index 06a637572..da0813060 100644 --- a/web/hooks/useFactoryReset.ts +++ b/web/hooks/useFactoryReset.ts @@ -30,6 +30,7 @@ export default function useFactoryReset() { // set the default jan data folder to user's home directory const configuration: AppConfiguration = { data_folder: defaultJanDataFolder, + quick_ask: appConfiguration?.quick_ask ?? false, } await window.core?.api?.updateAppConfiguration(configuration) } diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index 70cf335c2..0433ba75c 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, useCallback, ChangeEvent } from 'react' -import { openExternalUrl, fs } from '@janhq/core' +import { openExternalUrl, fs, AppConfiguration } from '@janhq/core' import { Switch, @@ -23,7 +23,7 @@ import { ScrollArea, } from '@janhq/uikit' -import { useAtom } from 'jotai' +import { useAtom, useAtomValue } from 'jotai' import { AlertTriangleIcon, AlertCircleIcon } from 'lucide-react' import ShortcutModal from '@/containers/ShortcutModal' @@ -42,6 +42,7 @@ import { proxyAtom, proxyEnabledAtom, vulkanEnabledAtom, + quickAskEnabledAtom, } from '@/helpers/atoms/AppConfig.atom' type GPU = { @@ -56,6 +57,8 @@ const Advanced = () => { ) const [vulkanEnabled, setVulkanEnabled] = useAtom(vulkanEnabledAtom) const [proxyEnabled, setProxyEnabled] = useAtom(proxyEnabledAtom) + const quickAskEnabled = useAtomValue(quickAskEnabledAtom) + const [proxy, setProxy] = useAtom(proxyAtom) const [ignoreSSL, setIgnoreSSL] = useAtom(ignoreSslAtom) @@ -87,6 +90,14 @@ const Advanced = () => { [setPartialProxy, setProxy] ) + const updateQuickAskEnabled = async (e: boolean) => { + const appConfiguration: AppConfiguration = + await window.core?.api?.getAppConfigurations() + appConfiguration.quick_ask = e + await window.core?.api?.updateAppConfiguration(appConfiguration) + window.core?.api?.relaunch() + } + useEffect(() => { const setUseGpuIfPossible = async () => { const settings = await readSettings() @@ -361,7 +372,7 @@ const Advanced = () => { Vulkan Support -

+

Enable Vulkan with AMD GPU/APU and Intel Arc GPU for better model performance (reload needed).

@@ -426,6 +437,34 @@ const Advanced = () => { /> +
+
+
+
+ Jan Quick Ask +
+
+

+ Enable Quick Ask to be triggered via the default hotkey{' '} +

+ {isMac ? '⌘' : 'Ctrl'} + J +
{' '} + (reload needed). +

+
+ { + toaster({ + title: 'Reload', + description: + 'Quick Ask settings updated. Reload now to apply the changes.', + }) + updateQuickAskEnabled(!quickAskEnabled) + }} + /> +
+ {/* Clear log */}
From 2082b81d64835edc714e0552f0c928950bc9cfbf Mon Sep 17 00:00:00 2001 From: Service Account Date: Tue, 12 Mar 2024 13:40:59 +0000 Subject: [PATCH 03/13] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 107a35d29..9fc133a8b 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From e5369ed151238acec7c5105360e1282f45c20a72 Mon Sep 17 00:00:00 2001 From: Louis Date: Wed, 13 Mar 2024 01:11:21 +0700 Subject: [PATCH 04/13] fix: do not migrate extensions from quick ask window (#2336) --- web/containers/Providers/index.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/containers/Providers/index.tsx b/web/containers/Providers/index.tsx index 73445f10a..e70a56ca8 100644 --- a/web/containers/Providers/index.tsx +++ b/web/containers/Providers/index.tsx @@ -4,6 +4,8 @@ import { PropsWithChildren, useEffect, useState } from 'react' import { Toaster } from 'react-hot-toast' +import { usePathname } from 'next/navigation' + import { TooltipProvider } from '@janhq/uikit' import GPUDriverPrompt from '@/containers/GPUDriverPromptModal' @@ -29,6 +31,7 @@ import { extensionManager } from '@/extension' const Providers = (props: PropsWithChildren) => { const { children } = props + const pathname = usePathname() const [setupCore, setSetupCore] = useState(false) const [activated, setActivated] = useState(false) @@ -40,6 +43,11 @@ const Providers = (props: PropsWithChildren) => { setTimeout(async () => { if (!isCoreExtensionInstalled()) { + // TODO: Proper window handle + // Do not migrate extension from quick ask window + if (pathname === '/search') { + return + } setSettingUp(true) await setupBaseExtensions() return From fb893b89f3d2f1b04b54ccbafd6482d1ca2282f9 Mon Sep 17 00:00:00 2001 From: Service Account Date: Tue, 12 Mar 2024 20:19:19 +0000 Subject: [PATCH 05/13] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9fc133a8b..a50691950 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From bff20ab87bdc3a4c5270226aa9bae470228a807f Mon Sep 17 00:00:00 2001 From: Louis Date: Wed, 13 Mar 2024 10:18:20 +0700 Subject: [PATCH 06/13] fix: put quick ask feature toggle under experimental feature (#2338) --- web/screens/Settings/Advanced/index.tsx | 52 +++++++++++++------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index 0433ba75c..6853a5104 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -437,33 +437,35 @@ const Advanced = () => { />
-
-
-
-
- Jan Quick Ask -
+ {experimentalEnabled && ( +
+
+
+
+ Jan Quick Ask +
+
+

+ Enable Quick Ask to be triggered via the default hotkey{' '} +

+ {isMac ? '⌘' : 'Ctrl'} + J +
{' '} + (reload needed). +

-

- Enable Quick Ask to be triggered via the default hotkey{' '} -

- {isMac ? '⌘' : 'Ctrl'} + J -
{' '} - (reload needed). -

+ { + toaster({ + title: 'Reload', + description: + 'Quick Ask settings updated. Reload now to apply the changes.', + }) + updateQuickAskEnabled(!quickAskEnabled) + }} + />
- { - toaster({ - title: 'Reload', - description: - 'Quick Ask settings updated. Reload now to apply the changes.', - }) - updateQuickAskEnabled(!quickAskEnabled) - }} - /> -
+ )} {/* Clear log */}
From f1db7a4f86090d1d1662a381824e5b7e9a488b17 Mon Sep 17 00:00:00 2001 From: Service Account Date: Wed, 13 Mar 2024 09:31:13 +0000 Subject: [PATCH 07/13] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a50691950..23604f01b 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 7048c85465eae1ee568f75b9965d16e22ee9e414 Mon Sep 17 00:00:00 2001 From: Service Account Date: Wed, 13 Mar 2024 20:17:51 +0000 Subject: [PATCH 08/13] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 23604f01b..bc206f4eb 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 24c6dd05bec712b768e3860ec380a02417279ca2 Mon Sep 17 00:00:00 2001 From: hiento09 <136591877+hiento09@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:12:30 +0700 Subject: [PATCH 09/13] Add icon file contain image size in file name to fix linux icon (#2344) Co-authored-by: Hien To --- electron/icons/512x512.png | Bin 0 -> 38651 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 electron/icons/512x512.png diff --git a/electron/icons/512x512.png b/electron/icons/512x512.png new file mode 100644 index 0000000000000000000000000000000000000000..289f99ded85e1e3cc91b17af6b607d2cafd97cd7 GIT binary patch literal 38651 zcmd3NWmjCmwr%4Uf(Q5D5G=U6yL%EW1b4UK1a~L627 z(G9z*man;Hg()jap&}C?gFqlu8EJ7<5C|Ih6&i#H4?M2jW*>nElAW}UBM5ZD{q_T8 z&oB)90>XEa&~{R@HFI+P>R<|Tb#?t@VPolN{MF9%ldXe!`WZg~2t)>w5f@Q+%Q#te z|6-ugba#1PbBFJsSV(Qb%YvJuz(BFo1ShX2j&b1EkHYO==}_rWY1nlbu6yJML!SpN z0p?aUY0PXeiDi%p9zkhy>@K2P$52}Ci^a0nQ4;T{Jd^LkVYpPaz^!}BP2JZ zV_sR8MO7MZs*_}@1M>^Z88lMOI8qIsPy_)FMoUmN)_*=B@H>I+eESBTj1THB@GOTK z{Qu}rXg|DdyxjjJ3qdPVE0K;wH%N-zgfvs&C4oa_zxq)~Wv-ZAJINH4yW@wh!M=w_ z4PyNIHCkdvfZ@#Q3FG(gDJ%FQ{Fb1KHif4Tu{Nb%)b!j%ZzD;>FP6s@j~&!L{D zuEZtV&HH_&0VSF4=6mO;JB#4>ZBOysTNs*Fc&O~tQ_E?U>(a9@gh=sX?^&X)XFEP7 z{KNkU-*-W;S;sP1TbACova zh$++D;L%B=T@gKY#|5fLeGVeR_~gxN;6ME5MK^n4Gq?4)LVSrlGsh11yD9FXl0dmD(^GRWw?u{ja8t#4YqB6QIT$>iAH5h5rx zMC8 z*}?FGu-osNbO`rz-qMNr%In4%PMIc5e0Rww-_dZ#O`aPpBgCTiIir7?ok<*%&xiNA zq+TL1!htE7J59&t}go z!DF=a*y&JjBfZ*NL*6SE+(abh_gtFw4de{gNI1dtGew0xUROnOu*D;N1#LAZkh;Js z@(=8SxOENAY>j9&zV6UFu^-c{Zfq&QcR$&Ez~Ao5lS>w|N`@_8r~<5>0I~Ygh%5rh z5pf_=#pfSJ==R?2Z=&Jv)yHOd{w4;ao|}`kzVYO&i5HRGw4;3Eg?d zS;;rip{jBkmxH2x{_-^Dpbh-%_^5?iGEc6WpKtKzUb4B9^JIbh@7d=8e;*NAkS41i z%bfF=#d^*@yMABHvRTSrhUR&#P%#x5dwAT8`9?fDU07&L=8j21hu~esKxIitt#lSa zF=BYf@rQUmFV$0IAehasdO7~&X-}SSw@dj*4{ksaRPJNE#&0j^%v~2}<|a>f!_W5d z-)8NV>YT(9iF13ao>~Ja_9`z5YI3nUb3Ex3P!A%FR(4rg_h7A%tI)*(o^$mAvb=AP z&=QhII_`qyUWOMq{$cn^@@^f{Ff9;f5LxOWDJ|xNIPPh#nErXA(wYPj@L_u0WrBjy zA+QaU4Q!{ArT22Lq*NshNXp zJ1?qA>u#Lbsr6Cy3=J7qBfE^;We5gJr~2C*ufOZ<4ZZ}arcwvHdH+xc3j4Buphs#9 zxRlUQiRXAWEL{v(2=;9uNl)Vwm_@5wOIlklP4z{(_Y1rvLgoBMdXaNXu$EFn@Yu5$ zmBh^aa%M{7a))}ZPOS8!VN26EW#))S$AoS4WyEuG=Hn^N-A)x^I!m=jm?{+@QEpB; zy)DdiaQowBWlE(r4zwq7C%sqnd}fOK!NF5}qJ}HgF#Y-&wOr>ct9_1sVx>oSJQK=J z{2nrHU_~)ZtBQx+*#NXZIgZRHcA{O>Yo`vSa26e69iKaJVuy|cq!dfOkT+rRM$KoZ z#7CDS!Vro#SIYCO3G&*JnD(|Fa5ChD?Bn;wV+TeX7yH`uFK2P3U8c2{$Y8Q76Bzxo_3*qT6kVx6Mgy%7T#TVk!tpb%^hYnM%Zb5PUC{@}D~eR&VrBA+ z>$zge6Q$-t&V5`aWP75D_jH({DQS#-;@2JVGOKQP0cD>)t6#;uo~*$ zoD?u<8hmZ7z~Om2`N)xqbHe-|==NXJ`&!X96f(_bhyhVPD2Gcq+%}BT;Udr-HafKH z^k+%6V5eE6+kvn?8C38M+(YeOYO^DEUrt@6lD_S|7Oisq*h-%#_97)4>_oJjG!6s3 zjVrn2%H~cd0|s3nT;N32WY^{|Vf@)o(*y@Edbt$kjC zYim4<@jgGl6#kM`25VOe(yH;C`OF=fWn+tvE}BPLqP0_1fX45Ilw&pP*_Jv(!$t4# z710w~Jn;D3@N!VR$N3N9B9fm5-u%2baxKQ7x4)ZEul;KsKRScVLnUR#y6!lp==~&8 zwoN#1mw)JX42k~nwn!9{l3mpyKdN~~HEL{jO2t!JE%!Luxnb22TBNT&{-Z)jO_5TQ z!PEq(FxoAVc8xiCrDJ$OQL*geACezV8%N$J_=lm_`NsfQ?Jjj!3gx)T`X`)R0{)b~8kqvM3a%@VOY|5Ce*+>0{k!qai46H3Y z;q!iBj0gT;8YpNfZhHe3+pL$91GJM4aRyyWsjL8L>HQSXFvP<&@`U#{9aDN6tADIl zB;sM0^kcLJtk0kgaGfJyFfTnB|0a<5eBm9}F^^m&KEGCkIb%2QB-IGN8v&Hn++1}T zCLX%YQsV=jqZnm3(oy6)oJ}-hH(D6x0_-BU=CN_RvyCaj_Vp>Au`=K49;5vZj&{__ zJ(Uo306AH8QebtvDJj>nTxZFZ*T=eEwsZ)9dgaAq~9dG4z zauw~509!sQC`CueltdQM0DG5EE~%0}lyfG@7cdP8<1Fr8;|Ae%*T?i2TBbZJ>ED#= zq_&11lhcN=cwl7({#OBL6{q9{r$=neDgXpMX!_gPW0$oeL}pkRufAt$wHs0 zT3LU$JoG7*I8<&{S8UrI1BF*HtNy|;<=ctuC0Cn@A8hTsV`(eNG`5XlmMmf07hQ@* z+3@K2SC_=0&S5}ySV*$)#3!$rFKTWB76SU%q8~wK^GXH_RYl(XcIMi~AhFng-~xqB z;wsDU-v`e|oib3$S#`B}rk$s+BIj$t!)tI~c8&5HiPY|3S=g%Im7aJ#zyU=aSnsP8 ze}5o2xO>$C!fYu8dGDq)%lf{5#LU9?)?#+#^g2ygt~?}mmvn9~CKu|j(?h8el70~S zd~;hVIFpCH&__({v7q(DUn;~y(Iv67iu$;;C_6>K_-H}_ShlAf>M!_wFt!M5H$=z} zu|O;Y;gCo>c4+cmH1A%>IE6XORSEE`KKs?Rc2pkK;k}58DMaY=h86_>Sba>D|5S) zZ;RFL@Jh7Qbr)cg=W5r-)RxGj_c!1~h(&cFjv!zX;N!t5TZEDb-u`x3%MkLYa|rfF zhL!tarY&EsuY#c~7|wz1(g}%Jau1JSz9x5aclVlWA@htW4xnsR@-vKBw4H`fTzop+ zLz7ESb66){M5rlJc=;5}#l=e@g!rVLXPT4=(tg01u~(6Dqonz8wHyNv^3Mswtl+{M zYK=O{L|&}aPUi81}O}@QW=Pj zSBT$^A({`9#__U4wOBq5st@_x8yL9kY=k!gg&I%eS~`dztfw%Cy;%B@NlsV^k4SxCP zc*GGPy{V|YMy8>$U8WS!d41al_D826p2aW9sQxps{9=P4YSF>SQl_vigp%i?boI*! zla(@D%!XuqcpwegPKz{z*92!Hy#;A-MRA$s>fP6K6o$~tbAPm@g=yt1% zlJHqIiIB&Q%h1zuVepKir7LgTIeGreho4?2XE+Rz>lX`?lar_J8AX#e#U}FK95PQ) zmHz>ib z%>VHtpcV=K{>zmp&wx5fo_NJCUQwg4PzYsOH|`Q{_*DCyw{t(fxswL3z`2w*k*K@9 zx{ZFl*IY><%0y|l2h9Pr+Ku=Q*b@j8PObgO(Qigx(25Osi=BS*^%scsPbl%m3IJWe z4>uXVM@54^&W)PVx^$wutgUQ#pvtlHG2yOJXWE=9)a{3S^(|sj-TaxUsb{{>8^V3ig(g;t_KQA z{o^|H^lsp=CK$fDQj=OJF!T~774}DsK=tf3*U6g_?h;T$99q&YL9j31jMdcsMsjG< zzRA4ON@+?S)n~ zlYd#7&4R_J9^o}fYs_u*J~uFr(r}5t6@B*#yR*g!0k=LIlmG@jkbTn`79yeJdX7Sn(=>d+q$0LlDsq8WXF65j zKh!^T%z&yX(YLhUC?J?gMOU%4mQgxPTeC4t63MB4vUQXLTgI5|ATy%CixWYWyTQ6@ zHeDg1g-5N>%9&_i0C`z{UW1!+(`offHMJjuK(b$2n{A<-WtP=b~2kJi=u1WI83d98_YmXhgcn))g zbT7j2umV1<%`*5SZTPn;5~K3_($rZQQ2Ekynu-Xg(%|_j5Dw}(^>z87-?wCzSxpswWekGPKa4b!d4o?y`|C5)h)hS z@_dtbm>9B_Fn6Eb&u{s5__5q0R31cTF&uX9a>;&IFYi84*TAxR_77=MrU;Bw%Inp8 z?V8IcW$+kpt&2Z|`Kwl0QD3>kIG4XcsNwxP`+Op25xl{qmriNoI_|#-qDB(=Y~%fo zk>C~nU3jWPJ^=e|4Es1Fo1K{y6kWeU7e7i3uJl~~l?|nd&qP0dC!gJRmvJ`2^4C`? z5C!*jZ?oGnUZC?$3I#^u#GRGb8#zAd(Cehd;wkLmZXIzh1=B(F}RyAEC2`_Donw!tlGE)AXzBvULzk%KZ1_0IDhGm z@O-$d@Jf!lL`L){TZPS1293q{qEs*W*0AbRtn+52sv)j z@XEAcuP|gUcGZ|=bx0&qKW$OVJHGfs9En-HP$RS7YB{Vc^15uIfbp{Ctza2x&_6RN z{9#iLUY2+&OJS5#c>p{Fgpcp;i+Uoi;>0~@VuaAqwoI&G*|~`7dj-TlDg%Q-q4#U( z#Vn0QA?J8e*n^`M@wV`7W7;AGr6DAFcY>p7oHv5AokJKQ5H|Gxd)-ZaUKMyuU|)Slm95GlhEAg#vffCzU)V2d zfyH=fDRL)HOsRE9;4AfnvtpH$O(t^Y%~_Vy-rj>n&}mX6-1OMc@nZd(qs%+=rGn1V z-w1(`vZ8cK09kV#a5$W{AteF0X*}n59YhPL4%EaTf28wqY{?MbexX-niakS-eHq3? zurI$_%51A&bgBaOXvQO5G{d)()>g!UyM%Xa9!w7KnJBPnttejKU!NadE0=4%0PL$w z*rXlxNFuYl(nNx)D(;Kt1OL6tmGzXN&J#0<5Pg3GTS=Pk`W;8;#j5WbeZV^@WRu(4 zp(~Ndn-S&4>$4CN-f8u=JEinTw8=iGk$1!{&w7!cl(fA8j9T+6UdIDzF|2nc9x=vr z0E{hSIa%=3m~K1ZAs>eY06`}bDk*O@@SwkAP-2-dA;EGvomZl3{Zg2g6c_2eApgg( zf-pSx4bH5K01)puv8R*D$@J~?k5%+69#nk{Bq}6je_0PF$ zhwt!Y6TiPML)ozgROj%TO4HChJOC229o%U(RbUuMErB!7T4RC2=5L!9@`BA%$$iUVe?C&W7UiQIV9dLXpGtnL8zdRC z0;Sm=>n6}aB>gJQX%IUqrV-*)D@#7+PR9De?1vbZ72z4xC9O#cxV45h(Ssywg}}M# zH!(81TD?9~_$^qiz+_kv+jwdk0NqXPY|+KV`Dd!&SJQH%im_;wpJF#Wji5OF~#jNA$8;7UdZd&l2vpZN;5KeKTWQM=}W4 ztA}0RZ93JzNTuLR=XsqLl_ur6NmakR^)#-D6?4C$vpb|&+|Km+x3(&6*qAu&O$-A= zhVV{O#64UX(_MV9{rV}SFQy!&v=Xi|=)$?>E3S@Y{ynRolL?}*{{c>?8D1R5;Eut) z*&DA#;5ozUxzu~p2?mAfPh6U%{`ZyUCwoOxTc__4(dFRR(L-6rno+y1YuqcVfPbT~l^KM+b3VRTcw#~fZP~?>Tem%mt=PtBs+w!=Bfi*f)N;;H zG(l+}pszEj`DsdYt zAo?B+CKmMF+2aiTYx+5V)>4I^`&&leO#7RIe-jSITk1$zL=R=qe*t^R5K8YD=~qe~ zgntfDDk(i88(|%&y}t0JzDluEeCfr-ZZ|DVyVK|#G?m7kT{u^nOMljyybI4^obLic zipD>-mP&YzS$GZtYEIZvw%q)TreKPPt6K%;+&*`4HKJton+*`${kJDko_T!=_Ik~a zmEx>+@@i`yci@l-WW%E5^j%4^ZT681Dqrmz7d4%7n74rPSHY(pi+Kx85N6tp7uLt; zn1W|Y`J->^D2}wf;Zj05USPcRy?1LbR>kMkfxut=QKu^RMp$*~z5f*a%`-V~cs2w( zCVq{0%9~cbAAR%qyOlyC5G0gW5`C-LpJ1kynncYe1J~5nFDyM~})|DU>)~|GJLnvZCefTSJ%A0vtiqt5(S^jO3)l|Q? zk-uW@&W8wQK|aS1x`s0360diNTi{lE74QM=T#=%qK$Hty9(_X{=+2&c>SJ;8N zKhA!=_uSC&RT>!rpn|)Fk;Nl+z%s*o?osbRXp+Q~c;p6zturNv$>%>3Huz9p2q3*; zw9(?6KEfqkuj|mEcKVJC$#@aQy-&x-T*w6X3BmNJa%V<3_Zm2zD@~398a6i%Zi|z* zXP49t+6Zsu|K*@rhyLgST+bS+zrU>j@$6iW1~vGedNJ1<1zj#FhOzScr4`Z0T8B}J zAi^YYfpQ(z1)ee9X&VfY-+{W=_|V~sTkLkume{i-^tRQ3d>T>-R1Tpqw`wLUmlfTw*N5yvgb3H!6hdFlpUxl)@M3EJ)(tE$FpItZ7dD-1rl$>z zVWicwpDg?wtUDHEr^aaV&!ysdVEUAE;0lAQ+B7Ll1Zt+bvYa#!B>B@+WZ~Rp&_A2~ z7I*`j+a)gQ#zXO!(2Mn_tEJa|kSU)XHJpebP2LIUZgGp8|G=A39XD6lWi1#xY=UxQ zRmRvnS+6uCKEN3=M7_Bo?o<_;xHV}Dqh`~70!B@mm-rpKIJB*Wj7@CS>)Z1&hL5B4Vny_3DL_*7lKf-=hq@i8~4lEEo5F*rJmg+mu(n-(m& zc5Jys!QI}~j<`eZp`CrltT5b<`$Op$N3Xctf=9F>A4eDtf7nvi6tPRE9=&@;yidu| z_P&dv$ueC%kW9D&3)YjDbcr3f+x%J|fZNq2Fg>v}U_cR7Lb-|$*kuTRv?idUcqD~2 zQaoUiy!~Y7=%Gdx^JD&=HoO(bEeDh7teqZ4bD$dRSiNenUwWitNIg?@x&xs5{F%2~ zCD)H`&0cgBo(RUIzMoFYb9B`f{00qF1`z3I>OzPoAnf1qofGf_#usqlDR#|XQe3wV zdBgAj%~$-XoUF?mySvJLR+@Y8$n28#XELY)Ltrlghr7__&D{b9(koNw2KtY8Q8) zLEvfDmA9d-T0*OB*KmGT0gc_UnA*w;RugF4inXf)cYb#gL-%3%$mZR*gU1`Hl=y=s zSuR;?Y!n*S&T>AUTzHa?`bun1-D1CpIv-Z*0a==U#t;`97Zf!JXg|HRT0} zgxJ5NqO*IOr_hqs(ORr0St;L_@gHf+hLA{B=T0wBvoXi#)YL6nc-xGmw$!&h3cK9s zk(iAg43U3C%Lc>!J}@#ZD0^px7}#7-(r!hebhd*q6=4~S^-Bd42pt*}G8gRk@M9H1 zB?@H9>+kF{77M`}GUHDmqNxUa!gUW~J}6qJNScqp$j2e$U$%1L?W+(RS1g&1p~zv( z<~!a!?%`n4O%Wau?x8!vqVJNCWx17zVfwtJziA~8Q%88X5-yBM&imLW|GsIhWF9k{gkhWya_ zG~q@f$w{I6=6XR^bM|7~yie$DTAL&n4RN7wIQ;Wp$`Woc-d2HWQ*^;_R+8D`|Hc1PnO3#+oPev*Sgh z%UUbQ5-E6$_hoh;#{lzpZ|_suJVBFpHP8dV#AevsU?Y%rOqRFPrgY@b6P6P?LRa1A zA@X>qhf>MWlKQ~zhLG--b%iA#e%+U2ez(~6l4#@+9nqRaqc{lhs3zU!6bJy%6dR`~ z>soHO{$VkGg60u&%OPqa#LXorM=|J)GsS6?fsFZ!<3WT5zMe=5V9jEzH%NtVm<$oL z68f#&LKS1XFHCd;J#)}ORWnwXUX)$LI_9N%S~z7ixyI?a^=no==il3DDdsws?Wl+BYH64&?`>`xq z9z%{1Cch<5z7*#@ROKXk47|t+a?ikd=DvwXd{_UksSW4NBYBV>AXzwNjtX*JDZVb_cKx z57zb=SvV_hcwL~CrN5bKk!@zUJL@M`_e7ED9G*wo?ce^HYpJ(}P)5%xa1f1^1GOzB z<`gS;O`K^WK#cez0gJHvc;ntt&q-*t?+#^$C9@7`J7L=Fu8TF8y@p3cjprk7>^r}T z%A1ilO$8cC*U*cQfY+F#Ph&^!z_&1UgSBtf6>{HFY!P(!dq+f!oLZNZb8jdEYo^_n zG@}Pp%Om8c;0k*}AGl_=BRqek1bF>lK4W2&!}l3sqTZaZCHojmk7w3^ibAnA7V_EZ z-2I#6oT0JWkRSr=5egtsWYG#5?MhpRFYN`Hp$j-;rcltzPW~9xiecj?*?Y_5c_?>J zgT|siL95FRj~<*Vnrr$l|1RS3TP*T6wfFdW%`l@Hkde2Swa{A{Y)zH){C@{uJiG!JrgH-K#^D>!TbHcR1*=|Rz_AyEspOkCGsMSmby1I8j81<2WMwF0i^`} z5r>?q=0h-L)O_6~!cF1)`x^q>j!Wt-m&^bM={I`$pC_`x%mi!#VQ_rtsH$QS5+1cnvjGjqT&b ziYQ`Am$oGbW(7*wcLQui*`E>E>VWR1X6eHS5lrmkxMO%TE4rWb#-1?Qv+D?jds(V@ z*U8{XJIaM~{n{MAEp45s={iWtIbA6eD4 zD961|mi~r&pw=gu$9Th_*VW~bB2f5p9GJ47xV>h#&+|z1{z^_9r{%LWkAEncCiWXK)hakFK&bq|{i@b*%gtE{N@~}%75&ufLBhiY?nwU0%Q<+;OWZYe zcxRy1otq}J`iLj6$I)A}dG|i`JoQ-}uL(IC(Na=DY^SGlQTXq#Y4rw*6-jDG%J}T? z=ZVQPsL<&iDJ}mG;voV6bj|sUXq3LEF*kIw^io!DR221|T-WaKUg=THk<9H>JB{$9 zFO#0tzNxB8CBn&tFaLFtd_K zV+X%<_>SnrNkE@I9suLgLG&1A^k24LtXEIVg#@S5-=D&g$zEQ&(PrrNd>g`)4S9ce zQ@-y$x3qd=30|`+MC<2Yi@Zn#(#;nuF|vsCETo3>-eVIsA%@%APT_Nom98`xa=Xi@ zAcxi*>Gq~if=VPjL8}NKLld2M%kepq^Z9>WP{tUG5zxVdzgC}ZzkNm>X2R3{MbHcW z1^DNI!+&RsWJc+ynP76AJ>tt^|o;C!(!lWz;<&2nh^A5gGASuS(D{) z_vHMV#&*7C)HH zGoz$~(i(H@D3F_@ubaeWc_hg~(LyjMM_0Ukq5A81Ww}L!-^&FWor>;6$d@ZN->LXK zJ*@0ocl9ZkT+@)ezrw#pxDd!5!r@+-|0d)S?hilF3o9SA_Z`sqjk{ z4V*pCVAn@zD`eQUqr#@VjZZs6W*smD2QJB?v9wH-hPodbqY!Z$*63*PLY=;6eem|& zX#1Hph%Rj^P{D>hkZd)tmvqctD((|b?n+TkDA3@FuS}n-=x7HPnUC}^`j__)NjM*% z9${TelolqXoGbW`S{d{<0Qjic)*f-^3SgvhB>c0zL<|)=d;6xitGwN|y{bl+X9AhLjlm#I04^`~ z4cViy9Wd(*M!yyh-nYYvjAMzX>9%JGZQ2}bm-1@gy;=mou)l#cYNeH8b0;jIoOH_RUTt8Hxx}ltna8a=D066vPqHG z&$TJK_)v*u<&K$z|7P{Wbt`snH=1+`NWD%Xa4?~Iyw7J-0_?gJ*W^PAm-l- z8GCNj)~}wR&7_9hn!Fc#^%`JASO#=A)8v9&aL5)?+SO$$Ugz|mZKey?b5}^6-(iUd zPV2(>)^`7{Q0(sD+Pv0sfw~eSrBRYCh2}yKCHiFDBmrJ~2T!(~cNZrX4UaK)7%B3R zM6fx0I!=g8K7u#g*8~2z=eXhr;OV97Oa2#b0GJ!gSou-s@F{RO)&Sq=J|a#VT+Wc`5=JUq3VL`4WcG4aMiog&w=(%ge#5{KrOtXtm5y zd(O@u+J8=Zgpw|oyeDNoIQGltCKu6~!wei!kzy6in2cgrLK43A!FBf2fALiNeg2A1 zYN_)PV#%gPW;ns+m)lVc-x<~??}a51xIKdDH7A@e_HO4T#b-K8M_gJ9;p`)W&Tbm* zr#W8p#{_eU%o;d%(~ho~0sdAsvjwFGx%Qe}6Ro{{;o|gt@bJu}&AVDGzv9IHq<{}z zLVjEh-&lu(AJ(u#5w&U=6YlV$b|-qx4dL1eo!_a&C2BMdZIm%q^dzU@%4GUWBXGVB zb@){vI!Ex!CTO8EAg!#r%FFS@t`xC4F6Afc*xn%(`>1@y5T&$hlB)Nazoy#;$gU5S zDmzCt9C^$z4*(n=5qH74D@)!ngwHY`)eM(`k_(`<3sIad)~n~Z!R_D2=&>v(7RIRA zAErO&209H%2Il2OeG@fPqTw@i2n@gAW6LkRVM~W` zlIq5jW#d!5O4h3pZrU+{a|6`KIGiqsHqP2yYfLmjsM8lL&h?OkVC8|qJZ*Z3T*fa5 zRyq1aKHsUQK!0!V8z{hzkINxK7rN^a%0QeUPx)&Br~Fv}YK_)_@AIJL1TT|0>lN~E zzJ+|6b6Aovz<5g1$V0K7+vyN0{<){awnA>ds1M2JpV}$N&CSBwY*BZ+Tchl%i^V=x z(kRl38F#=VAFpAkeQZs-oPtc(fixzw#WTB^F>e*}iL-~+3B;Q_4Kit;_ z5e(1XZq7$Q=vdRwsplI9AQ9q}1$$55rfjIrNE+IbcPmU<{F_UK?yABKb2z_O7qiJC z0LY0}y=&yr&{zPtH$jU%RG$qqrZR9b64vL*1^1YL5mes?#t4XgPW}6pV-gr_sm1Lk zQgarN+jZUYy*iz`bai>5_vq~gojI^3(o*XtM3pS4ypNR!NvC97Zcw&z#irZeVyU9~A-{?!i`Hsa^e1`mWy>OH$+GL&rwY``>rm zds~x7Qk~L{WW^^g5A9+6FX4?`A6EYa^88zECH50ICC(vk1KdaSjZu&=0`wQ^7N;yL zf#;3~J$=T9C|a@?ck?3OZ~1=fQI364ZZ7;_AHYoY>tZ+a>n(iD=-hRDzwm{w<8J~O zqqU&ux)pM@pSVXf^HJ@f*gF6$Vq84?_vegJS;-bw(UMqV-Fla%ld=z2R4p>ht z;=~n9p)+hMmINGlqH75RRpaw(xLsR=W%~I7Un7knY&dB^w;JFjc!Vd?bO~$P^V~6n zyZJmX0j3Iwb+|hzO!xB@PwV<^q9YN*M|A7__}=z| z5=>aQo^CrD?7F_n?^gNXvrIlHWhDs7o;J`khxXexA~@*)bGq=Nk)Jj{o4uYWc$U?( zGC00}xNOH;ye~v*Ti_wQ2WM&cT>Q``oXPTbz&}V*{vq{I^V@G8bh7S-p3@=a4H80j zq|NSPP2DF#oiE?CZB9)PfhP0##iM*C=E+D63)Nx@D5i7c-D&in+>8=@8w)S%hnBPHx21So8%I%m&XB9 z9f;to#O5>5@fgCyuF11;!SIQd5XY|V33n+OBYL%ZS7AR0+zFNwI}3_l4+M(M)|A1F zy*_D076mR*ECZgg*obchR|3z@!5efNvR4oCu)* zrP9GIwDB87STVX~Rw*^WG~iVdR>hU3b)Kf;tdxV}-C-=mzHB2QZRi0$qPGyWaQA#T zZKv0ZmeC>HCwuD!?c@n)!~si2HU$20()})slCqVkZ>s5cj`PlNgB$N@i&0NI_F5e6 zHX%GkTk3+hK?-h;cuoUPOe|Fj5`>+tH!>w?uQ*HTX9O_rNoma@0cfL-3y?pPJihD` zAv}eE*qSf(3`K4z9=jF~#C71P0-)Q5Ao?Q|4}m58JOFgSNB?p}DD829j`imU#^%;M z9GDX(*3foqCSB#>&t)?P`z#S+(1_8!dxD&?d-dZWc9bJ9?d_hgySj0G<-XWiY1&lu zJB9s?C;zV&wvpoqj~bwg!4OMeZ8pCVe7mH*_md1!tfEX#(7A2OU5$0^>~Z!&oFv=U zt>Hh<5lR29|GWw@`n8?p4UI?Q9Klj@lx_$q3-*Kt0Vso8LUZ;hnpB>L?R$PAnmt<{ zqr`lK4y(73O_a-b26>e%l%5TG}6 zfk>Xe5rjPZXn*;7yT0;zeX^>uC42*RQ+4nBr6TQ%W=L7(Do(j3ezm#6tdl-21 z!>cQV-Fr1+_zjmfle_MuT#S34P(%|)8rH4uNf0z7T3=l^e*wX;v{ygmSoC-luTFkI zitOi*PnPrf%0!qg%?cK%O-#Gb_@ppI`?SA@m7yycclKflr6dtpDDAAMZoXxQXOU`L zXZfTgNU+j#?^tIRB=3DJ^VH#))rCojy?O0K-!!^fg>{YU*@{zXtk)<7>QV%SPiUX| zXGPy6KON1~iNX8{3TLadjveL1E3s{@sDydh(<|YAUmzC21gGEdN9DQBmp`WQD*n~w zBYJ1#fC2{sK_UJpl&oUEP@hjnh!2yiRoOEqFkVM@>m1@szC2;6=Y&+|j_@x-Fr#&t)VDvMBDsXjb$Q_yXmFe~A1E3#?+Aswx&o z5sxxs1CBBmqMjwWD_<%MH(BP05IX}H>em_J=N%}j3(wxMcxBXrOj}xpTuyx%l8+au zKxNDT@d0kCKxKfXpw#OTWFfmhbF#mee98nT zZR8lnpElzBvo1G;l1WIlc>mF^G5UkD#hmg>*cD;MHNINP1C{3Y3`Z;ebw0H#Z&!@{;8s>~Y}*Z8fSeLZCZt8zns z88l8u%pQM;wc$r+$?)Bo;bw;~X0xo|ufFCGqnbM4f@4PXI~V9l(uj31@>Z&C4*>>O z)^98A-&}t7N{waKRYb-hkFqVR&;8M<>W@P0!8Ao$m^EVO2(N=R z=ZzbP7cmrY_v*k%(Wf5g8q1FaZ2s#fkTcyT7bYEbKd?`z=LYbajgZv(e*V6s)odmm zBdGwP&S>%W&%kL#XEB+`6gHff$%`qiV7)SbFEwD${w-JeU^QYZ4a zWKX06cEC*g6h6`s^YTk0sq7}5x(@OwnDUZoNUJx)~#dqfxa$wjW{yCyVxF}sm> z9;dFazh>OcEfdk+Q_K739|zjW<;f^uitTseD;TRN@A!ypFK@;lzFCK4Dp zS`}TAlk;amqFq^=L`=#)`FMZ(+Ya*uawX(_>lacWUHUH9?OS$Yy?ONAUT2T}H>jl% z2DLH-$F?SQ1QuWbA>xbu$nBm~j%RWNV_|Y=w+CNQJ*Ee3g+@|jm-++a9no^qj$C1Q zsz#?R&ATJPUf51&aKkGaAfS#p+r@wZDgUhEyp*jsh$9-&qx!r%DG2k`V0(Q0-2XP+ zIMp-pJ-*qFNdbMbRfoVvq`$eTrYhz^RD`$bCWE_m%p7j?TqfdM#qw|V z>eln-yuFpb4zvca86H~z=gGS+^DWb&`!5&3^VoPOuCryjZ!?4mWmh1W;&lBVu4|ry zMX|9s&`f!x&%xq4etTxTM~&jm_Hn#+hD^_yrNJQOSCa(QeNG`JtAy~mk~+8>eFLs1 zr0&@~zyD9-1gI==4>~1D>7V z-b^2zfZO-U<(X3buR4}_coxnLzNN*6@YCy@H@T{PuXv4^&e6 zLv74oc~h34Z%_iwznqkvXsQY*95Y2H&cp!3e!o!|ZH1f`)&-hHIl^l6FM2Ny0 z(@A{}`LW;H#kg;8AeL`eLPZNmhJ&tat`0BNdG7^|x~axccA+W#Z_T^=nWP~{BOtS6 zXul@^BxwFhqilFT0Gf{=Mq%uuY|bkM298kd6of+2@9UmOJF$S5b#nsrm)}N_yulVg zhWtsAh>1~Qt3wtqZ|V_`+(xuJ2egyDv#Nf=n43w|9db}uTmlh0u2>q_m<6W&0OaoV z+cZ|(5hh!Y6NcqlDBrGRKlbp1e`g}h0QxWZ+jF6%A3$4A(bVO8$Aw=!WE95BXknAk zqhRWhsY2-b^R5nPkjhPzIc&oG)u4v&8=`kmUb7c`ARj<14Mo7MJSpu9D!{XdW`-A`^ISrFuds;#l*sc3@IwqFUA-g-2NOA+Y$7LE@_!l|a zI?9g}38<_!*c)A@ZifRF!_bYSf}|h~0!m1XfPi#&r$|ee z3?bd!E!`j>Nau%;2I-LQZkYG-{XOsB^V~CcpL6!wYp=b}xmbReq4dKbx9zp&pu6wY zyn9A4%6%qaF7DxSW!dAbu)!@z?vlt`LkRc%0!}I)q}7s`k(aU%jE`whB{H4VRdX?r z<5zg_oj05K8oDsSC=QP=%ALjoef$K8_{0hR#~*y{_FKjL9)jY+ob#3;G%c1aoZeap zv`k~$$JZ6$9*mBd=cKyU14(FD;h>Zn+Ma5mz3rJOhDBxf-v|~(Jre1I-~{Xv+_^Le z@ZGg8@$NSZ>~>i${LZ9?A79kHunTXPXwSc_mDkSXWB=g!%7qiv=@!wuEt)@Re?dRm zG84v_BXP!_P@bCVmKj8v!oOD6(bOpVGC~^jIF*dyvAmy%D>O9pv*S2n8inia7TV&_ zvlaZHin}0HwTS9X7Sr{w2XR?q%q4d$nq&3WjVj#nB;X&l;bzWZPyG^yjNOp(0Ppt z_@g^RUPr7=NQA!ZmoI>hHPHO=Z*7J2&(Q-c%5xV?qc$Mi4?ji57j7ymI$$vQ+Om_U ze4#T3w<9apf{p(ab1;@@m3?e`DgmZvlEDT6TG5i;ztu3ojkK3C-MSsTIGiAkSu{6~ zSyLe}T|Q2Sb(Udk6565Tm{`1dp2he_y{|1WoNP(nP&YlRoQ%2{x2bE`|z*BHkIn5 zzISUT*|%KYS{Wedc=v!S*FERHGvo}rlh@np8k!Hs!xS8?k)pH#3Bo)=dt;(Kr?C8E zL4(DE5nr{T-HF<-0-Kywy97esg{WO`2`iFlRp;`UJYRIWOUA+Xwnt}sJ8T_jY?Mz> zxM%ggz_gC%91QaxLrhCAb*VOZJ4HI3QmuX151N>(wive%Gm7-&5IaGl*h zqBGWP4tZ@3r8{QM_*{VnZI$>TN8!P?ls46P5^#)Ck0 zg6&_HzOPfZ;Yw`yQz&`GPTgm+A75%~4LRA0@sIezqbK=cgR^j01&L&J)f;PD{H@kO z;h7IqiaBD3@WCW4NMLvL`U7kDCo;Dj6Uq#i1*&>YxukT~XKa;<(-grN78zOH!_s~& zRH}nvY|pt<_#Us9AA*kq2X`nEhyY2$3gMH~_G}tE2^MA%DstUo$pKUvP0ccE?5%IL zJiWcsH#|CKiuAp2*&xwzLQ@g;WYr%MkH4 z)E~P>v*Kos)TjfokcA(w{x-ca0Z8pgucb=tR!{)T8l;sLQV9dx!Fy8seHLa9YPggZ zf2d?;xZgm|sDEU^2;^a+v2`x7JNKn)oR5UbwDls|s9b+XIRFH!V>|rj7!H5Vc6K86 zkqJV$Y=iR8#q7k3QCU0EP|)$1`%bYd? z$?tBHfC>!)=?NSn+?SD`d+-bAJLBqzNKBZD5eJ|9k|Xr0wtICms%^z{9}s`ism0%R zcD9uC7Dw87Feg=_Al)HavB@EHWw!Lx_#w6|l&qew3SxPHxGZdLvO=2W6+8Do15idq=25a_`OR4 z_q800TD-n|FrzLCwV@*Gm$RWT;-N4*1k6XQz}GktiTX^R|3Mu8VZ8iUg3oV_;58rl zXh-8d5}}W3Qi53Vp&imvc-c_E-)!k^nxG@w&RH|!(8mlpYg?r225I9PGRqgn{-uhF z{6p$-1&T^qxKANES$n*XvQ>m;d}wqyw;RX&~X6{ZN+bxih4XGpvV+ zk0L>~J2;)*3h|vv+wD=F*Oyqm$$3-3@hcT=iPidHhi#iGIdNkA;h`fOV?*OifK)}G z;9ds;5>?af4ZMAk6WzC+I~iwLhiQ_IsvGP5l#cB12>rsNzC@Y(xG0XRwq9@Dn? zOv{b!`f=9P*yliB!!lqsR#{wnkbs>@j8mPF!zZniem4~gfq%UIr1KKT+HSQ+=`|gm z{v*ue4gugZ{60`-hQ?Ze2$lV>4$pa`_yuNAr2W?`ty=V}`r$lOq!b-84Pr%n(Lcs# z5f%~#`(O55-emk6s6;YcEUE<<_dcdZJ|@6d8EE_El z>#)LFD)hWEYRWH-8R<643V-jh+oSkpB-Wb^b2>L+0MANd{oPJDJ_c89%&J$Cg>(Jo7&LZ#XHfyrX-IKszc5s330tfi5#{P=(`odi2GI zTK9fzV&(oGb;729RgH92m{zn-RFCBwDpI2FGWLojn&K-{5%a(V*?*Oc|6X3zJWsx7 z2H;Biud~s_7>i!mB2?LPC<1kuLN8^nD`!4ZG2*L6gCCOH z{@-WSPAU9M+bqeYcS6q_Ea`&Vwk&bku}Fw$b$7j9FhkqsxTDe|L=SfFXxcp~Ueldw z=Zo4TnWBDknH>8|&InL*^m3(z@Q`jxsRLYLt5KItfFH`oKTS>qy~*u_nF{y|bN7yb zmEdrPSxgNkF`Jq{>gv%Q$!JJD-6R-(B;-kTJG~_s_Z#olXaaW#ysm-~ubgi~fYY^I zi6$0+6$KVZkfL8`UnOPvtM=D4Pm1HU@&?$jaUZ73NpaS z*>5c9GHE9RJ#jh9g1Jp8(b67$96*KO2Hm|}Yt!DG*zS64v(79zrTLa<3xP3NyD}v} z1}l+(^dAI<03{GhK^6CD~^Xwe~%#y(3?IIV{?{Dy2Bul4j zFv1X2aKXJ5dew1kXpFM6#@Tc9@tDo;WOGadlxvPVwh((@URT+i=Lzk^HDvQusTCmB6c#*cx z+0m}PV}=nhyL`KheQsctCOPvLH(L4|PJA;jXTaTU8kEU4O!WGu;%>*EUamLf3)PB( zIx|MT>yNh5{TGM!ELpw-wMek}b!SY;yyvKxB@%osu8$MmSc2Bibr=Q$q6d$fB|z~1PiwX+1kuLcpY zR${XMWP%+|E{RGvwak1u0`2^O*wNd$(eZxj%;r@-@9|lM9?{UEG3gMY!F-E zj+QKBL(g23whQ{EA*8`s5Qdicc$N!r#_g~n{C^ufqMeol$qSPh(RLaT_@l(ju@AVH zDV9EihRw^YYT4^sfbG@OLW0&zQD9P(&{ua)JO9DZ5rEylEW`-DCkgIV)phiyg5<;p z8szw5!>80qh0O{kAo#!=m&SN$ZEzU@K%Fyp%4~J*S%3^h#%P4^aiayBe_z`pJIvr@ zefGI&0G)KJ(3{mVMaZM|9z);t@DK*XxJ1z)GtNJ91Sa(s+TVaMnJFGM8r`v5 zYHP1F3oP-kY>Irw$YNmlCD(Ol40QfJ4i`56@e2i}47lYtJK#j4+!cri;}ko2{I%vI zQ#eXi$drYN`ymL!)fF-`K8$@3OAq&W1Xk~Y^zvQ=J4E0BBmxSs^;#*d@8g>=b!O+p z8be`=0#B6aAHcB$P9ZRKCJ^CEJ+7$umO{H-39V^Dto? zAh0f;d?`f5@mahBejvi)$`B_ec`PwPs=R15vH#^zDl`wHvm{7gVN$bdL5^cDcq_Td z80BV^G~XiS^LzY~{mgyrx*(Tw^q)|@10`IN`*8t#A{;(JPJ!0#^fJn2QDMhsAkyj^ zVYWe$6l|R%Vc0jBKl*Uo?On}T+t}nvnA-L|E6{8A4JCl|x4cRAxgkD^*voKy)b{q* zbC@R<#C_8f<~?>0H@n`fG$UUr)!`i2NExNQm^h&XJ6gqTQc4&LZjT5aWnah~uFq^cZtL*!9ga91sH% zY420=Ief6dOabK(avV8hna?87e}ZmA+sSbyETvI8U!uj4+6 zyyB>U6f~e1eozPQ^GbI{3~kQ^BAj9hO37#_Cf`o}6q1S~E^zVky)Fke=WTAGXH0Aw z&Lyd^>ksotO?ERd4*|{ivj+A6PaeJD2qSZuPZVv&=Ju(j6sbgjurkO#?w9ooje=qc zI>Q-x1nci0x3119FZ)t-t|0eX>mBL<(2c6WME+|aoIlVvNqbZ2;*C#q_91&x|9jlN zSSpH7N)$fElpAN-9NVan44u3Mx}-UoMen|odlgn(G{*D>^v}6+RXRe zDQU~w^72Z&)sTcm+vlcH&XZd3#v~)}Rqo%=;}N?5_eZycu2<<X?OyfiTA+Bs7a_$@OrE)wQ@-X5O3L*8F@WDRWnmme1fYdI7lAZ_ zRBAso9!IolboH}#r=(mb;rqjaqSUfT6Wz0~oXZ(87B?`&wc$?_4LnZ?%U?D0x-nUQ zl^xv_rs|Unes}`-07y){z6|vu|NTucH0$o30Dq$I1bs(`PgJ6&IC9b%nLm;8-#d7N z`MQ>Hc6L;k((~2eJ(R$HZx?-Q$=v5DSt5y_T#^-cV7R7VNhF*l}!$(XydLk@SzGa8VCc%afnoL6YLP3Z^#d9q8PWZ0CJrt zBsua}TZUFWnMW%pX*6a3`A?sR{!-$=D00V1#E9Q^Lxo7+Yup8FMIJu%{5{4O?`XGt zRxz)t&1N6WJ;@SmzCZL&ONXRGj(??=duLDn1je(@wh6VV;eI(?FPFE>R+&0e&=M1m$FFMZM$xG(WepBp@`84YzG&G>E-<21EH!ia$i;Fp3Rbjpd3@x97^;@HZVX zU`*Iosui^faBRjy;r1^-$AU*ksRL3|-XrNhZ{uUoQ4An-+^+xh#^-C$T5KZGy5GEM z)%AFbkoYJsIc2X*hLV%i?$lH|a`L90HGj>!Di%YNiI5sLfwP^1A8yD<*T*IPXf!A~ zKg}i`c-10X0#H_q1eoo*99JXop#Qe>}U%sfVmG6tWa! zW|R)(7I6*pN=NWT0U)+TXaE;L%|(%3yG~g&|KtI#(mwZPgiDpP&9n=~LKIJVnP>Y)-2%kUYf;a1iaRk5&Jjs>4zu*>H1;(85 z0>GUGK>G=iZWD_{t*1Sq3!edv0428I&H*E4+<0Q0fFCw^W6zmS1!&86Z~=~ABnH8P z<_v4wi_MZ+zaI}1fB7g*9b&?26hK4XCdN?@LGCC1jIq-_``CfM72!ul)=iRWnWB(K zobTLNogUD`OC9!}8*^3CI(2)Y`em6twj{g3!~5h61{e@6+rc~EuMGTsQ&IeUsKGrW z5VONiRE>L=r9<=A7DPbpV^x>zXAk0T5qtW2eD~m4(XBrRRk6FfU+A6EmYP=7Sp|BQtvG4W{Or1Z>BLCs%NY7M*dqm*F2kjv8nR^rX%&%uU!GfJb+e^Qo z#Y>bw9J+e67CG41Tt>jy_zDGPJ9@)_zjP*mL!4{okI$;em0}z4QFHc*C8A#%S$A^T z0kwA8mY6XS9+&&OcDH%a=E(TL8;=)lQ0odnGkGtodP^DLdsFYu0h`(CrtqW%H+Dxb zI1XIk+RO5kzB=phijg)Tq zw3|cPkEf1bG6AZ=D6J?fW|b3)zC15MN9VI8s#_+@s8;qSXeUXwMb37CssvLcZtseZ z9A;E><0i##+;1;;{Y^;$x$S!98*;+YveFnjj#p+8egM8vIHYX5DGQZF>HsmEVEc^p2T0CyO+^SiMcSi^{+W1SG+@;vI~T5NWqEGJ9X zc#Xg_w?ct|k2i*p>p)m|D^L$fnOQ+8VV1u&ir#7R=!QkD1T*+#*axW6)e@wsq0k{| zagdFMr1v;Yux=L`NjmPm3^c@zU1?lIW8V1>1_KanbIH&-Ytd9DJ zCIT3AKLPcFG)W7A$hSs=?>G@jeq{muFN$0BosQ;KZyjkj{7=mnj^4})K@D&AGWJSM zzb2Y3AswrOQ~PNGuMYd2>URob$b}8wxM^QZN!k(ggcgM#W7X4~D@2}@{DHzyJU4jV zsIQH#=1!801#th|nTVS_tp>#E7v{#ABP;S*=f@^?0Dvflm7B*1^cVEYsI ztg{VQ_sV2q$@jAacUxH4cmFq>uooIUNdR7&(*`)#R&a+pa| zWKTKP_E$-zUb{sM3HgjR{58X&@QRFiBZjpKky1mcgn_BneG8wTA2H#-)fT--$u6D- zAi&J2`qDUHhxUP!q;5EmgFjSxk&mj|IrQtZP7M#8OX}H)ga6G2esHg@kG#u&5lHaA z8{T=c%|5d=n!c22`2_Mn9(N8}R(bjJqd(ZP@Adg?_;~&i1EEd3*OE1b^ARom2KOG^ zkDiCRX{2@r!Z!HIw^jg-a!NeDzO<8bwpHQ#i3eN~R>rxu^P}6m8tIw_w2b-X>WkT$ z(!abusr)K{;A~YZmnFRqdgm07i}r}Xe)V<*p|AgTC(Uu|))c?1Ns0TzTwZ@in|jC} z+zS)Q=U4+sjg&vFD=T+IB=jJ9I@#}-Xnq^@{`*3f#90<1>iN!lx5fDUj8%2t=HGaEz%^f!dylQU4G|U54Od3_y+}{d((5)Znd-IgJ z+}0>kS`EIdOPV}13%%hLdPg17O`a*iPZ%3Cm?M(JV;zeGOHuV)jxZFqG!!ZK9$B;= z=R8@|f3?HR+00kn^?r3tS$tqh?g0frlCNehv&m4Xop?N+O(Xws-8>!;(4P&u;1%9O zOkihD*N@q#urDR~fTF}i9Z&1jS-|(f>A!cVJAt3l-XUF=fbU1ItR48ICL|*h0Yz46 z3a%?I(noCuqVl%Q!3#;d2OsSA^k=eE6n|6=yRtjl|J%7}^jT_d{nX8Fo}T$`eJU zYFqfvPIKB;7U(Zd9HPB)K3c+H**H8&kh-A`z)Xm^CF!#_mc;}T=Em=DjC@axNE>5A z^xo=R`8vG)TBdf~vA~Rk&IqiMg4&^o&X&@_Ndt=JH1EghVAp-In&ycqivXEHe5T=F z^{-lgmX3%V=`Fu9CeL|u5cRq7zX(-|=gW<#D^zY7fob4Cm*mjSuJS@7D==Z7UP3F{ zt;%v+U8e!$ome0=s$6jx=lhEXyWD;#--8oBMU^@6-(XgLf2c)@zKDR6@RqgU42j5W zaSuto`nP#JXQ`q~nVG@;p9JLV$;A|SN8bMpbu~V7;q*quEs(pWyr2>Y zjkmDfd84TqES0kFh>56;{d9ilZ08UwCH5hK=hmUS<1LJBC+n)h?pbwdJqZJR%f`!T zl8zSo@#9kFvzGfm;Vi?d_ z(L>rN0YTi;4%~u?I!~^}zL)&W4$0In#@d;CRCvgx z4HDOQ9lNMk(Sw&1Pot^0t$pwH0r#7MbK(8j(uEvtWR)2xAx``V(1!QPe*oh_B(pOOS8+pm&(A;+ zkcMX~KMEI_AV#Vs)d5|YrumV{{V%gY{fJ_D$&}sRHpo4E#?sWUfSi#+fCAKSa!~d& zX7IW(6$MXU1<35n;gWP>sdI(2dLQaVuoY3D$|Z^U4Dc1@feH4c$qI&NR{r>s;gv-^ zL!LHL1H8ItkB5t1Q!!vI<~7~B#xI8cpTPPr>r?#~)cPnqsJFkxiZV`4)R5a3zn01O zipIWH8v0Kf)A@F*B@NuixSBN{&NQ#9(5T-#vKlDtFA6u+*H5cvCNo*OF%f2%F5esljO=LfZkgalt+pAvQe4PF|sk+IB6hnp?gzI_`ZPQw#%(U;h z=d6>p@hULMpwFegEMYKZId$T-!XfguRvMMaTg5j#56{n1cpL@?4ez67EZshEBcT(M z&}j>!qlg9OW`89EOON6sf8D3GEI?xqnX`r?yz@iP*!lm68}Chez;2= zP}NmhytV?OKstgi8Ucn6u%uVf9f?yBYc8^u{Ivw&^zV>NF{<@PjnZ24KUU-vs~ z9r1H?b*jR@=`OcJIYNQRHcxD5!YdntuEHu5r|m$ox%;R4}GBiWmiJ~HLet;y_1+_z!SP?1#=>Y z=ZYcteg7D7&mMQ5jj%M=m1|wU2j&ZJM+aPnfE*GOo4{j4dF zs$`%?jS4)1NMZo3^0^~LrxEU!rjOa)SEc0D7A8w&eYn$q6_3UO$oBS-T?>Rjx*FxD zYE{hF2>@LAnbGx+EQmL9YQhz%qpEl-@|6?#qS&$^urUhWVQ2e{DijrITFS0QRtPuq zPE9(N1JpQq<9EPj5qR9}yUG0X-=m+fY4`rj+qDV3ulUqD<@KJz+7q3I%RhzS=x?E* zt@SAW#+#lo{+;zW<2#EyE7Orgu|! z;vwcn@zoRJN*SzYQNt?Hs_hb!KwY8Wc3#BtMW-3Dv`Hw$z6#mUd8>m2Ak?otDK6cc zu2AEu54|g-;MYUp%L#`zl=Q!*19N_iQI6m8V(1QhED6d>m%PgAG-wFH71Ka{tvAZM zTn@cIk=kVqoo};pV__Ty70f~TMvKtwkx$rxmj@tyFjdfD3-yLRDfeRNCq^L^xIa$n zU_g$s8JnuOK)Nd6L1_&cYIQRvWPEtPE8h4?4fm%?!9L{L$U>=6llxlo3)cX=nu8&x zGl9C!JtGu<4SFK~69txE);Xr@u^RAzWZ2d0UWEbxhGjJ;#Y|xug1i!1nAQ)|4*0p` ziL3?}4k$aOy|WQ^9qZ)Tw-qt?og~~T@aL{|WH&V0)WL=wM#E|!1Nk(8m1^Fn*#Phz z{WGFsY`s3H;c|~;)kHqGdKoyT#r;j}kFAgCbU;B5b-z16<3&C8qy9vpPPg^JbI=7P z`fI&Uc49LNkZNvKL#gAXrzhrLqm@Wfmc@@NntcBG;f$A2>a}Ch&YmGZ9(BMB7*hs} zcRHy3szuFC0&w0fuVTL}sCe2(fB(f6|KnqC;f!?wp5&ig(hui9ni z_v=X<_OFYY@|~Rtz$7mvsVH`^SwVc{15Q#r^z>j^=UrQuiZC$%z*fK*7l~mIm`i{4 z%W+E^`(P^)N`NYvx9;_pChm`|7Z%Af`qR6B@ol%_(GqL|$k~7Jj5t6DM3fG7FCrOYek+1$ zL>&nIxn=>>t6kuaX{c_zv6-vmBY4pKz&PISquJzS>2k4@Ur-6X4gwJ(MuSZ1@|+Rl zV``wHQb??3CzB^JY{>>PIT^l#RMhduBE?~qZg7UV`I|<-wGiyK1tZeXiB|R88^wXk zg7vl08uQaTaP`Xeb`+%Q7<6>fMkl^J`|oS!q8MB~6P$(ip{V(LMii$qmQNHp z!NWPFLyE3cs=nbijFu~H^AD^L-sy7P_p)m2{|U_>p`9_%caSx(=~sJ%f=|0=85it_#I^@xSF9mJl}76P;v{&{}Iz^z|*=T5tDZrXJ#t#+gP z&Uu^=5c$FIBBLAQwk1{bIZuTphUOnA`YNhgmyhW9b)V0^EoA*4O*=IB815e>8Jm8e zSYrx&ZnuHg{3koZTH*63@UfapGRYkx+Sn2P-tJbS3777fT6BbB2%9GgJfV^W+_{O5 z_~|Y7b`?6U9{D9+>=^<#ZvofRP_)l&?&~RPg~ee>5EK3oKo63528b5LOY?C6pdanr zZ;82J0memXm_J|=SnwTE*}osDbqJBJuuz=56tqNvCeN}X;BPsjZjKQd9xo410CCS? zc$A$7x%F^$O9g$vHNH8P`ndZ@lHmuJmuv&1^;kfsa8EH`>`5FwNm(1;ji7IZfpVKi ze2G`Z<0QWgz-09fd1VL}_g=#2_fYca{j2PvPqFy!f25iD9x5$s?vGTg5zWfY1A^%T zZjfjSE5S$gADw;pOIT$|+uM}EJS_ILvNT?+i{ z5E8GL2NA+%t|&nI77FYj`3(lFd07d+H~c$J>4;MNv*eVHO&`Al95#VxVLMl0yIEAF z%=!|OH5gnKfvN5oqU&tLEMUW!ub*rH`ClFzZ~2icHUOw{x+P3s6nPzS!i^N4(8tZ5 zToc06B&H^aL<_%^T67%2Q6}c5>$O|JNY)TtJV2*8jZCp`MFLd-B2(Ihl@)x$ZszG`1p=FVtw>)9=xGM%*B+Kdo^oRyZW5!j1z7K=DtExMwq_6!A|q-prxyw@+J z5wHI`2T|lk8imeYSkYx^Cbkg61L#5UZbwE;TVhY|RNN)8si-q7dz}q`;yTePy1$f3 z7FeI;{#MAUU!cm1Ac8M4MT)M!9b82>qZOM@hnQJbg=w?|WA{+CsrgH|HQZvF$w$W2zO16x*Vy-tf@ zxzP6)3B0B&Ms&UDk-O)zm4?zVzzn6m^6%!U2;4h)uvI{3V<#Y_an_R;CVwSP+f?wN z{Ymt0@ySH7nJrG$NLy1mx&yngy{xQkc5_r!-WGICg9`y1nwswuNg~17D1KjJ2?n%W z5D#3JMZ?|v3Rv(tEDmcl9Uo=<-8ihI%9I@HKX<}YK8h#g%qB>|_~#%02Uly2hcA}z z-{WtBay$NP`z4mK2w0oALkOFDdIseKeBs6PKL26&xtjC5Y%8()tmxt3(mVO74aqHJ zAQ`)&ZfKG_O87xHVW6=BtKJ)Tg#qM3oktX`1W@4I-o^RZ2rsU>8#v6Q+1t1CJe>x_ z0WY4ftYl!Wc}ek=EdsqRJw^&+Vz(2{3YBas6f;6p^Rqa~Nzm;}0b-i_A(y1ahjW;e z5<@3Wy`5V>NqdYE*M$ixMZ+K3mBf!MGwP-NJ8(oYU$q)t6RKelxo41}7Y|uorlNbq znDf7e->8GPeqaDuwTQ%mXYV8Kp3k!aQuRRO{&?8PVu^41Pqf2A!jl%6Bz84zQ3C@%bX}(_OA&MRGqjK54eYkpeczdlUs`WY)>5Hn z{{{tLTHR+0Kaa+K$QX=7#=-2~dXolNJW;_mLo{OGzsI)96qhT+^ZA!-IKR4GQQF^5 zFgSe{Ad1K}!?AvF-g&|z!Xs#9na5_cp?;CT;kT^NAXg!^1~2z&nUkqvGeVLIq-(uO z6@pH@^z-8hCowHbQ6Uz~v-BpsBm{)&iOwyzJd6M>C5#JI`Ih%uN@Z8W8SiJtJv(QR z&i!`(0;+J2mkYK5HKJ-|8nh;<~F+KEI>tkg(M^6J28R4e7O)rni&-4P)=c0h8+-> z%8z;HS%6n8fMxS(?te#+vN9A70{PLpNpE`>`K@4cc_9m^r63k8YxnO%BUv~-Xxn2F zI8Ms7El2F;2B6&{{}El`Gw!#Relo-58s$&(j+C)XT97LGYS%lvlxlsWeg*at1MlWe zx1+RgOY0s;3-7Td^SRGBNTw=p@+x)E6y{D*$!28YsAHniS^vcm0j*{X)~+Q-WL!bs zGf<%44=Eee3V3FJV<>XH2Ay-XwWY}DKvwMp_nr4{{%fAxL9Jn3e>&QgO_~F51rFV6 z(HoaQuBnJ9=yl8!(X9qOsPQ!55o@KY>NH5tSijP1(DZ$z zffY9n69b)XM1^^>(?kGZKYOF1Z>%Yz`aBN4Ysi7XVPC)o;98_|?uoO%rKY_6^Jo_o zJHh*B$~;ziG_p+k0(x<%)t7=lp^J=DoS(UJNozLtNi$eyS-#_80^&c_r#q4oJm&B) z=D2|V@IJm*uz%>Ruxx@SRd)SQAK!O`YiP0wg!CbYGyZz;qjH47M^DjhVe>Nz3==Sr z?y=!8bRp&K$dIqbG}VJB>R=IO^BZmq?k?-rd>!5*>hoHD%89Jd<&cO-If8PY(JnN& zZKU3Wy?7zL;H{q8v}SBQhDp#su8=*ZsQ3TX{Qlapb&R#sa_}lb9^8bJwc6 z1C}{T3?3|d9F|=_4gYO9I{!HXcxCB~xKXW2*Z5Vkl^A&X?v1=C0vj+4TPaxO{KRc* z-%lq!+*R~j)oTD=zY%z*mJm?#vtkfh=?qhSJ?br!splQD)_BzHJXd!OE>NY#${rxw zkO>8-u~E)yK3-UTJ4}Q6P08te@WN?7e6%C&6a$Bny|G&59ST>?8XnY5BmycvDV~M_ z?^tljjl(Ftoo;q(iTfU+Q&e6*s7^S%Ss4{CxXXk8|fMyoor7ebia-CU{WJg-8R?{CfwJC2ribb5+}N&(&q@uGip&1o?-N1ug&1Em2ji7q z&G>h}+^3BUlZ}1|L+#@l+|hF~7m0zvcyQe6=6i+q(k`pnpAz=WiWLj*^;Fw7U6G48 zkkQ{O50MUM3d~;E%FC!%@F#U#`G3xzADKI%X<>BwM{m+M$L{cQp7RS2>j4VzF>`}D z!1%vRTc_45@SCz>V1H5mP#c}_=~!!neo%t+j|YC*`_^;m)}HV2@&DZt!=a+y2M9dF z3ypm;HOS{ds#1doy8#m_ZriubW?NEnuzjGOZ(lnBt&PQ^#6uao3l!&49Y3PSL99An3`! z(}KCe%|-mt^t}+7G(v31a2ECWyJ^G2C0Dnbx)F25f3x7MN;95(G!{HmcJuGJV%7b1 zOU~XSk~=r2?PxyS?I}t0U7oIYm|B_3asa9A2W{lDr~b|Dz9*+6tOw`MoU53R&`7eT zZ&UPDUad8mzG_;;`>#zvQE~C79=dbO#s&(p7h`&18WoQZi>}KyM@vz1CVi1|@*+Ot zSuVt=k_1M$a1Dw8neU8svTC-{g0tr@IYoAyUx@xSYXGY9f#Yrvf>R>ZLZy~b%J zQQ>-Mx;}(&CDoEpzzI_i*~N~F5JDPF>}K@oVOPmk=%1VUWO}XF?c^tJIqCH7@c9n6 zEw?PAUKu@yhkDudr21;Rx-KpY_}Gp!v*=%=fN`o$%YtX@rnKX5EfF^WAkq9J{y{Zn zyWR(PmaAG9T#u(lTIg=3r>!EV84n=X+tXK~IX$b(pUz1O3FqE%L~NB;5S^a0^Qgbs zNCYjL;+NgeY(H=7K?TXQ2Nc&u58KXyE^iUly^B@!&~cU;4Pa!@x{5H z5>j7d^SYK_{0DY%KVD~{jyomQJnI|(WkMeNt^o)>DIPDa-8%lN&t5z&pHlPM=zGDl z&p*>fXpYUQt8T_SZ?_POPvelf>W>hvu4syt zTvQDkW)R>UZ!tvWo)E~iKjTME&;4M~@_8eb-&XiFhwR^RU)8S+a1xMM8)qZMXwGF- z$9}~h#R@rOe=XFX`9gotb|}X|ZVBYqiaIJ2YO*0I$gMMruN}hCuh=RK9vYf92E!J; z)g#lcZ}+J8q=8w-Zy(lw!gx|IEx(s2Xe+UR>^7~@^Wr!=AA)m?(uP-uDmM9&KNS1Ihr?)D&4-GoNb(zqJbI z3}?9|QUGlzUEm}Lxdf&E8+0488keo~5PSPpLNf3RxjpB!L~ODZ&@~k}1~|@MY&4B>|`bmKwHBvs;g&?d7X}n>YVN zAY~^YcZqzBFINe0{s8(J%_TZ#PZ3DpcTB^HfS_a$*N6u(z0x5wp8vS*N#Y()*AlD_ zLEW>srFZql8f;n}cQz2Nu4#}|>c0PP*kU99M8_2=8N+mDM<>RB zr%UQiAUh?vIX;3fK5oOuSWw@A&cdwu^tUx~d}QD5Os0h|0+kxt!v7yqnTYd+v)4hY z*Q9Rew8tj6j_cdUPCLN!aKDmcfZGP0x#e#Yms_>vHzOUU zC@XxRE_b3uXXs54E^w9A`vR85flk`iRP=aLeJH{aq{{mTdQ1pP25X;@P(DQxTO? zg9N*2F`Kb($P7!U_a}gu0JCYnSVN;RxKHA6{5jNK>v!Hu>sjH=o`OJ6)y+6dWgx%U zWguRNyFk8myr#?&ndjGiqa&&C*@8Q}2wlUZ^}gRHZpi*As9fL@OXq>8kw4tavv!^Q z-20$hUUX7Jw<8TKSfMZh)~U`N{0com@%QiBmr-l2hW&ZO+J?&i^6%~UHW;2-17E|n zuS3m|4aHHHUi`JS5ODAcsgT!{*JdX?R8NZF%?IqWU51joK^}P|drq%_awcTxvbx4D zlSlpjkF7RNjV&tmjBY?>04AC+t~glp?F?^yC&}Hj#Io0(S2BJQN(5j|eycIx0#V=R zkMvjyvWM!4tnax+Na(kJsbHrP;;DLx1|9~(b$bV-F@ehx{X;vqc@XS#!}94KPQM^b z1N%QdPxr*Vlg|=xs7Eg{+pNB^WIbw&GY&TllT zs~x0rL$c{UYvdTEC*sdyFCA++1PP-8G+*#h^(rv76g|%az-werO=VBcplcT1v3uMY zL1*%;PLUVI>2SmC)mt-JHdfX0;)}&NrNWXD0}fNRW9~2vle+@0(FmJbCdR0S_5Gn$ zRVpwa$E$7n{S6bmqXsrz18)1j`m(#kn!2?2R+Pe{zPG54lIj8@R(rVadn_hhF}bgo z{hGc}%tt@x95drx*VD&L>1s(e_HvMG|1DAlE#V(>YAn(>7b+oP+!UY0+? zSL2JE*Q}6-wa{@RI1r|BPWINa+KyHGl>f=}7SLHT``IVd>dn_0g)chL@ZtA2gh2^8 zLvi)Oxz1qiKBod2{^`a>x$6X&=%*emp5{Jt3>b*uw@{hij*^k?n8l={%;t(K5)lwO zIEp`qu?&<2YXysmg3BD{o$^iDG&siGjrgaw9G1y!ev~L2Pmh`_0!&Sc{tx=H!R0$z z!K^$fC|1GaD;snmh$X_Dx6Gs_79P(Cv)l|{|43{{=0kA6eN4VuW}Jc}&HG4+VmUbh z-8+^V)}_-fg-YkU8km_?Aa18iPWpDkNTXhq5!dd<=bZn`$pGecaD$( zl2G_%ix3a4$YmP(rH@6bY!juT|L{0Jn?4!t&<*v@0_+<& zowl}wA>z$E=agezETA76TD0*F+RtIPY?I*HO1|5ycH8|>X0ZfJWUK`Rg3YdIT7J*|N2yjkjH;rV!40oz;r(*JsKaz4QU7(`hcWW z-w%Qo9-+kL(RLae8P7tg=4>z3^Z=+Tzz1(l((TL9meBiv5%bj-<99US(5%AhTw`fu zWvK~+M3_F-9ys7~w!l=Y)vz|MZYQ1@vx(^ah?n-Fc2g5{=nw?dpgT( zgr4@vyC6v1-+Xnt5Lv=l9@aFg5jnf)L@t`@6}A}uqFP{1Q2E#)RZDBogtpB{@Mh(1 zz5s%L=gXJNv2XcX<)WNTu;%f`2Jucfjl!+w!u9L@7*Kj4;;DQ#6rj1t zGk5cq({Ly;2qH$mRMjURB@bL|B{=2bYo=KjOg& zZvaL3thAfC+jV|T$-geajkFQm-zx0I*G&>F*WzU$kvq-^29l8vBAvFPw-=L6(rDYQ zx_=?Do1QZ*J z3eB{tZ&v)8Z|0ojZ_(}q@N{?20zxJ)AhWc6-ob7DJ3~xFn#P-TgX0I0v8~Tpwq4*$gCIoA;vOIf9h+oF$KwEfxCK*KbA#x`Kq(Gi1s1Ku-K` zw+Qq?*k&d0V4By(k}~@FUI)=uCHnm<8~q=zQ>cv{FdRIs0<)uwcmm1y(a!8&cRs?x z_=tMa#yJ|N)djgyZGoeaH$P%VVfc6S;Drz%6ZHtxC7i?p<1u`ONuEoITU&-XAQJ6I ztLIgkyy+VMTOHmkHLY)5AEEh z#I=;qnzs`t`9V}}^jT({yPs;(gcm}1n&aQsHdHk%x@2Utd}98Nl&b%Dt@JxtN=JW10t8_2R*%v5_}*j&zE#>*u?192)N#G+_FZW5 zcGbQtP=7u^cTc}N>}42jXiR=m^^vD<^~7CU0>b%3#arI1VcRNor5KpkSQ@OZeuZBT zIMs*-fD$&(e^n#ZIWwoqIEF-C=3{?WYUi*n^hA+MTa2c*axoea(A5y)Q2-*t!#;Xq z=&WhPv04Sn-gooSJtEI}@mN|C3yNb|);^}sl?wz6Cgz9zE6A1M@x{x@7`}5F&lyu9 zkfhz@?JfceWHYTw3zb7>#1S8Yw#KBTs?jeDW{T4n**Ps(!`CEb<0eHJo+;`Bt@w^Z zk_xn=4bg#t;2*_bx9SQCWteg0d(2K&DV(q#$xL#c&Mi+-A&`+0Tx~8NKX`luM_mAZ zZM{mm`NnyV{et-wAq*S$D@=+blB{Jn#e=DLuE}5OX49z!O=4tX)cTzt)BE^auiVa$@{> zyU&n^AoV0i>9;_6hLg$Y?dHC}J4!@u=fR|S1`2sahK6I%uB-)j^on_kwRrgUm=QIhX__LcYX*+n<|s5{BykKGq;~|7K9eJc9jP@%pYCb+g?yn z@d{qZbj)_-zSJ`rLJqStL2D;W4=-0Z5hUC1;Km($ni*e@`D8;~t-hhvC`_ zFCUDab3cMtzoT4K%-m8gs0)bAH(C!&PWwgL-ye3w@0(mYJUNlRn4!Ulvj6AO57V?v zwHH+_Z{D_VY)tfzTfD8UkLvk7*2(Lh)R~C!JbI`UFox{C18LnLeR%$Ja~Dxo3^X@h zIbY1QI_PzseM4bEjWj}VbexzxvBG)dJ7}VVrMo=}$ynBvRbX;cqOW>|c{iiwILDM! zQiR02*0prCu_02O=6@z!wTso<-m}|9f6HcKvDejb{5L56_&1&PoQ->;iEvKMhl9uQ zA^1l?ZRgaSPoT%cNl4won_Kcm_Acd9F?`th`g~e>@~byJJtxu19hZ^H`c5yoc3?cp$?mK9gvp1a26Cd1Uq zux!-Y_-50@OL>Mz@Iol^pYadR_oh~fjr9Q90%E`)E0fjC37cMaUp-nh|tqXA;3^ks*_Vm=g5G>%N5MG zsK(q4e*v{PbW?1(Ukub-Zdz*so~i_<(><~r?xZm$%qoSQ_s3M zDsXPC*>7vFt=O`uTD*7#!$lqmcUjmx6!kXX?TJ6)~s3bp?C^vT{}CCZ>2N_HE9397Ulg z*;K7>L;#Dzw+D^}32T~s!qCJi3sM5p%O1toW{LThpsYB;x|ex_t#$8Sx1S$nyx#g0 zXo4&#vA5pxOC?MwkHE^jtz0nt&S;Zu9WrJi@5;h{cif?)CPxbC9Y80;&3{@T z+Hclh&OwF#AzFL%(nKJ>TEFi}pL13F45p0ikUz0q-Sj;vI>%)gZF`st{&UwEbQfVj zy@@>(%RKT0t-X}8mu*w)GffIEdfrnWKWj_a=8Mg_OL#kowq;ed?$l>QX+`-wIkEbt z&hm}FCx&lWLCcwW=0;ZclK+KkEzv!T&>G9^ik8^5 zjzRyHpKn}e`WaI-&Q}{E-LyVJ7>CvE`Clz@&D6JwF-s=IFBD&KY>7syw&h&CV!pU} zwEIUlo|1U06D%IV@V5_7d+47RwXMGCt7>`eRgKe|Ab~OE5oP~tCsD#a1UsR2^;92( zTcTBrLW7>%y|-SzNmYm5uNvZY Date: Thu, 14 Mar 2024 14:07:22 +0700 Subject: [PATCH 10/13] feat: Nitro-Tensorrt-LLM Extension (#2280) * feat: tensorrt-llm-extension * fix: loading * feat: add download tensorrt llm runner Signed-off-by: James * feat: update to rollupjs instead of webpack for monitoring extension Signed-off-by: James * feat: move update nvidia info to monitor extension Signed-off-by: James * allow download tensorrt Signed-off-by: James * update Signed-off-by: James * allow download tensor rt based on gpu setting Signed-off-by: James * update downloaded models Signed-off-by: James * feat: add extension compatibility * dynamic tensor rt engines Signed-off-by: James * update models Signed-off-by: James * chore: remove ts-ignore * feat: getting installation state from extension Signed-off-by: James * chore: adding type for decompress Signed-off-by: James * feat: update according Louis's comment Signed-off-by: James * feat: add progress for installing extension Signed-off-by: James * chore: remove args from extension installation * fix: model download does not work properly * fix: do not allow user to stop tensorrtllm inference * fix: extension installed style * fix: download tensorrt does not update state Signed-off-by: James * chore: replace int4 by fl16 * feat: modal for installing extension Signed-off-by: James * fix: start download immediately after press install Signed-off-by: James * fix: error switching between engines * feat: rename inference provider to ai engine and refactor to core * fix: missing ulid * fix: core bundler * feat: add cancel extension installing Signed-off-by: James * remove mocking for mac Signed-off-by: James * fix: show models only when extension is ready * add tensorrt badge for model Signed-off-by: James * fix: copy * fix: add compatible check (#2342) * fix: add compatible check Signed-off-by: James * fix: copy * fix: font * fix: copy * fix: broken monitoring extension * chore: bump engine * fix: copy * fix: model copy * fix: copy * fix: model json --------- Signed-off-by: James Co-authored-by: James Co-authored-by: Louis * fix: vulkan support * fix: installation button padding * fix: empty script * fix: remove hard code string --------- Signed-off-by: James Co-authored-by: James Co-authored-by: NamH --- .gitignore | 20 +- core/package.json | 12 +- core/rollup.config.ts | 2 +- core/src/api/index.ts | 3 + core/src/core.ts | 36 +- core/src/extension.ts | 44 +++ core/src/extensions/ai-engines/AIEngine.ts | 60 ++++ .../extensions/ai-engines/LocalOAIEngine.ts | 63 ++++ core/src/extensions/ai-engines/OAIEngine.ts | 116 +++++++ core/src/extensions/ai-engines/helpers/sse.ts | 67 ++++ core/src/extensions/ai-engines/index.ts | 3 + core/src/extensions/index.ts | 5 + core/src/extensions/model.ts | 3 +- core/src/extensions/monitoring.ts | 3 +- core/src/node/api/processors/download.ts | 36 +- core/src/node/api/restful/helper/builder.ts | 50 +-- core/src/types/file/index.ts | 35 +- .../miscellaneous/fileDownloadRequest.ts | 8 + core/src/types/miscellaneous/index.ts | 2 + core/src/types/miscellaneous/networkConfig.ts | 4 + .../types/miscellaneous/systemResourceInfo.ts | 28 ++ core/src/types/model/modelEntity.ts | 1 + core/src/types/model/modelInterface.ts | 7 +- core/src/types/monitoring/index.ts | 1 + core/src/types/monitoring/resourceInfo.ts | 6 + core/tsconfig.json | 4 +- extensions/huggingface-extension/src/index.ts | 7 +- .../inference-nitro-extension/download.bat | 2 +- .../inference-nitro-extension/package.json | 2 +- .../inference-nitro-extension/src/index.ts | 3 - .../src/node/accelerator.ts | 237 ------------- .../src/node/execute.ts | 9 +- .../src/node/index.ts | 20 +- extensions/model-extension/src/index.ts | 87 ++++- extensions/monitoring-extension/bin/.gitkeep | 0 extensions/monitoring-extension/download.bat | 2 + extensions/monitoring-extension/package.json | 29 +- .../monitoring-extension/rollup.config.ts | 68 ++++ .../src/@types/global.d.ts | 19 +- extensions/monitoring-extension/src/index.ts | 19 +- extensions/monitoring-extension/src/module.ts | 92 ----- .../monitoring-extension/src/node/index.ts | 317 ++++++++++++++++++ .../monitoring-extension/webpack.config.js | 35 -- extensions/tensorrt-llm-extension/README.md | 79 +++++ extensions/tensorrt-llm-extension/models.json | 49 +++ .../tensorrt-llm-extension/package.json | 75 +++++ .../tensorrt-llm-extension/rollup.config.ts | 73 ++++ .../src/@types/global.d.ts | 10 + .../tensorrt-llm-extension/src/index.ts | 147 ++++++++ .../tensorrt-llm-extension/src/node/index.ts | 191 +++++++++++ .../tensorrt-llm-extension/tsconfig.json | 20 ++ web/containers/DropdownListSidebar/index.tsx | 3 +- .../InstallingExtensionModal.tsx | 87 +++++ .../BottomBar/InstallingExtension/index.tsx | 52 +++ web/containers/Layout/BottomBar/index.tsx | 2 + web/containers/Layout/index.tsx | 3 + web/containers/Providers/EventListener.tsx | 59 +++- web/extension/ExtensionManager.ts | 4 +- web/helpers/atoms/Extension.atom.ts | 40 +++ web/hooks/useActiveModel.ts | 12 +- web/hooks/useDownloadModel.ts | 28 +- web/hooks/useDownloadState.ts | 202 +++++------ web/hooks/useGpuSetting.ts | 21 ++ web/next.config.js | 1 + .../ExploreModels/ExploreModelItem/index.tsx | 13 +- .../ExploreModelItemHeader/index.tsx | 18 + .../CoreExtensions/TensorRtExtensionItem.tsx | 225 +++++++++++++ web/screens/Settings/CoreExtensions/index.tsx | 110 +++--- web/services/appService.ts | 24 ++ web/services/coreService.ts | 7 +- web/types/index.d.ts | 1 + 71 files changed, 2497 insertions(+), 626 deletions(-) create mode 100644 core/src/extensions/ai-engines/AIEngine.ts create mode 100644 core/src/extensions/ai-engines/LocalOAIEngine.ts create mode 100644 core/src/extensions/ai-engines/OAIEngine.ts create mode 100644 core/src/extensions/ai-engines/helpers/sse.ts create mode 100644 core/src/extensions/ai-engines/index.ts create mode 100644 core/src/types/miscellaneous/fileDownloadRequest.ts create mode 100644 core/src/types/miscellaneous/networkConfig.ts create mode 100644 core/src/types/monitoring/resourceInfo.ts delete mode 100644 extensions/inference-nitro-extension/src/node/accelerator.ts create mode 100644 extensions/monitoring-extension/bin/.gitkeep create mode 100644 extensions/monitoring-extension/download.bat create mode 100644 extensions/monitoring-extension/rollup.config.ts delete mode 100644 extensions/monitoring-extension/src/module.ts create mode 100644 extensions/monitoring-extension/src/node/index.ts delete mode 100644 extensions/monitoring-extension/webpack.config.js create mode 100644 extensions/tensorrt-llm-extension/README.md create mode 100644 extensions/tensorrt-llm-extension/models.json create mode 100644 extensions/tensorrt-llm-extension/package.json create mode 100644 extensions/tensorrt-llm-extension/rollup.config.ts create mode 100644 extensions/tensorrt-llm-extension/src/@types/global.d.ts create mode 100644 extensions/tensorrt-llm-extension/src/index.ts create mode 100644 extensions/tensorrt-llm-extension/src/node/index.ts create mode 100644 extensions/tensorrt-llm-extension/tsconfig.json create mode 100644 web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx create mode 100644 web/containers/Layout/BottomBar/InstallingExtension/index.tsx create mode 100644 web/helpers/atoms/Extension.atom.ts create mode 100644 web/hooks/useGpuSetting.ts create mode 100644 web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx create mode 100644 web/services/appService.ts diff --git a/.gitignore b/.gitignore index ae0691605..d9787d87b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,16 +22,16 @@ package-lock.json core/lib/** # Nitro binary files -extensions/inference-nitro-extension/bin/*/nitro -extensions/inference-nitro-extension/bin/*/*.metal -extensions/inference-nitro-extension/bin/*/*.exe -extensions/inference-nitro-extension/bin/*/*.dll -extensions/inference-nitro-extension/bin/*/*.exp -extensions/inference-nitro-extension/bin/*/*.lib -extensions/inference-nitro-extension/bin/saved-* -extensions/inference-nitro-extension/bin/*.tar.gz -extensions/inference-nitro-extension/bin/vulkaninfoSDK.exe -extensions/inference-nitro-extension/bin/vulkaninfo +extensions/*-extension/bin/*/nitro +extensions/*-extension/bin/*/*.metal +extensions/*-extension/bin/*/*.exe +extensions/*-extension/bin/*/*.dll +extensions/*-extension/bin/*/*.exp +extensions/*-extension/bin/*/*.lib +extensions/*-extension/bin/saved-* +extensions/*-extension/bin/*.tar.gz +extensions/*-extension/bin/vulkaninfoSDK.exe +extensions/*-extension/bin/vulkaninfo # Turborepo diff --git a/core/package.json b/core/package.json index 2bf3e1735..2f4f6b576 100644 --- a/core/package.json +++ b/core/package.json @@ -45,11 +45,12 @@ "start": "rollup -c rollup.config.ts -w" }, "devDependencies": { - "jest": "^29.7.0", "@types/jest": "^29.5.12", "@types/node": "^12.0.2", - "eslint-plugin-jest": "^27.9.0", "eslint": "8.57.0", + "eslint-plugin-jest": "^27.9.0", + "jest": "^29.7.0", + "rimraf": "^3.0.2", "rollup": "^2.38.5", "rollup-plugin-commonjs": "^9.1.8", "rollup-plugin-json": "^3.1.0", @@ -58,7 +59,10 @@ "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", "tslib": "^2.6.2", - "typescript": "^5.3.3", - "rimraf": "^3.0.2" + "typescript": "^5.3.3" + }, + "dependencies": { + "rxjs": "^7.8.1", + "ulid": "^2.3.0" } } diff --git a/core/rollup.config.ts b/core/rollup.config.ts index ebea8e237..95305bf25 100644 --- a/core/rollup.config.ts +++ b/core/rollup.config.ts @@ -64,7 +64,7 @@ export default [ // Allow json resolution json(), // Compile TypeScript files - typescript({ useTsconfigDeclarationDir: true }), + typescript({ useTsconfigDeclarationDir: true, exclude: ['src/*.ts', 'src/extensions/**'] }), // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) commonjs(), // Allow node_modules resolution, so you can use 'external' to control diff --git a/core/src/api/index.ts b/core/src/api/index.ts index e62b49087..f97593934 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -33,6 +33,8 @@ export enum AppRoute { stopServer = 'stopServer', log = 'log', logServer = 'logServer', + systemInformations = 'systemInformations', + showToast = 'showToast', } export enum AppEvent { @@ -56,6 +58,7 @@ export enum DownloadEvent { onFileDownloadUpdate = 'onFileDownloadUpdate', onFileDownloadError = 'onFileDownloadError', onFileDownloadSuccess = 'onFileDownloadSuccess', + onFileUnzipSuccess = 'onFileUnzipSuccess', } export enum LocalImportModelEvent { diff --git a/core/src/core.ts b/core/src/core.ts index 6e2442c2b..b8cbd3162 100644 --- a/core/src/core.ts +++ b/core/src/core.ts @@ -1,4 +1,4 @@ -import { FileStat } from './types' +import { DownloadRequest, FileStat, NetworkConfig } from './types' /** * Execute a extension module function in main process @@ -17,18 +17,16 @@ const executeOnMain: (extension: string, method: string, ...args: any[]) => Prom /** * Downloads a file from a URL and saves it to the local file system. - * @param {string} url - The URL of the file to download. - * @param {string} fileName - The name to use for the downloaded file. - * @param {object} network - Optional object to specify proxy/whether to ignore SSL certificates. + * + * @param {DownloadRequest} downloadRequest - The request to download the file. + * @param {NetworkConfig} network - Optional object to specify proxy/whether to ignore SSL certificates. + * * @returns {Promise} A promise that resolves when the file is downloaded. */ -const downloadFile: ( - url: string, - fileName: string, - network?: { proxy?: string; ignoreSSL?: boolean } -) => Promise = (url, fileName, network) => { - return global.core?.api?.downloadFile(url, fileName, network) -} +const downloadFile: (downloadRequest: DownloadRequest, network?: NetworkConfig) => Promise = ( + downloadRequest, + network +) => global.core?.api?.downloadFile(downloadRequest, network) /** * Aborts the download of a specific file. @@ -108,6 +106,20 @@ const log: (message: string, fileName?: string) => void = (message, fileName) => const isSubdirectory: (from: string, to: string) => Promise = (from: string, to: string) => global.core.api?.isSubdirectory(from, to) +/** + * Get system information + * @returns {Promise} - A promise that resolves with the system information. + */ +const systemInformations: () => Promise = () => global.core.api?.systemInformations() + +/** + * Show toast message from browser processes. + * @param title + * @param message + * @returns + */ +const showToast: (title: string, message: string) => void = (title, message) => + global.core.api?.showToast(title, message) /** * Register extension point function type definition */ @@ -134,5 +146,7 @@ export { log, isSubdirectory, getUserHomePath, + systemInformations, + showToast, FileStat, } diff --git a/core/src/extension.ts b/core/src/extension.ts index 3b3edc7b3..22accb4b4 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -10,6 +10,22 @@ export enum ExtensionTypeEnum { export interface ExtensionType { type(): ExtensionTypeEnum | undefined } + +export interface Compatibility { + platform: string[] + version: string +} + +const ALL_INSTALLATION_STATE = [ + 'NotRequired', // not required. + 'Installed', // require and installed. Good to go. + 'NotInstalled', // require to be installed. + 'Corrupted', // require but corrupted. Need to redownload. +] as const + +export type InstallationStateTuple = typeof ALL_INSTALLATION_STATE +export type InstallationState = InstallationStateTuple[number] + /** * Represents a base extension. * This class should be extended by any class that represents an extension. @@ -33,4 +49,32 @@ export abstract class BaseExtension implements ExtensionType { * Any cleanup logic for the extension should be put here. */ abstract onUnload(): void + + /** + * The compatibility of the extension. + * This is used to check if the extension is compatible with the current environment. + * @property {Array} platform + */ + compatibility(): Compatibility | undefined { + return undefined + } + + /** + * Determine if the prerequisites for the extension are installed. + * + * @returns {boolean} true if the prerequisites are installed, false otherwise. + */ + async installationState(): Promise { + return 'NotRequired' + } + + /** + * Install the prerequisites for the extension. + * + * @returns {Promise} + */ + // @ts-ignore + async install(...args): Promise { + return + } } diff --git a/core/src/extensions/ai-engines/AIEngine.ts b/core/src/extensions/ai-engines/AIEngine.ts new file mode 100644 index 000000000..608b5c193 --- /dev/null +++ b/core/src/extensions/ai-engines/AIEngine.ts @@ -0,0 +1,60 @@ +import { getJanDataFolderPath, joinPath } from '../../core' +import { events } from '../../events' +import { BaseExtension } from '../../extension' +import { fs } from '../../fs' +import { Model, ModelEvent } from '../../types' + +/** + * Base AIEngine + * Applicable to all AI Engines + */ +export abstract class AIEngine extends BaseExtension { + // The inference engine + abstract provider: string + // The model folder + modelFolder: string = 'models' + + abstract models(): Promise + + /** + * On extension load, subscribe to events. + */ + onLoad() { + this.prePopulateModels() + } + + /** + * Pre-populate models to App Data Folder + */ + prePopulateModels(): Promise { + return this.models().then((models) => { + const prePoluateOperations = models.map((model) => + getJanDataFolderPath() + .then((janDataFolder) => + // Attempt to create the model folder + joinPath([janDataFolder, this.modelFolder, model.id]).then((path) => + fs + .mkdirSync(path) + .catch() + .then(() => path) + ) + ) + .then((path) => joinPath([path, 'model.json'])) + .then((path) => { + // Do not overwite existing model.json + return fs.existsSync(path).then((exist: any) => { + if (!exist) return fs.writeFileSync(path, JSON.stringify(model, null, 2)) + }) + }) + .catch((e: Error) => { + console.error('Error', e) + }) + ) + Promise.all(prePoluateOperations).then(() => + // Emit event to update models + // So the UI can update the models list + events.emit(ModelEvent.OnModelsUpdate, {}) + ) + }) + } +} diff --git a/core/src/extensions/ai-engines/LocalOAIEngine.ts b/core/src/extensions/ai-engines/LocalOAIEngine.ts new file mode 100644 index 000000000..79dbcbf5e --- /dev/null +++ b/core/src/extensions/ai-engines/LocalOAIEngine.ts @@ -0,0 +1,63 @@ +import { executeOnMain, getJanDataFolderPath, joinPath } from '../../core' +import { events } from '../../events' +import { Model, ModelEvent } from '../../types' +import { OAIEngine } from './OAIEngine' + +/** + * Base OAI Local Inference Provider + * Added the implementation of loading and unloading model (applicable to local inference providers) + */ +export abstract class LocalOAIEngine extends OAIEngine { + // The inference engine + loadModelFunctionName: string = 'loadModel' + unloadModelFunctionName: string = 'unloadModel' + isRunning: boolean = false + + /** + * On extension load, subscribe to events. + */ + onLoad() { + super.onLoad() + // These events are applicable to local inference providers + events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)) + events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)) + } + + /** + * Load the model. + */ + async onModelInit(model: Model) { + if (model.engine.toString() !== this.provider) return + + const modelFolder = await joinPath([await getJanDataFolderPath(), this.modelFolder, model.id]) + + const res = await executeOnMain(this.nodeModule, this.loadModelFunctionName, { + modelFolder, + model, + }) + + if (res?.error) { + events.emit(ModelEvent.OnModelFail, { + ...model, + error: res.error, + }) + return + } else { + this.loadedModel = model + events.emit(ModelEvent.OnModelReady, model) + this.isRunning = true + } + } + /** + * Stops the model. + */ + onModelStop(model: Model) { + if (model.engine?.toString() !== this.provider) return + + this.isRunning = false + + executeOnMain(this.nodeModule, this.unloadModelFunctionName).then(() => { + events.emit(ModelEvent.OnModelStopped, {}) + }) + } +} diff --git a/core/src/extensions/ai-engines/OAIEngine.ts b/core/src/extensions/ai-engines/OAIEngine.ts new file mode 100644 index 000000000..3e583c9b9 --- /dev/null +++ b/core/src/extensions/ai-engines/OAIEngine.ts @@ -0,0 +1,116 @@ +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { AIEngine } from './AIEngine' +import { + ChatCompletionRole, + ContentType, + InferenceEvent, + MessageEvent, + MessageRequest, + MessageRequestType, + MessageStatus, + Model, + ModelInfo, + ThreadContent, + ThreadMessage, +} from '../../types' +import { events } from '../../events' + +/** + * Base OAI Inference Provider + * Applicable to all OAI compatible inference providers + */ +export abstract class OAIEngine extends AIEngine { + // The inference engine + abstract inferenceUrl: string + abstract nodeModule: string + + // Controller to handle stop requests + controller = new AbortController() + isCancelled = false + + // The loaded model instance + loadedModel: Model | undefined + + /** + * On extension load, subscribe to events. + */ + onLoad() { + super.onLoad() + events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => this.inference(data)) + events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped()) + } + + /** + * On extension unload + */ + onUnload(): void {} + + /* + * Inference request + */ + inference(data: MessageRequest) { + if (data.model?.engine?.toString() !== this.provider) return + + const timestamp = Date.now() + const message: ThreadMessage = { + id: ulid(), + thread_id: data.threadId, + type: data.type, + assistant_id: data.assistantId, + role: ChatCompletionRole.Assistant, + content: [], + status: MessageStatus.Pending, + created: timestamp, + updated: timestamp, + object: 'thread.message', + } + + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } + + this.isCancelled = false + this.controller = new AbortController() + + const model: ModelInfo = { + ...(this.loadedModel ? this.loadedModel : {}), + ...data.model, + } + + requestInference(this.inferenceUrl, data.messages ?? [], model, this.controller).subscribe({ + next: (content: any) => { + const messageContent: ThreadContent = { + type: ContentType.Text, + text: { + value: content.trim(), + annotations: [], + }, + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) + }, + complete: async () => { + message.status = message.content.length ? MessageStatus.Ready : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + }, + error: async (err: any) => { + if (this.isCancelled || message.content.length) { + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return + } + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + }, + }) + } + + /** + * Stops the inference. + */ + onInferenceStopped() { + this.isCancelled = true + this.controller?.abort() + } +} diff --git a/core/src/extensions/ai-engines/helpers/sse.ts b/core/src/extensions/ai-engines/helpers/sse.ts new file mode 100644 index 000000000..3d810d934 --- /dev/null +++ b/core/src/extensions/ai-engines/helpers/sse.ts @@ -0,0 +1,67 @@ +import { Observable } from 'rxjs' +import { ModelRuntimeParams } from '../../../types' +/** + * Sends a request to the inference server to generate a response based on the recent messages. + * @param recentMessages - An array of recent messages to use as context for the inference. + * @returns An Observable that emits the generated response as a string. + */ +export function requestInference( + inferenceUrl: string, + recentMessages: any[], + model: { + id: string + parameters: ModelRuntimeParams + }, + controller?: AbortController +): Observable { + return new Observable((subscriber) => { + const requestBody = JSON.stringify({ + messages: recentMessages, + model: model.id, + stream: true, + ...model.parameters, + }) + fetch(inferenceUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Accept': model.parameters.stream ? 'text/event-stream' : 'application/json', + }, + body: requestBody, + signal: controller?.signal, + }) + .then(async (response) => { + if (model.parameters.stream === false) { + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') + } else { + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' + + while (true && reader) { + const { done, value } = await reader.read() + if (done) { + break + } + const text = decoder.decode(value) + const lines = text.trim().split('\n') + for (const line of lines) { + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') + } + subscriber.next(content) + } + } + } + } + subscriber.complete() + }) + .catch((err) => subscriber.error(err)) + }) +} diff --git a/core/src/extensions/ai-engines/index.ts b/core/src/extensions/ai-engines/index.ts new file mode 100644 index 000000000..f4da62a7c --- /dev/null +++ b/core/src/extensions/ai-engines/index.ts @@ -0,0 +1,3 @@ +export * from './AIEngine' +export * from './OAIEngine' +export * from './LocalOAIEngine' diff --git a/core/src/extensions/index.ts b/core/src/extensions/index.ts index c6834482c..c049f3b3a 100644 --- a/core/src/extensions/index.ts +++ b/core/src/extensions/index.ts @@ -28,3 +28,8 @@ export { ModelExtension } from './model' * Hugging Face extension for converting HF models to GGUF. */ export { HuggingFaceExtension } from './huggingface' + +/** + * Base AI Engines. + */ +export * from './ai-engines' diff --git a/core/src/extensions/model.ts b/core/src/extensions/model.ts index 79202398b..33eec0afc 100644 --- a/core/src/extensions/model.ts +++ b/core/src/extensions/model.ts @@ -1,5 +1,5 @@ import { BaseExtension, ExtensionTypeEnum } from '../extension' -import { ImportingModel, Model, ModelInterface, OptionType } from '../index' +import { GpuSetting, ImportingModel, Model, ModelInterface, OptionType } from '../index' /** * Model extension for managing models. @@ -14,6 +14,7 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter abstract downloadModel( model: Model, + gpuSettings?: GpuSetting, network?: { proxy: string; ignoreSSL?: boolean } ): Promise abstract cancelModelDownload(modelId: string): Promise diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts index ba193f0f4..8d61580fc 100644 --- a/core/src/extensions/monitoring.ts +++ b/core/src/extensions/monitoring.ts @@ -1,5 +1,5 @@ import { BaseExtension, ExtensionTypeEnum } from '../extension' -import { MonitoringInterface } from '../index' +import { GpuSetting, MonitoringInterface } from '../index' /** * Monitoring extension for system monitoring. @@ -13,6 +13,7 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit return ExtensionTypeEnum.SystemMonitoring } + abstract getGpuSetting(): Promise abstract getResourcesInfo(): Promise abstract getCurrentLoad(): Promise } diff --git a/core/src/node/api/processors/download.ts b/core/src/node/api/processors/download.ts index 4ddeff160..8e8e08f2f 100644 --- a/core/src/node/api/processors/download.ts +++ b/core/src/node/api/processors/download.ts @@ -5,7 +5,7 @@ import { getJanDataFolderPath } from '../../helper' import { DownloadManager } from '../../helper/download' import { createWriteStream, renameSync } from 'fs' import { Processor } from './Processor' -import { DownloadState } from '../../../types' +import { DownloadRequest, DownloadState, NetworkConfig } from '../../../types' export class Downloader implements Processor { observer?: Function @@ -20,24 +20,27 @@ export class Downloader implements Processor { return func(this.observer, ...args) } - downloadFile(observer: any, url: string, localPath: string, network: any) { + downloadFile(observer: any, downloadRequest: DownloadRequest, network?: NetworkConfig) { const request = require('request') const progress = require('request-progress') const strictSSL = !network?.ignoreSSL const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined + + const { localPath, url } = downloadRequest + let normalizedPath = localPath if (typeof localPath === 'string') { - localPath = normalizeFilePath(localPath) + normalizedPath = normalizeFilePath(localPath) } - const array = localPath.split(sep) + const array = normalizedPath.split(sep) const fileName = array.pop() ?? '' const modelId = array.pop() ?? '' - const destination = resolve(getJanDataFolderPath(), localPath) + const destination = resolve(getJanDataFolderPath(), normalizedPath) const rq = request({ url, strictSSL, proxy }) // Put request to download manager instance - DownloadManager.instance.setRequest(localPath, rq) + DownloadManager.instance.setRequest(normalizedPath, rq) // Downloading file to a temp file first const downloadingTempFile = `${destination}.download` @@ -56,16 +59,25 @@ export class Downloader implements Processor { total: 0, transferred: 0, }, + children: [], downloadState: 'downloading', + extensionId: downloadRequest.extensionId, + downloadType: downloadRequest.downloadType, + localPath: normalizedPath, } DownloadManager.instance.downloadProgressMap[modelId] = initialDownloadState + if (downloadRequest.downloadType === 'extension') { + observer?.(DownloadEvent.onFileDownloadUpdate, initialDownloadState) + } + progress(rq, {}) .on('progress', (state: any) => { + const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] const downloadState: DownloadState = { + ...currentDownloadState, ...state, - modelId, - fileName, + fileName: fileName, downloadState: 'downloading', } console.debug('progress: ', downloadState) @@ -76,22 +88,22 @@ export class Downloader implements Processor { const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] const downloadState: DownloadState = { ...currentDownloadState, + fileName: fileName, error: error.message, downloadState: 'error', } - if (currentDownloadState) { - DownloadManager.instance.downloadProgressMap[modelId] = downloadState - } observer?.(DownloadEvent.onFileDownloadError, downloadState) + DownloadManager.instance.downloadProgressMap[modelId] = downloadState }) .on('end', () => { const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] - if (currentDownloadState && DownloadManager.instance.networkRequests[localPath]) { + if (currentDownloadState && DownloadManager.instance.networkRequests[normalizedPath]) { // Finished downloading, rename temp file to actual file renameSync(downloadingTempFile, destination) const downloadState: DownloadState = { ...currentDownloadState, + fileName: fileName, downloadState: 'end', } observer?.(DownloadEvent.onFileDownloadSuccess, downloadState) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index 7001c0c76..6b9bbb3a8 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -1,7 +1,16 @@ -import fs from 'fs' +import { + existsSync, + readdirSync, + readFileSync, + writeFileSync, + mkdirSync, + appendFileSync, + createWriteStream, + rmdirSync, +} from 'fs' import { JanApiRouteConfiguration, RouteConfiguration } from './configuration' import { join } from 'path' -import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../index' +import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../types' import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper' import { DEFAULT_CHAT_COMPLETION_URL } from './consts' @@ -9,12 +18,12 @@ import { DEFAULT_CHAT_COMPLETION_URL } from './consts' export const getBuilder = async (configuration: RouteConfiguration) => { const directoryPath = join(getJanDataFolderPath(), configuration.dirName) try { - if (!fs.existsSync(directoryPath)) { + if (!existsSync(directoryPath)) { console.debug('model folder not found') return [] } - const files: string[] = fs.readdirSync(directoryPath) + const files: string[] = readdirSync(directoryPath) const allDirectories: string[] = [] for (const file of files) { @@ -46,8 +55,8 @@ export const getBuilder = async (configuration: RouteConfiguration) => { } const readModelMetadata = (path: string): string | undefined => { - if (fs.existsSync(path)) { - return fs.readFileSync(path, 'utf-8') + if (existsSync(path)) { + return readFileSync(path, 'utf-8') } else { return undefined } @@ -81,7 +90,7 @@ export const deleteBuilder = async (configuration: RouteConfiguration, id: strin } const objectPath = join(directoryPath, id) - fs.rmdirSync(objectPath, { recursive: true }) + rmdirSync(objectPath, { recursive: true }) return { id: id, object: configuration.delete.object, @@ -96,20 +105,19 @@ export const getMessages = async (threadId: string): Promise => const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId) const messageFile = 'messages.jsonl' try { - const files: string[] = fs.readdirSync(threadDirPath) + const files: string[] = readdirSync(threadDirPath) if (!files.includes(messageFile)) { console.error(`${threadDirPath} not contains message file`) return [] } const messageFilePath = join(threadDirPath, messageFile) - if (!fs.existsSync(messageFilePath)) { + if (!existsSync(messageFilePath)) { console.debug('message file not found') return [] } - const lines = fs - .readFileSync(messageFilePath, 'utf-8') + const lines = readFileSync(messageFilePath, 'utf-8') .toString() .split('\n') .filter((line: any) => line !== '') @@ -157,11 +165,11 @@ export const createThread = async (thread: any) => { const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id) const threadJsonPath = join(threadDirPath, threadMetadataFileName) - if (!fs.existsSync(threadDirPath)) { - fs.mkdirSync(threadDirPath) + if (!existsSync(threadDirPath)) { + mkdirSync(threadDirPath) } - await fs.writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) + await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) return updatedThread } catch (err) { return { @@ -191,7 +199,7 @@ export const updateThread = async (threadId: string, thread: any) => { const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id) const threadJsonPath = join(threadDirPath, threadMetadataFileName) - await fs.writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) + await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2)) return updatedThread } catch (err) { return { @@ -233,10 +241,10 @@ export const createMessage = async (threadId: string, message: any) => { const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId) const threadMessagePath = join(threadDirPath, threadMessagesFileName) - if (!fs.existsSync(threadDirPath)) { - fs.mkdirSync(threadDirPath) + if (!existsSync(threadDirPath)) { + mkdirSync(threadDirPath) } - fs.appendFileSync(threadMessagePath, JSON.stringify(threadMessage) + '\n') + appendFileSync(threadMessagePath, JSON.stringify(threadMessage) + '\n') return threadMessage } catch (err) { return { @@ -259,8 +267,8 @@ export const downloadModel = async ( } const directoryPath = join(getJanDataFolderPath(), 'models', modelId) - if (!fs.existsSync(directoryPath)) { - fs.mkdirSync(directoryPath) + if (!existsSync(directoryPath)) { + mkdirSync(directoryPath) } // path to model binary @@ -281,7 +289,7 @@ export const downloadModel = async ( .on('end', function () { console.debug('end') }) - .pipe(fs.createWriteStream(modelBinaryPath)) + .pipe(createWriteStream(modelBinaryPath)) } return { diff --git a/core/src/types/file/index.ts b/core/src/types/file/index.ts index cc7274a28..d941987ef 100644 --- a/core/src/types/file/index.ts +++ b/core/src/types/file/index.ts @@ -4,16 +4,43 @@ export type FileStat = { } export type DownloadState = { - modelId: string + modelId: string // TODO: change to download id fileName: string time: DownloadTime speed: number - percent: number + percent: number size: DownloadSize - children?: DownloadState[] - error?: string downloadState: 'downloading' | 'error' | 'end' + children?: DownloadState[] + + error?: string + extensionId?: string + downloadType?: DownloadType + localPath?: string +} + +export type DownloadType = 'model' | 'extension' + +export type DownloadRequest = { + /** + * The URL to download the file from. + */ + url: string + + /** + * The local path to save the file to. + */ + localPath: string + + /** + * The extension ID of the extension that initiated the download. + * + * Can be extension name. + */ + extensionId?: string + + downloadType?: DownloadType } type DownloadTime = { diff --git a/core/src/types/miscellaneous/fileDownloadRequest.ts b/core/src/types/miscellaneous/fileDownloadRequest.ts new file mode 100644 index 000000000..83131aa71 --- /dev/null +++ b/core/src/types/miscellaneous/fileDownloadRequest.ts @@ -0,0 +1,8 @@ +export type FileDownloadRequest = { + downloadId: string + url: string + localPath: string + fileName: string + displayName: string + metadata: Record +} diff --git a/core/src/types/miscellaneous/index.ts b/core/src/types/miscellaneous/index.ts index e9c205a73..b4ef68ab6 100644 --- a/core/src/types/miscellaneous/index.ts +++ b/core/src/types/miscellaneous/index.ts @@ -1,3 +1,5 @@ export * from './systemResourceInfo' export * from './promptTemplate' export * from './appUpdate' +export * from './fileDownloadRequest' +export * from './networkConfig' \ No newline at end of file diff --git a/core/src/types/miscellaneous/networkConfig.ts b/core/src/types/miscellaneous/networkConfig.ts new file mode 100644 index 000000000..2d27f4223 --- /dev/null +++ b/core/src/types/miscellaneous/networkConfig.ts @@ -0,0 +1,4 @@ +export type NetworkConfig = { + proxy?: string + ignoreSSL?: boolean +} diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts index 1472cda47..f7dd4a82b 100644 --- a/core/src/types/miscellaneous/systemResourceInfo.ts +++ b/core/src/types/miscellaneous/systemResourceInfo.ts @@ -2,3 +2,31 @@ export type SystemResourceInfo = { numCpuPhysicalCore: number memAvailable: number } + +export type RunMode = 'cpu' | 'gpu' + +export type GpuSetting = { + notify: boolean + run_mode: RunMode + nvidia_driver: { + exist: boolean + version: string + } + cuda: { + exist: boolean + version: string + } + gpus: GpuSettingInfo[] + gpu_highest_vram: string + gpus_in_use: string[] + is_initial: boolean + // TODO: This needs to be set based on user toggle in settings + vulkan: boolean +} + +export type GpuSettingInfo = { + id: string + vram: string + name: string + arch?: string +} diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts index 11d3e0526..74568686b 100644 --- a/core/src/types/model/modelEntity.ts +++ b/core/src/types/model/modelEntity.ts @@ -19,6 +19,7 @@ export enum InferenceEngine { nitro = 'nitro', openai = 'openai', triton_trtllm = 'triton_trtllm', + nitro_tensorrt_llm = 'nitro-tensorrt-llm', tool_retrieval_enabled = 'tool_retrieval_enabled', } diff --git a/core/src/types/model/modelInterface.ts b/core/src/types/model/modelInterface.ts index 93d5867ee..639c7c8d3 100644 --- a/core/src/types/model/modelInterface.ts +++ b/core/src/types/model/modelInterface.ts @@ -1,3 +1,4 @@ +import { GpuSetting } from '../miscellaneous' import { Model } from './modelEntity' /** @@ -10,7 +11,11 @@ export interface ModelInterface { * @param network - Optional object to specify proxy/whether to ignore SSL certificates. * @returns A Promise that resolves when the model has been downloaded. */ - downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise + downloadModel( + model: Model, + gpuSettings?: GpuSetting, + network?: { ignoreSSL?: boolean; proxy?: string } + ): Promise /** * Cancels the download of a specific model. diff --git a/core/src/types/monitoring/index.ts b/core/src/types/monitoring/index.ts index 5828dae8b..b96c518fd 100644 --- a/core/src/types/monitoring/index.ts +++ b/core/src/types/monitoring/index.ts @@ -1 +1,2 @@ export * from './monitoringInterface' +export * from './resourceInfo' diff --git a/core/src/types/monitoring/resourceInfo.ts b/core/src/types/monitoring/resourceInfo.ts new file mode 100644 index 000000000..b19da5462 --- /dev/null +++ b/core/src/types/monitoring/resourceInfo.ts @@ -0,0 +1,6 @@ +export type ResourceInfo = { + mem: { + totalMemory: number + usedMemory: number + } +} diff --git a/core/tsconfig.json b/core/tsconfig.json index b112079d2..daeb7eeff 100644 --- a/core/tsconfig.json +++ b/core/tsconfig.json @@ -13,7 +13,7 @@ "declarationDir": "dist/types", "outDir": "dist/lib", "importHelpers": true, - "types": ["@types/jest"] + "types": ["@types/jest"], }, - "include": ["src"] + "include": ["src"], } diff --git a/extensions/huggingface-extension/src/index.ts b/extensions/huggingface-extension/src/index.ts index d8f755080..88292ce58 100644 --- a/extensions/huggingface-extension/src/index.ts +++ b/extensions/huggingface-extension/src/index.ts @@ -13,6 +13,7 @@ import { events, DownloadEvent, log, + DownloadRequest, } from '@janhq/core' import { ggufMetadata } from 'hyllama' @@ -148,7 +149,11 @@ export default class JanHuggingFaceExtension extends HuggingFaceExtension { if (this.interrupted) return if (!(await fs.existsSync(localPath))) { - downloadFile(url, localPath, network) + const downloadRequest: DownloadRequest = { + url, + localPath, + } + downloadFile(downloadRequest, network) filePaths.push(filePath) } } diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat index 2ef3165c1..bb8c4ffdc 100644 --- a/extensions/inference-nitro-extension/download.bat +++ b/extensions/inference-nitro-extension/download.bat @@ -1,3 +1,3 @@ @echo off set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json index e6365ad92..dd5798764 100644 --- a/extensions/inference-nitro-extension/package.json +++ b/extensions/inference-nitro-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro && download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro", "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", "downloadnitro:win32": "download.bat", "downloadnitro": "run-script-os", diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 979b4cfac..70244a5d9 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -108,9 +108,6 @@ export default class JanInferenceNitroExtension extends InferenceExtension { events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped() ) - - // Attempt to fetch nvidia info - await executeOnMain(NODE, 'updateNvidiaInfo', {}) } /** diff --git a/extensions/inference-nitro-extension/src/node/accelerator.ts b/extensions/inference-nitro-extension/src/node/accelerator.ts deleted file mode 100644 index 1ffdbc5bd..000000000 --- a/extensions/inference-nitro-extension/src/node/accelerator.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { writeFileSync, existsSync, readFileSync } from 'fs' -import { exec, spawn } from 'child_process' -import path from 'path' -import { getJanDataFolderPath, log } from '@janhq/core/node' - -/** - * Default GPU settings - * TODO: This needs to be refactored to support multiple accelerators - **/ -const DEFALT_SETTINGS = { - notify: true, - run_mode: 'cpu', - nvidia_driver: { - exist: false, - version: '', - }, - cuda: { - exist: false, - version: '', - }, - gpus: [], - gpu_highest_vram: '', - gpus_in_use: [], - is_initial: true, - // TODO: This needs to be set based on user toggle in settings - vulkan: false -} - -/** - * Path to the settings file - **/ -export const GPU_INFO_FILE = path.join( - getJanDataFolderPath(), - 'settings', - 'settings.json' -) - -/** - * Current nitro process - */ -let nitroProcessInfo: NitroProcessInfo | undefined = undefined - -/** - * Nitro process info - */ -export interface NitroProcessInfo { - isRunning: boolean -} - -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export async function updateNvidiaInfo() { - if (process.platform !== 'darwin') { - let data - try { - data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - } catch (error) { - data = DEFALT_SETTINGS - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - } - updateNvidiaDriverInfo() - updateGpuInfo() - } -} - -/** - * Retrieve current nitro process - */ -export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { - nitroProcessInfo = { - isRunning: subprocess != null, - } - return nitroProcessInfo -} - -/** - * Validate nvidia and cuda for linux and windows - */ -export async function updateNvidiaDriverInfo(): Promise { - exec( - 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', - (error, stdout) => { - let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - - if (!error) { - const firstLine = stdout.split('\n')[0].trim() - data['nvidia_driver'].exist = true - data['nvidia_driver'].version = firstLine - } else { - data['nvidia_driver'].exist = false - } - - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - Promise.resolve() - } - ) -} - -/** - * Check if file exists in paths - */ -export function checkFileExistenceInPaths( - file: string, - paths: string[] -): boolean { - return paths.some((p) => existsSync(path.join(p, file))) -} - -/** - * Validate cuda for linux and windows - */ -export function updateCudaExistence( - data: Record = DEFALT_SETTINGS -): Record { - let filesCuda12: string[] - let filesCuda11: string[] - let paths: string[] - let cudaVersion: string = '' - - if (process.platform === 'win32') { - filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] - filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] - } else { - filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] - filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : [] - paths.push('/usr/lib/x86_64-linux-gnu/') - } - - let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ) - - if (!cudaExists) { - cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ) - if (cudaExists) { - cudaVersion = '11' - } - } else { - cudaVersion = '12' - } - - data['cuda'].exist = cudaExists - data['cuda'].version = cudaVersion - console.debug(data['is_initial'], data['gpus_in_use']) - if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) { - data.run_mode = 'gpu' - } - data.is_initial = false - return data -} - -/** - * Get GPU information - */ -export async function updateGpuInfo(): Promise { - let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - - // Cuda - if (data['vulkan'] === true) { - // Vulkan - exec( - process.platform === 'win32' - ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` - : `${__dirname}/../bin/vulkaninfo --summary`, - (error, stdout) => { - if (!error) { - const output = stdout.toString() - log(output) - const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g - - let gpus = [] - let match - while ((match = gpuRegex.exec(output)) !== null) { - const id = match[1] - const name = match[2] - gpus.push({ id, vram: 0, name }) - } - data.gpus = gpus - - if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { - data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] - } - - data = updateCudaExistence(data) - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - } - Promise.resolve() - } - ) - } else { - exec( - 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', - (error, stdout) => { - if (!error) { - log(stdout) - // Get GPU info and gpu has higher memory first - let highestVram = 0 - let highestVramId = '0' - let gpus = stdout - .trim() - .split('\n') - .map((line) => { - let [id, vram, name] = line.split(', ') - vram = vram.replace(/\r/g, '') - if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram) - highestVramId = id - } - return { id, vram, name } - }) - - data.gpus = gpus - data.gpu_highest_vram = highestVramId - } else { - data.gpus = [] - data.gpu_highest_vram = '' - } - - if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { - data.gpus_in_use = [data['gpu_highest_vram']] - } - - data = updateCudaExistence(data) - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - Promise.resolve() - } - ) - } -} diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index f9a668507..8bcc75ae4 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -1,12 +1,19 @@ +import { getJanDataFolderPath } from '@janhq/core/node' import { readFileSync } from 'fs' import * as path from 'path' -import { GPU_INFO_FILE } from './accelerator' export interface NitroExecutableOptions { executablePath: string cudaVisibleDevices: string vkVisibleDevices: string } + +export const GPU_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) + /** * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts index 9b2684a6c..c57eb262d 100644 --- a/extensions/inference-nitro-extension/src/node/index.ts +++ b/extensions/inference-nitro-extension/src/node/index.ts @@ -4,7 +4,6 @@ import { ChildProcessWithoutNullStreams, spawn } from 'child_process' import tcpPortUsed from 'tcp-port-used' import fetchRT from 'fetch-retry' import { log, getSystemResourceInfo } from '@janhq/core/node' -import { getNitroProcessInfo, updateNvidiaInfo } from './accelerator' import { Model, InferenceEngine, @@ -385,11 +384,26 @@ function dispose() { killSubprocess() } +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean +} + +/** + * Retrieve current nitro process + */ +const getCurrentNitroProcessInfo = (): NitroProcessInfo => { + return { + isRunning: subprocess != null, + } +} + export default { runModel, stopModel, killSubprocess, dispose, - updateNvidiaInfo, - getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), + getCurrentNitroProcessInfo, } diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index fb1f26885..d05e7d07f 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -17,6 +17,8 @@ import { ImportingModel, LocalImportModelEvent, baseName, + GpuSetting, + DownloadRequest, } from '@janhq/core' import { extractFileName } from './helpers/path' @@ -29,10 +31,14 @@ export default class JanModelExtension extends ModelExtension { private static readonly _modelMetadataFileName = 'model.json' private static readonly _supportedModelFormat = '.gguf' private static readonly _incompletedModelFileName = '.download' - private static readonly _offlineInferenceEngine = InferenceEngine.nitro - + private static readonly _offlineInferenceEngine = [ + InferenceEngine.nitro, + InferenceEngine.nitro_tensorrt_llm, + ] + private static readonly _tensorRtEngineFormat = '.engine' private static readonly _configDirName = 'config' private static readonly _defaultModelFileName = 'default-model.json' + private static readonly _supportedGpuArch = ['turing', 'ampere', 'ada'] /** * Called when the extension is loaded. @@ -89,12 +95,52 @@ export default class JanModelExtension extends ModelExtension { */ async downloadModel( model: Model, + gpuSettings?: GpuSetting, network?: { ignoreSSL?: boolean; proxy?: string } ): Promise { // create corresponding directory const modelDirPath = await joinPath([JanModelExtension._homeDir, model.id]) if (!(await fs.existsSync(modelDirPath))) await fs.mkdirSync(modelDirPath) + if (model.engine === InferenceEngine.nitro_tensorrt_llm) { + if (!gpuSettings || gpuSettings.gpus.length === 0) { + console.error('No GPU found. Please check your GPU setting.') + return + } + const firstGpu = gpuSettings.gpus[0] + if (!firstGpu.name.toLowerCase().includes('nvidia')) { + console.error('No Nvidia GPU found. Please check your GPU setting.') + return + } + const gpuArch = firstGpu.arch + if (gpuArch === undefined) { + console.error( + 'No GPU architecture found. Please check your GPU setting.' + ) + return + } + + if (!JanModelExtension._supportedGpuArch.includes(gpuArch)) { + console.error( + `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` + ) + return + } + + const os = 'windows' // TODO: remove this hard coded value + + const newSources = model.sources.map((source) => { + const newSource = { ...source } + newSource.url = newSource.url + .replace(//g, os) + .replace(//g, gpuArch) + return newSource + }) + model.sources = newSources + } + + console.debug(`Download sources: ${JSON.stringify(model.sources)}`) + if (model.sources.length > 1) { // path to model binaries for (const source of model.sources) { @@ -105,8 +151,11 @@ export default class JanModelExtension extends ModelExtension { if (source.filename) { path = await joinPath([modelDirPath, source.filename]) } - - downloadFile(source.url, path, network) + const downloadRequest: DownloadRequest = { + url: source.url, + localPath: path, + } + downloadFile(downloadRequest, network) } // TODO: handle multiple binaries for web later } else { @@ -115,7 +164,11 @@ export default class JanModelExtension extends ModelExtension { JanModelExtension._supportedModelFormat ) const path = await joinPath([modelDirPath, fileName]) - downloadFile(model.sources[0]?.url, path, network) + const downloadRequest: DownloadRequest = { + url: model.sources[0]?.url, + localPath: path, + } + downloadFile(downloadRequest, network) if (window && window.core?.api && window.core.api.baseApiUrl) { this.startPollingDownloadProgress(model.id) @@ -238,7 +291,7 @@ export default class JanModelExtension extends ModelExtension { async getDownloadedModels(): Promise { return await this.getModelsMetadata( async (modelDir: string, model: Model) => { - if (model.engine !== JanModelExtension._offlineInferenceEngine) + if (!JanModelExtension._offlineInferenceEngine.includes(model.engine)) return true // model binaries (sources) are absolute path & exist @@ -247,22 +300,32 @@ export default class JanModelExtension extends ModelExtension { ) if (existFiles.every((exist) => exist)) return true - return await fs + const result = await fs .readdirSync(await joinPath([JanModelExtension._homeDir, modelDir])) .then((files: string[]) => { // Model binary exists in the directory // Model binary name can match model ID or be a .gguf file and not be an incompleted model file return ( files.includes(modelDir) || - files.filter( - (file) => + files.filter((file) => { + if ( + file.endsWith(JanModelExtension._incompletedModelFileName) + ) { + return false + } + return ( file .toLowerCase() - .includes(JanModelExtension._supportedModelFormat) && - !file.endsWith(JanModelExtension._incompletedModelFileName) - )?.length >= model.sources.length + .includes(JanModelExtension._supportedModelFormat) || + file + .toLowerCase() + .includes(JanModelExtension._tensorRtEngineFormat) + ) + })?.length > 0 // TODO: NamH find better way (can use basename to check the file name with source url) ) }) + + return result } ) } diff --git a/extensions/monitoring-extension/bin/.gitkeep b/extensions/monitoring-extension/bin/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/extensions/monitoring-extension/download.bat b/extensions/monitoring-extension/download.bat new file mode 100644 index 000000000..f1cf8b7ea --- /dev/null +++ b/extensions/monitoring-extension/download.bat @@ -0,0 +1,2 @@ +@echo off +.\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin \ No newline at end of file diff --git a/extensions/monitoring-extension/package.json b/extensions/monitoring-extension/package.json index 582f7cd7b..73d28ab37 100644 --- a/extensions/monitoring-extension/package.json +++ b/extensions/monitoring-extension/package.json @@ -3,21 +3,40 @@ "version": "1.0.10", "description": "This extension provides system health and OS level data", "main": "dist/index.js", - "module": "dist/module.js", + "node": "dist/node/index.cjs.js", "author": "Jan ", "license": "AGPL-3.0", "scripts": { - "build": "tsc -b . && webpack --config webpack.config.js", + "build": "tsc --module commonjs && rollup -c rollup.config.ts && npm run download-artifacts", + "download-artifacts": "run-script-os && cpx \"bin/**\" \"dist/bin\"", + "download-artifacts:darwin": "echo 'No artifacts to download for darwin'", + "download-artifacts:win32": "download.bat", + "download-artifacts:linux": "download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/node/index.cjs.js" + }, "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^15.2.3", + "@types/node": "^20.11.4", + "@types/node-os-utils": "^1.3.4", + "run-script-os": "^1.1.6", + "cpx": "^1.5.0", "rimraf": "^3.0.2", - "webpack": "^5.88.2", - "webpack-cli": "^5.1.4", - "ts-loader": "^9.5.0" + "rollup": "^2.38.5", + "rollup-plugin-define": "^1.0.1", + "rollup-plugin-sourcemaps": "^0.6.3", + "rollup-plugin-typescript2": "^0.36.0", + "typescript": "^5.3.3", + "download-cli": "^1.1.1" }, "dependencies": { "@janhq/core": "file:../../core", + "@rollup/plugin-replace": "^5.0.5", "node-os-utils": "^1.3.7" }, "files": [ diff --git a/extensions/monitoring-extension/rollup.config.ts b/extensions/monitoring-extension/rollup.config.ts new file mode 100644 index 000000000..1b7a40bad --- /dev/null +++ b/extensions/monitoring-extension/rollup.config.ts @@ -0,0 +1,68 @@ +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') + +export default [ + { + input: `src/index.ts`, + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [], + watch: { + include: 'src/**', + }, + plugins: [ + replace({ + NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + }), + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Compile TypeScript files + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: ['.js', '.ts', '.svelte'], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, + { + input: `src/node/index.ts`, + output: [ + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, + ], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: ['@janhq/core/node'], + watch: { + include: 'src/node/**', + }, + plugins: [ + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: ['.ts', '.js', '.json'], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, +] diff --git a/extensions/monitoring-extension/src/@types/global.d.ts b/extensions/monitoring-extension/src/@types/global.d.ts index 8106353cf..dfa96a0b1 100644 --- a/extensions/monitoring-extension/src/@types/global.d.ts +++ b/extensions/monitoring-extension/src/@types/global.d.ts @@ -1 +1,18 @@ -declare const MODULE: string +declare const NODE: string + +type CpuGpuInfo = { + cpu: { + usage: number + } + gpu: GpuInfo[] +} + +type GpuInfo = { + id: string + name: string + temperature: string + utilization: string + memoryTotal: string + memoryFree: string + memoryUtilization: string +} diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index ce9b2fc14..c7f53455d 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,4 +1,4 @@ -import { MonitoringExtension, executeOnMain } from '@janhq/core' +import { GpuSetting, MonitoringExtension, executeOnMain } from '@janhq/core' /** * JanMonitoringExtension is a extension that provides system monitoring functionality. @@ -8,19 +8,30 @@ export default class JanMonitoringExtension extends MonitoringExtension { /** * Called when the extension is loaded. */ - async onLoad() {} + async onLoad() { + // Attempt to fetch nvidia info + await executeOnMain(NODE, 'updateNvidiaInfo') + } /** * Called when the extension is unloaded. */ onUnload(): void {} + /** + * Returns the GPU configuration. + * @returns A Promise that resolves to an object containing the GPU configuration. + */ + async getGpuSetting(): Promise { + return executeOnMain(NODE, 'getGpuConfig') + } + /** * Returns information about the system resources. * @returns A Promise that resolves to an object containing information about the system resources. */ getResourcesInfo(): Promise { - return executeOnMain(MODULE, 'getResourcesInfo') + return executeOnMain(NODE, 'getResourcesInfo') } /** @@ -28,6 +39,6 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the current system load. */ getCurrentLoad(): Promise { - return executeOnMain(MODULE, 'getCurrentLoad') + return executeOnMain(NODE, 'getCurrentLoad') } } diff --git a/extensions/monitoring-extension/src/module.ts b/extensions/monitoring-extension/src/module.ts deleted file mode 100644 index 27781a5d6..000000000 --- a/extensions/monitoring-extension/src/module.ts +++ /dev/null @@ -1,92 +0,0 @@ -const nodeOsUtils = require('node-os-utils') -const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath -const path = require('path') -const { readFileSync } = require('fs') -const exec = require('child_process').exec - -const NVIDIA_INFO_FILE = path.join( - getJanDataFolderPath(), - 'settings', - 'settings.json' -) - -const getResourcesInfo = () => - new Promise((resolve) => { - nodeOsUtils.mem.used().then((ramUsedInfo) => { - const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 - const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 - const response = { - mem: { - totalMemory, - usedMemory, - }, - } - resolve(response) - }) - }) - -const getCurrentLoad = () => - new Promise((resolve, reject) => { - nodeOsUtils.cpu.usage().then((cpuPercentage) => { - let data = { - run_mode: 'cpu', - gpus_in_use: [], - } - if (process.platform !== 'darwin') { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) - } - if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { - const gpuIds = data['gpus_in_use'].join(',') - if (gpuIds !== '' && data['vulkan'] !== true) { - exec( - `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, - (error, stdout, _) => { - if (error) { - console.error(`exec error: ${error}`) - reject(error) - return - } - const gpuInfo = stdout - .trim() - .split('\n') - .map((line) => { - const [ - id, - name, - temperature, - utilization, - memoryTotal, - memoryFree, - memoryUtilization, - ] = line.split(', ').map((item) => item.replace(/\r/g, '')) - return { - id, - name, - temperature, - utilization, - memoryTotal, - memoryFree, - memoryUtilization, - } - }) - resolve({ - cpu: { usage: cpuPercentage }, - gpu: gpuInfo, - }) - } - ) - } else { - // Handle the case where gpuIds is empty - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) - } - } else { - // Handle the case where run_mode is not 'gpu' or no GPUs are in use - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) - } - }) - }) - -module.exports = { - getResourcesInfo, - getCurrentLoad, -} diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts new file mode 100644 index 000000000..1d65704de --- /dev/null +++ b/extensions/monitoring-extension/src/node/index.ts @@ -0,0 +1,317 @@ +import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core' +import { getJanDataFolderPath, log } from '@janhq/core/node' +import { mem, cpu } from 'node-os-utils' +import { exec } from 'child_process' +import { writeFileSync, existsSync, readFileSync } from 'fs' +import path from 'path' + +/** + * Path to the settings file + **/ +export const GPU_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) + +/** + * Default GPU settings + * TODO: This needs to be refactored to support multiple accelerators + **/ +const DEFAULT_SETTINGS: GpuSetting = { + notify: true, + run_mode: 'cpu', + nvidia_driver: { + exist: false, + version: '', + }, + cuda: { + exist: false, + version: '', + }, + gpus: [], + gpu_highest_vram: '', + gpus_in_use: [], + is_initial: true, + // TODO: This needs to be set based on user toggle in settings + vulkan: false, +} + +export const getGpuConfig = async (): Promise => { + if (process.platform === 'darwin') return undefined + return JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) +} + +export const getResourcesInfo = async (): Promise => { + const ramUsedInfo = await mem.used() + const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 + const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 + + const resourceInfo: ResourceInfo = { + mem: { + totalMemory, + usedMemory, + }, + } + + return resourceInfo +} + +export const getCurrentLoad = () => + new Promise(async (resolve, reject) => { + const cpuPercentage = await cpu.usage() + let data = { + run_mode: 'cpu', + gpus_in_use: [], + } + + if (process.platform !== 'darwin') { + data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + } + + if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { + const gpuIds = data.gpus_in_use.join(',') + if (gpuIds !== '' && data['vulkan'] !== true) { + exec( + `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, + (error, stdout, _) => { + if (error) { + console.error(`exec error: ${error}`) + throw new Error(error.message) + } + const gpuInfo: GpuInfo[] = stdout + .trim() + .split('\n') + .map((line) => { + const [ + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + ] = line.split(', ').map((item) => item.replace(/\r/g, '')) + return { + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + } + }) + + resolve({ + cpu: { usage: cpuPercentage }, + gpu: gpuInfo, + }) + } + ) + } else { + // Handle the case where gpuIds is empty + resolve({ + cpu: { usage: cpuPercentage }, + gpu: [], + }) + } + } else { + // Handle the case where run_mode is not 'gpu' or no GPUs are in use + resolve({ + cpu: { usage: cpuPercentage }, + gpu: [], + }) + } + }) + +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export const updateNvidiaInfo = async () => { + // ignore if macos + if (process.platform === 'darwin') return + + try { + JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + } catch (error) { + writeFileSync(GPU_INFO_FILE, JSON.stringify(DEFAULT_SETTINGS, null, 2)) + } + + await updateNvidiaDriverInfo() + await updateGpuInfo() +} + +const updateNvidiaDriverInfo = async () => + new Promise((resolve, reject) => { + exec( + 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', + (error, stdout) => { + const data: GpuSetting = JSON.parse( + readFileSync(GPU_INFO_FILE, 'utf-8') + ) + + if (!error) { + const firstLine = stdout.split('\n')[0].trim() + data.nvidia_driver.exist = true + data.nvidia_driver.version = firstLine + } else { + data.nvidia_driver.exist = false + } + + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + resolve({}) + } + ) + }) + +const getGpuArch = (gpuName: string): string => { + if (!gpuName.toLowerCase().includes('nvidia')) return 'unknown' + + if (gpuName.includes('20')) return 'turing' + else if (gpuName.includes('30')) return 'ampere' + else if (gpuName.includes('40')) return 'ada' + else return 'unknown' +} + +const updateGpuInfo = async () => + new Promise((resolve, reject) => { + let data: GpuSetting = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + + // Cuda + if (data.vulkan === true) { + // Vulkan + exec( + process.platform === 'win32' + ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` + : `${__dirname}/../bin/vulkaninfo --summary`, + (error, stdout) => { + if (!error) { + const output = stdout.toString() + + log(output) + const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g + + const gpus: GpuSettingInfo[] = [] + let match + while ((match = gpuRegex.exec(output)) !== null) { + const id = match[1] + const name = match[2] + const arch = getGpuArch(name) + gpus.push({ id, vram: '0', name, arch }) + } + data.gpus = gpus + + if (!data.gpus_in_use || data.gpus_in_use.length === 0) { + data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + resolve({}) + } else { + reject(error) + } + } + ) + } else { + exec( + 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', + (error, stdout) => { + if (!error) { + log(stdout) + // Get GPU info and gpu has higher memory first + let highestVram = 0 + let highestVramId = '0' + const gpus: GpuSettingInfo[] = stdout + .trim() + .split('\n') + .map((line) => { + let [id, vram, name] = line.split(', ') + const arch = getGpuArch(name) + vram = vram.replace(/\r/g, '') + if (parseFloat(vram) > highestVram) { + highestVram = parseFloat(vram) + highestVramId = id + } + return { id, vram, name, arch } + }) + + data.gpus = gpus + data.gpu_highest_vram = highestVramId + } else { + data.gpus = [] + data.gpu_highest_vram = '' + } + + if (!data.gpus_in_use || data.gpus_in_use.length === 0) { + data.gpus_in_use = [data.gpu_highest_vram] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + resolve({}) + } + ) + } + }) + +/** + * Check if file exists in paths + */ +const checkFileExistenceInPaths = (file: string, paths: string[]): boolean => { + return paths.some((p) => existsSync(path.join(p, file))) +} + +/** + * Validate cuda for linux and windows + */ +const updateCudaExistence = ( + data: GpuSetting = DEFAULT_SETTINGS +): GpuSetting => { + let filesCuda12: string[] + let filesCuda11: string[] + let paths: string[] + let cudaVersion: string = '' + + if (process.platform === 'win32') { + filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] + filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] + } else { + filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] + filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path.delimiter) + : [] + paths.push('/usr/lib/x86_64-linux-gnu/') + } + + let cudaExists = filesCuda12.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + if (cudaExists) { + cudaVersion = '11' + } + } else { + cudaVersion = '12' + } + + data.cuda.exist = cudaExists + data.cuda.version = cudaVersion + + console.debug(data.is_initial, data.gpus_in_use) + + if (cudaExists && data.is_initial && data.gpus_in_use.length > 0) { + data.run_mode = 'gpu' + } + + data.is_initial = false + return data +} diff --git a/extensions/monitoring-extension/webpack.config.js b/extensions/monitoring-extension/webpack.config.js deleted file mode 100644 index c8c3a34f7..000000000 --- a/extensions/monitoring-extension/webpack.config.js +++ /dev/null @@ -1,35 +0,0 @@ -const path = require('path') -const webpack = require('webpack') -const packageJson = require('./package.json') - -module.exports = { - experiments: { outputModule: true }, - entry: './src/index.ts', // Adjust the entry point to match your project's main file - mode: 'production', - module: { - rules: [ - { - test: /\.tsx?$/, - use: 'ts-loader', - exclude: /node_modules/, - }, - ], - }, - output: { - filename: 'index.js', // Adjust the output file name as needed - path: path.resolve(__dirname, 'dist'), - library: { type: 'module' }, // Specify ESM output format - }, - plugins: [ - new webpack.DefinePlugin({ - MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - }), - ], - resolve: { - extensions: ['.ts', '.js'], - }, - optimization: { - minimize: false, - }, - // Add loaders and other configuration as needed for your project -} diff --git a/extensions/tensorrt-llm-extension/README.md b/extensions/tensorrt-llm-extension/README.md new file mode 100644 index 000000000..34a670516 --- /dev/null +++ b/extensions/tensorrt-llm-extension/README.md @@ -0,0 +1,79 @@ +# Tensorrt-LLM Extension + +Created using Jan extension example + +# Create a Jan Extension using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 + +## Create Your Own Extension + +To create your own extension, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your extension directory now + +## Update the Extension Metadata + +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your extension. + +## Update the Extension Code + +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your extension code: + +- Most Jan Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { events, MessageEvent, MessageRequest } from '@janhq/core' + + function onStart(): Promise { + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) + } + ``` + + For more information about the Jan Extension Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/tensorrt-llm-extension/models.json b/extensions/tensorrt-llm-extension/models.json new file mode 100644 index 000000000..bc6a78256 --- /dev/null +++ b/extensions/tensorrt-llm-extension/models.json @@ -0,0 +1,49 @@ +[ + { + "sources": [ + { + "filename": "config.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/config.json" + }, + { + "filename": "rank0.engine", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/rank0.engine" + }, + { + "filename": "tokenizer.model", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer.model" + }, + { + "filename": "special_tokens_map.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/special_tokens_map.json" + }, + { + "filename": "tokenizer.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer.json" + }, + { + "filename": "tokenizer_config.json", + "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer_config.json" + } + ], + "id": "llamacorn-1.1b-chat-fp16", + "object": "model", + "name": "LlamaCorn 1.1B Chat FP16", + "version": "1.0", + "description": "LlamaCorn is a refined version of TinyLlama-1.1B, optimized for conversational quality, running on consumer devices through TensorRT-LLM", + "format": "TensorRT-LLM", + "settings": { + "ctx_len": 2048 + }, + "parameters": { + "stream": true, + "max_tokens": 4096 + }, + "metadata": { + "author": "LLama", + "tags": ["TensorRT-LLM", "1B", "Finetuned"], + "size": 2151000000 + }, + "engine": "nitro-tensorrt-llm" + } +] diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json new file mode 100644 index 000000000..01ff3e2c6 --- /dev/null +++ b/extensions/tensorrt-llm-extension/package.json @@ -0,0 +1,75 @@ +{ + "name": "@janhq/tensorrt-llm-extension", + "version": "0.0.2", + "description": "Enables accelerated inference leveraging Nvidia's TensorRT-LLM for optimal GPU hardware optimizations. Compatible with models in TensorRT-LLM format. Requires Nvidia GPU driver and CUDA Toolkit installation.", + "main": "dist/index.js", + "node": "dist/node/index.cjs.js", + "author": "Jan ", + "license": "AGPL-3.0", + "config": { + "host": "127.0.0.1", + "port": "3928" + }, + "compatibility": { + "platform": [ + "win32", + "linux" + ], + "app": [ + "0.1.0" + ] + }, + "scripts": { + "build": "tsc --module commonjs && rollup -c rollup.config.ts", + "build:publish:win32": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:linux": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:darwin": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish": "run-script-os" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/node/index.cjs.js" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-replace": "^5.0.5", + "@types/node": "^20.11.4", + "@types/os-utils": "^0.0.4", + "@types/tcp-port-used": "^1.0.4", + "@types/decompress": "4.2.7", + "cpx": "^1.5.0", + "download-cli": "^1.1.1", + "rimraf": "^3.0.2", + "rollup": "^2.38.5", + "rollup-plugin-define": "^1.0.1", + "rollup-plugin-sourcemaps": "^0.6.3", + "rollup-plugin-typescript2": "^0.36.0", + "run-script-os": "^1.1.6", + "typescript": "^5.2.2" + }, + "dependencies": { + "@janhq/core": "file:../../core", + "decompress": "^4.2.1", + "fetch-retry": "^5.0.6", + "path-browserify": "^1.0.1", + "rxjs": "^7.8.1", + "tcp-port-used": "^1.0.2", + "ulid": "^2.3.0" + }, + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ], + "bundleDependencies": [ + "tcp-port-used", + "fetch-retry", + "decompress", + "@janhq/core" + ] +} diff --git a/extensions/tensorrt-llm-extension/rollup.config.ts b/extensions/tensorrt-llm-extension/rollup.config.ts new file mode 100644 index 000000000..33e45823b --- /dev/null +++ b/extensions/tensorrt-llm-extension/rollup.config.ts @@ -0,0 +1,73 @@ +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') + +export default [ + { + input: `src/index.ts`, + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], + watch: { + include: 'src/**', + }, + plugins: [ + replace({ + EXTENSION_NAME: JSON.stringify(packageJson.name), + TENSORRT_VERSION: JSON.stringify('0.1.5'), + DOWNLOAD_RUNNER_URL: + process.platform === 'darwin' || process.platform === 'win32' + ? JSON.stringify( + 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/windows-v/nitro-windows-v-amd64-tensorrt-llm-.tar.gz' + ) + : JSON.stringify( + 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/linux-v/nitro-linux-v-amd64-tensorrt-llm-.tar.gz' + ), + NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + INFERENCE_URL: JSON.stringify( + process.env.INFERENCE_URL || + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/v1/chat/completions` + ), + COMPATIBILITY: JSON.stringify(packageJson.compatibility), + }), + json(), + typescript({ useTsconfigDeclarationDir: true }), + commonjs(), + resolve({ + extensions: ['.js', '.ts', '.svelte'], + }), + sourceMaps(), + ], + }, + { + input: `src/node/index.ts`, + output: [ + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, + ], + external: ['@janhq/core/node'], + watch: { + include: 'src/node/**', + }, + plugins: [ + replace({ + LOAD_MODEL_URL: JSON.stringify( + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/tensorrtllm/loadmodel` + ), + TERMINATE_ENGINE_URL: JSON.stringify( + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/processmanager/destroy` + ), + ENGINE_HOST: JSON.stringify(packageJson.config?.host ?? '127.0.0.1'), + ENGINE_PORT: JSON.stringify(packageJson.config?.port ?? '3928'), + }), + json(), + typescript({ useTsconfigDeclarationDir: true }), + commonjs(), + resolve({ + extensions: ['.ts', '.js', '.json'], + }), + sourceMaps(), + ], + }, +] diff --git a/extensions/tensorrt-llm-extension/src/@types/global.d.ts b/extensions/tensorrt-llm-extension/src/@types/global.d.ts new file mode 100644 index 000000000..905e86380 --- /dev/null +++ b/extensions/tensorrt-llm-extension/src/@types/global.d.ts @@ -0,0 +1,10 @@ +declare const NODE: string +declare const INFERENCE_URL: string +declare const LOAD_MODEL_URL: string +declare const TERMINATE_ENGINE_URL: string +declare const ENGINE_HOST: string +declare const ENGINE_PORT: string +declare const DOWNLOAD_RUNNER_URL: string +declare const TENSORRT_VERSION: string +declare const COMPATIBILITY: object +declare const EXTENSION_NAME: string diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts new file mode 100644 index 000000000..076951c3f --- /dev/null +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -0,0 +1,147 @@ +/** + * @module tensorrt-llm-extension/src/index + */ + +import { + Compatibility, + DownloadEvent, + DownloadRequest, + DownloadState, + GpuSetting, + InstallationState, + Model, + baseName, + downloadFile, + events, + executeOnMain, + joinPath, + showToast, + systemInformations, + LocalOAIEngine, + fs, +} from '@janhq/core' +import models from '../models.json' + +/** + * TensorRTLLMExtension - Implementation of LocalOAIEngine + * @extends BaseOAILocalInferenceProvider + * Provide pre-populated models for TensorRTLLM + */ +export default class TensorRTLLMExtension extends LocalOAIEngine { + /** + * Override custom function name for loading and unloading model + * Which are implemented from node module + */ + override provider = 'nitro-tensorrt-llm' + override inferenceUrl = INFERENCE_URL + override nodeModule = NODE + + private supportedGpuArch = ['turing', 'ampere', 'ada'] + + compatibility() { + return COMPATIBILITY as unknown as Compatibility + } + /** + * models implemented by the extension + * define pre-populated models + */ + async models(): Promise { + if ((await this.installationState()) === 'Installed') + return models as unknown as Model[] + return [] + } + + override async install(): Promise { + const info = await systemInformations() + console.debug( + `TensorRTLLMExtension installing pre-requisites... ${JSON.stringify(info)}` + ) + const gpuSetting: GpuSetting | undefined = info.gpuSetting + if (gpuSetting === undefined || gpuSetting.gpus.length === 0) { + console.error('No GPU setting found. Please check your GPU setting.') + return + } + + // TODO: we only check for the first graphics card. Need to refactor this later. + const firstGpu = gpuSetting.gpus[0] + if (!firstGpu.name.toLowerCase().includes('nvidia')) { + console.error('No Nvidia GPU found. Please check your GPU setting.') + return + } + + if (firstGpu.arch === undefined) { + console.error('No GPU architecture found. Please check your GPU setting.') + return + } + + if (!this.supportedGpuArch.includes(firstGpu.arch)) { + console.error( + `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` + ) + return + } + + const binaryFolderPath = await executeOnMain( + this.nodeModule, + 'binaryFolder' + ) + if (!(await fs.existsSync(binaryFolderPath))) { + await fs.mkdirSync(binaryFolderPath) + } + + const placeholderUrl = DOWNLOAD_RUNNER_URL + const tensorrtVersion = TENSORRT_VERSION + + const url = placeholderUrl + .replace(//g, tensorrtVersion) + .replace(//g, firstGpu.arch) + + const tarball = await baseName(url) + + const tarballFullPath = await joinPath([binaryFolderPath, tarball]) + const downloadRequest: DownloadRequest = { + url, + localPath: tarballFullPath, + extensionId: EXTENSION_NAME, + downloadType: 'extension', + } + downloadFile(downloadRequest) + + // TODO: wrap this into a Promise + const onFileDownloadSuccess = async (state: DownloadState) => { + // if other download, ignore + if (state.fileName !== tarball) return + events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + await executeOnMain(this.nodeModule, 'decompressRunner', tarballFullPath) + events.emit(DownloadEvent.onFileUnzipSuccess, state) + + // Prepopulate models as soon as it's ready + this.prePopulateModels().then(() => { + showToast( + 'Extension installed successfully.', + 'New models are added to Model Hub.' + ) + }) + } + events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + } + + override async installationState(): Promise { + // For now, we just check the executable of nitro x tensor rt + const isNitroExecutableAvailable = await executeOnMain( + this.nodeModule, + 'isNitroExecutableAvailable' + ) + + return isNitroExecutableAvailable ? 'Installed' : 'NotInstalled' + } + + override onInferenceStopped() { + if (!this.isRunning) return + showToast( + 'Unable to Stop Inference', + 'The model does not support stopping inference.' + ) + return Promise.resolve() + } +} diff --git a/extensions/tensorrt-llm-extension/src/node/index.ts b/extensions/tensorrt-llm-extension/src/node/index.ts new file mode 100644 index 000000000..252468fc1 --- /dev/null +++ b/extensions/tensorrt-llm-extension/src/node/index.ts @@ -0,0 +1,191 @@ +import path from 'path' +import { ChildProcessWithoutNullStreams, spawn } from 'child_process' +import tcpPortUsed from 'tcp-port-used' +import fetchRT from 'fetch-retry' +import { log } from '@janhq/core/node' +import { existsSync } from 'fs' +import decompress from 'decompress' + +// Polyfill fetch with retry +const fetchRetry = fetchRT(fetch) + +/** + * The response object for model init operation. + */ +interface ModelLoadParams { + engine_path: string + ctx_len: number +} + +// The subprocess instance for Engine +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined + +/** + * Initializes a engine subprocess to load a machine learning model. + * @param params - The model load settings. + */ +async function loadModel(params: any): Promise<{ error: Error | undefined }> { + // modelFolder is the absolute path to the running model folder + // e.g. ~/jan/models/llama-2 + let modelFolder = params.modelFolder + + const settings: ModelLoadParams = { + engine_path: modelFolder, + ctx_len: params.model.settings.ctx_len ?? 2048, + } + return runEngineAndLoadModel(settings) +} + +/** + * Stops a Engine subprocess. + */ +function unloadModel(): Promise { + const controller = new AbortController() + setTimeout(() => controller.abort(), 5000) + debugLog(`Request to kill engine`) + + subprocess?.kill() + return fetch(TERMINATE_ENGINE_URL, { + method: 'DELETE', + signal: controller.signal, + }) + .then(() => { + subprocess = undefined + }) + .catch(() => {}) // Do nothing with this attempt + .then(() => tcpPortUsed.waitUntilFree(parseInt(ENGINE_PORT), 300, 5000)) // Wait for port available + .then(() => debugLog(`Engine process is terminated`)) + .catch((err) => { + debugLog( + `Could not kill running process on port ${ENGINE_PORT}. Might be another process running on the same port? ${err}` + ) + throw 'PORT_NOT_AVAILABLE' + }) +} +/** + * 1. Spawn engine process + * 2. Load model into engine subprocess + * @returns + */ +async function runEngineAndLoadModel(settings: ModelLoadParams) { + return unloadModel() + .then(runEngine) + .then(() => loadModelRequest(settings)) + .catch((err) => { + // TODO: Broadcast error so app could display proper error message + debugLog(`${err}`, 'Error') + return { error: err } + }) +} + +/** + * Loads a LLM model into the Engine subprocess by sending a HTTP POST request. + */ +function loadModelRequest( + settings: ModelLoadParams +): Promise<{ error: Error | undefined }> { + debugLog(`Loading model with params ${JSON.stringify(settings)}`) + return fetchRetry(LOAD_MODEL_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + }) + .then((res) => { + debugLog(`Load model success with response ${JSON.stringify(res)}`) + return Promise.resolve({ error: undefined }) + }) + .catch((err) => { + debugLog(`Load model failed with error ${err}`, 'Error') + return Promise.resolve({ error: err }) + }) +} + +/** + * Spawns engine subprocess. + */ +function runEngine(): Promise { + debugLog(`Spawning engine subprocess...`) + + return new Promise((resolve, reject) => { + // Current directory by default + let binaryFolder = path.join(__dirname, '..', 'bin') + // Binary path + const binary = path.join( + binaryFolder, + process.platform === 'win32' ? 'nitro.exe' : 'nitro' + ) + + const args: string[] = ['1', ENGINE_HOST, ENGINE_PORT] + // Execute the binary + debugLog(`Spawn nitro at path: ${binary}, and args: ${args}`) + subprocess = spawn(binary, args, { + cwd: binaryFolder, + env: { + ...process.env, + }, + }) + + // Handle subprocess output + subprocess.stdout.on('data', (data: any) => { + debugLog(`${data}`) + }) + + subprocess.stderr.on('data', (data: any) => { + debugLog(`${data}`) + }) + + subprocess.on('close', (code: any) => { + debugLog(`Engine exited with code: ${code}`) + subprocess = undefined + reject(`child process exited with code ${code}`) + }) + + tcpPortUsed.waitUntilUsed(parseInt(ENGINE_PORT), 300, 30000).then(() => { + debugLog(`Engine is ready`) + resolve() + }) + }) +} + +function debugLog(message: string, level: string = 'Debug') { + log(`[TENSORRT_LLM_NITRO]::${level}:${message}`) +} + +const binaryFolder = async (): Promise => { + return path.join(__dirname, '..', 'bin') +} + +const decompressRunner = async (zipPath: string) => { + const output = path.join(__dirname, '..', 'bin') + console.debug(`Decompressing ${zipPath} to ${output}...`) + try { + const files = await decompress(zipPath, output) + console.debug('Decompress finished!', files) + } catch (err) { + console.error(`Decompress ${zipPath} failed: ${err}`) + } +} + +const isNitroExecutableAvailable = async (): Promise => { + const binary = path.join( + __dirname, + '..', + 'bin', + process.platform === 'win32' ? 'nitro.exe' : 'nitro' + ) + + return existsSync(binary) +} + +export default { + binaryFolder, + decompressRunner, + loadModel, + unloadModel, + dispose: unloadModel, + isNitroExecutableAvailable, +} diff --git a/extensions/tensorrt-llm-extension/tsconfig.json b/extensions/tensorrt-llm-extension/tsconfig.json new file mode 100644 index 000000000..478a05728 --- /dev/null +++ b/extensions/tensorrt-llm-extension/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "moduleResolution": "node", + "target": "es5", + "module": "ES2020", + "lib": ["es2015", "es2016", "es2017", "dom"], + "strict": true, + "sourceMap": true, + "declaration": true, + "allowSyntheticDefaultImports": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declarationDir": "dist/types", + "outDir": "dist", + "importHelpers": true, + "resolveJsonModule": true, + "typeRoots": ["node_modules/@types"] + }, + "include": ["src"] +} diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index c05d26e51..ef7532063 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -73,8 +73,9 @@ const DropdownListSidebar = ({ const [copyId, setCopyId] = useState('') + // TODO: Update filter condition for the local model const localModel = downloadedModels.filter( - (model) => model.engine === InferenceEngine.nitro + (model) => model.engine !== InferenceEngine.openai ) const remoteModel = downloadedModels.filter( (model) => model.engine === InferenceEngine.openai diff --git a/web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx b/web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx new file mode 100644 index 000000000..d46764e88 --- /dev/null +++ b/web/containers/Layout/BottomBar/InstallingExtension/InstallingExtensionModal.tsx @@ -0,0 +1,87 @@ +import { useCallback, useEffect } from 'react' + +import { abortDownload } from '@janhq/core' +import { + Button, + Modal, + ModalContent, + ModalHeader, + ModalTitle, + Progress, +} from '@janhq/uikit' +import { atom, useAtom, useAtomValue } from 'jotai' + +import { + formatDownloadPercentage, + formatExtensionsName, +} from '@/utils/converter' + +import { + InstallingExtensionState, + installingExtensionAtom, +} from '@/helpers/atoms/Extension.atom' + +export const showInstallingExtensionModalAtom = atom(false) + +const InstallingExtensionModal: React.FC = () => { + const [showInstallingExtensionModal, setShowInstallingExtensionModal] = + useAtom(showInstallingExtensionModalAtom) + const installingExtensions = useAtomValue(installingExtensionAtom) + + useEffect(() => { + if (installingExtensions.length === 0) { + setShowInstallingExtensionModal(false) + } + }, [installingExtensions, setShowInstallingExtensionModal]) + + const onAbortInstallingExtensionClick = useCallback( + (item: InstallingExtensionState) => { + if (item.localPath) { + abortDownload(item.localPath) + } + }, + [] + ) + + return ( + setShowInstallingExtensionModal(false)} + > + + + Installing Extension + + {Object.values(installingExtensions).map((item) => ( +
+ +
+
+

+ {formatExtensionsName(item.extensionId)} +

+ {formatDownloadPercentage(item.percentage)} +
+ +
+
+ ))} + + + ) +} + +export default InstallingExtensionModal diff --git a/web/containers/Layout/BottomBar/InstallingExtension/index.tsx b/web/containers/Layout/BottomBar/InstallingExtension/index.tsx new file mode 100644 index 000000000..05e803881 --- /dev/null +++ b/web/containers/Layout/BottomBar/InstallingExtension/index.tsx @@ -0,0 +1,52 @@ +import { Fragment, useCallback } from 'react' + +import { Progress } from '@janhq/uikit' +import { useAtomValue, useSetAtom } from 'jotai' + +import { showInstallingExtensionModalAtom } from './InstallingExtensionModal' + +import { installingExtensionAtom } from '@/helpers/atoms/Extension.atom' + +const InstallingExtension: React.FC = () => { + const installingExtensions = useAtomValue(installingExtensionAtom) + const setShowInstallingExtensionModal = useSetAtom( + showInstallingExtensionModalAtom + ) + const shouldShowInstalling = installingExtensions.length > 0 + + let totalPercentage = 0 + let totalExtensions = 0 + for (const installation of installingExtensions) { + totalPercentage += installation.percentage + totalExtensions++ + } + const progress = (totalPercentage / totalExtensions) * 100 + + const onClick = useCallback(() => { + setShowInstallingExtensionModal(true) + }, [setShowInstallingExtensionModal]) + + return ( + + {shouldShowInstalling ? ( +
+

+ Installing Extension +

+ +
+ + + {progress.toFixed(2)}% + +
+
+ ) : null} + + ) +} + +export default InstallingExtension diff --git a/web/containers/Layout/BottomBar/index.tsx b/web/containers/Layout/BottomBar/index.tsx index 2373ac3d4..3683d23db 100644 --- a/web/containers/Layout/BottomBar/index.tsx +++ b/web/containers/Layout/BottomBar/index.tsx @@ -16,6 +16,7 @@ import ProgressBar from '@/containers/ProgressBar' import { appDownloadProgress } from '@/containers/Providers/Jotai' import ImportingModelState from './ImportingModelState' +import InstallingExtension from './InstallingExtension' import SystemMonitor from './SystemMonitor' import UpdatedFailedModal from './UpdateFailedModal' @@ -46,6 +47,7 @@ const BottomBar = () => { +
diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index 7e3ad38ab..fb08bc6ac 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -22,6 +22,8 @@ import ImportModelOptionModal from '@/screens/Settings/ImportModelOptionModal' import ImportingModelModal from '@/screens/Settings/ImportingModelModal' import SelectingModelModal from '@/screens/Settings/SelectingModelModal' +import InstallingExtensionModal from './BottomBar/InstallingExtension/InstallingExtensionModal' + import { mainViewStateAtom } from '@/helpers/atoms/App.atom' const BaseLayout = (props: PropsWithChildren) => { @@ -68,6 +70,7 @@ const BaseLayout = (props: PropsWithChildren) => { {importModelStage === 'IMPORTING_MODEL' && } {importModelStage === 'EDIT_MODEL_INFO' && } {importModelStage === 'CONFIRM_CANCEL' && } +
) } diff --git a/web/containers/Providers/EventListener.tsx b/web/containers/Providers/EventListener.tsx index bfc87917b..20fc6dde2 100644 --- a/web/containers/Providers/EventListener.tsx +++ b/web/containers/Providers/EventListener.tsx @@ -7,6 +7,10 @@ import { useSetAtom } from 'jotai' import { setDownloadStateAtom } from '@/hooks/useDownloadState' +import { formatExtensionsName } from '@/utils/converter' + +import { toaster } from '../Toast' + import AppUpdateListener from './AppUpdateListener' import ClipboardListener from './ClipboardListener' import EventHandler from './EventHandler' @@ -14,46 +18,89 @@ import EventHandler from './EventHandler' import ModelImportListener from './ModelImportListener' import QuickAskListener from './QuickAskListener' +import { + InstallingExtensionState, + removeInstallingExtensionAtom, + setInstallingExtensionAtom, +} from '@/helpers/atoms/Extension.atom' + const EventListenerWrapper = ({ children }: PropsWithChildren) => { const setDownloadState = useSetAtom(setDownloadStateAtom) + const setInstallingExtension = useSetAtom(setInstallingExtensionAtom) + const removeInstallingExtension = useSetAtom(removeInstallingExtensionAtom) const onFileDownloadUpdate = useCallback( async (state: DownloadState) => { console.debug('onFileDownloadUpdate', state) - setDownloadState(state) + if (state.downloadType === 'extension') { + const installingExtensionState: InstallingExtensionState = { + extensionId: state.extensionId!, + percentage: state.percent, + localPath: state.localPath, + } + setInstallingExtension(state.extensionId!, installingExtensionState) + } else { + setDownloadState(state) + } }, - [setDownloadState] + [setDownloadState, setInstallingExtension] ) const onFileDownloadError = useCallback( (state: DownloadState) => { console.debug('onFileDownloadError', state) - setDownloadState(state) + if (state.downloadType === 'extension') { + removeInstallingExtension(state.extensionId!) + } else { + setDownloadState(state) + } }, - [setDownloadState] + [setDownloadState, removeInstallingExtension] ) const onFileDownloadSuccess = useCallback( (state: DownloadState) => { console.debug('onFileDownloadSuccess', state) - setDownloadState(state) + if (state.downloadType !== 'extension') { + setDownloadState(state) + } }, [setDownloadState] ) + const onFileUnzipSuccess = useCallback( + (state: DownloadState) => { + console.debug('onFileUnzipSuccess', state) + toaster({ + title: 'Success', + description: `Install ${formatExtensionsName(state.extensionId!)} successfully.`, + type: 'success', + }) + removeInstallingExtension(state.extensionId!) + }, + [removeInstallingExtension] + ) + useEffect(() => { console.debug('EventListenerWrapper: registering event listeners...') events.on(DownloadEvent.onFileDownloadUpdate, onFileDownloadUpdate) events.on(DownloadEvent.onFileDownloadError, onFileDownloadError) events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + events.on(DownloadEvent.onFileUnzipSuccess, onFileUnzipSuccess) return () => { console.debug('EventListenerWrapper: unregistering event listeners...') events.off(DownloadEvent.onFileDownloadUpdate, onFileDownloadUpdate) events.off(DownloadEvent.onFileDownloadError, onFileDownloadError) events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + events.off(DownloadEvent.onFileUnzipSuccess, onFileUnzipSuccess) } - }, [onFileDownloadUpdate, onFileDownloadError, onFileDownloadSuccess]) + }, [ + onFileDownloadUpdate, + onFileDownloadError, + onFileDownloadSuccess, + onFileUnzipSuccess, + ]) return ( diff --git a/web/extension/ExtensionManager.ts b/web/extension/ExtensionManager.ts index 1259021f7..c976010c6 100644 --- a/web/extension/ExtensionManager.ts +++ b/web/extension/ExtensionManager.ts @@ -23,7 +23,9 @@ export class ExtensionManager { * @param type - The type of the extension to retrieve. * @returns The extension, if found. */ - get(type: ExtensionTypeEnum): T | undefined { + get( + type: ExtensionTypeEnum | string + ): T | undefined { return this.extensions.get(type) as T | undefined } diff --git a/web/helpers/atoms/Extension.atom.ts b/web/helpers/atoms/Extension.atom.ts new file mode 100644 index 000000000..7af755e35 --- /dev/null +++ b/web/helpers/atoms/Extension.atom.ts @@ -0,0 +1,40 @@ +import { atom } from 'jotai' + +type ExtensionId = string + +export type InstallingExtensionState = { + extensionId: ExtensionId + percentage: number + localPath?: string +} + +export const installingExtensionAtom = atom([]) + +export const setInstallingExtensionAtom = atom( + null, + (get, set, extensionId: string, state: InstallingExtensionState) => { + const current = get(installingExtensionAtom) + + const isExists = current.some((e) => e.extensionId === extensionId) + if (isExists) { + const newCurrent = current.map((e) => { + if (e.extensionId === extensionId) { + return state + } + return e + }) + set(installingExtensionAtom, newCurrent) + } else { + set(installingExtensionAtom, [...current, state]) + } + } +) + +export const removeInstallingExtensionAtom = atom( + null, + (get, set, extensionId: string) => { + const current = get(installingExtensionAtom) + const newCurrent = current.filter((e) => e.extensionId !== extensionId) + set(installingExtensionAtom, newCurrent) + } +) diff --git a/web/hooks/useActiveModel.ts b/web/hooks/useActiveModel.ts index 600e10783..e6c519f9f 100644 --- a/web/hooks/useActiveModel.ts +++ b/web/hooks/useActiveModel.ts @@ -40,6 +40,16 @@ export function useActiveModel() { console.debug(`Model ${modelId} is already initialized. Ignore..`) return } + + let model = downloadedModelsRef?.current.find((e) => e.id === modelId) + + // Switch between engines + if (model && activeModel && activeModel.engine !== model.engine) { + stopModel() + // TODO: Refactor inference provider would address this + await new Promise((res) => setTimeout(res, 1000)) + } + // TODO: incase we have multiple assistants, the configuration will be from assistant setLoadModelError(undefined) @@ -47,8 +57,6 @@ export function useActiveModel() { setStateModel({ state: 'start', loading: true, model: modelId }) - let model = downloadedModelsRef?.current.find((e) => e.id === modelId) - if (!model) { toaster({ title: `Model ${modelId} not found!`, diff --git a/web/hooks/useDownloadModel.ts b/web/hooks/useDownloadModel.ts index 9f6334c71..d0d13d93b 100644 --- a/web/hooks/useDownloadModel.ts +++ b/web/hooks/useDownloadModel.ts @@ -8,12 +8,15 @@ import { joinPath, ModelArtifact, DownloadState, + GpuSetting, } from '@janhq/core' import { useAtomValue, useSetAtom } from 'jotai' import { setDownloadStateAtom } from './useDownloadState' +import useGpuSetting from './useGpuSetting' + import { extensionManager } from '@/extension/ExtensionManager' import { ignoreSslAtom, @@ -29,6 +32,8 @@ export default function useDownloadModel() { const setDownloadState = useSetAtom(setDownloadStateAtom) const addDownloadingModel = useSetAtom(addDownloadingModelAtom) + const { getGpuSettings } = useGpuSetting() + const downloadModel = useCallback( async (model: Model) => { const childProgresses: DownloadState[] = model.sources.map( @@ -68,10 +73,22 @@ export default function useDownloadModel() { }) addDownloadingModel(model) - - await localDownloadModel(model, ignoreSSL, proxyEnabled ? proxy : '') + const gpuSettings = await getGpuSettings() + await localDownloadModel( + model, + ignoreSSL, + proxyEnabled ? proxy : '', + gpuSettings + ) }, - [ignoreSSL, proxy, proxyEnabled, addDownloadingModel, setDownloadState] + [ + ignoreSSL, + proxy, + proxyEnabled, + getGpuSettings, + addDownloadingModel, + setDownloadState, + ] ) const abortModelDownload = useCallback(async (model: Model) => { @@ -90,8 +107,9 @@ export default function useDownloadModel() { const localDownloadModel = async ( model: Model, ignoreSSL: boolean, - proxy: string + proxy: string, + gpuSettings?: GpuSetting ) => extensionManager .get(ExtensionTypeEnum.Model) - ?.downloadModel(model, { ignoreSSL, proxy }) + ?.downloadModel(model, gpuSettings, { ignoreSSL, proxy }) diff --git a/web/hooks/useDownloadState.ts b/web/hooks/useDownloadState.ts index 06de9bef6..03a8883cb 100644 --- a/web/hooks/useDownloadState.ts +++ b/web/hooks/useDownloadState.ts @@ -18,123 +18,129 @@ export const modelDownloadStateAtom = atom>({}) export const setDownloadStateAtom = atom( null, (get, set, state: DownloadState) => { - const currentState = { ...get(modelDownloadStateAtom) } + try { + const currentState = { ...get(modelDownloadStateAtom) } - if (state.downloadState === 'end') { - const modelDownloadState = currentState[state.modelId] + if (state.downloadState === 'end') { + const modelDownloadState = currentState[state.modelId] - const updatedChildren: DownloadState[] = - modelDownloadState.children!.filter( - (m) => m.fileName !== state.fileName + const updatedChildren: DownloadState[] = ( + modelDownloadState.children ?? [] + ).filter((m) => m.fileName !== state.fileName) + updatedChildren.push(state) + modelDownloadState.children = updatedChildren + currentState[state.modelId] = modelDownloadState + + const isAllChildrenDownloadEnd = modelDownloadState.children?.every( + (m) => m.downloadState === 'end' ) - updatedChildren.push(state) - modelDownloadState.children = updatedChildren - currentState[state.modelId] = modelDownloadState - const isAllChildrenDownloadEnd = modelDownloadState.children?.every( - (m) => m.downloadState === 'end' - ) + if (isAllChildrenDownloadEnd) { + // download successfully + delete currentState[state.modelId] + set(removeDownloadingModelAtom, state.modelId) - if (isAllChildrenDownloadEnd) { - // download successfully + const model = get(configuredModelsAtom).find( + (e) => e.id === state.modelId + ) + if (model) set(downloadedModelsAtom, (prev) => [...prev, model]) + toaster({ + title: 'Download Completed', + description: `Download ${state.modelId} completed`, + type: 'success', + }) + } + } else if (state.downloadState === 'error') { + // download error delete currentState[state.modelId] set(removeDownloadingModelAtom, state.modelId) - - const model = get(configuredModelsAtom).find( - (e) => e.id === state.modelId - ) - if (model) set(downloadedModelsAtom, (prev) => [...prev, model]) - toaster({ - title: 'Download Completed', - description: `Download ${state.modelId} completed`, - type: 'success', - }) - } - } else if (state.downloadState === 'error') { - // download error - delete currentState[state.modelId] - set(removeDownloadingModelAtom, state.modelId) - if (state.error === 'aborted') { - toaster({ - title: 'Cancel Download', - description: `Model ${state.modelId} download cancelled`, - type: 'warning', - }) - } else { - let error = state.error - if ( - typeof error?.includes === 'function' && - state.error?.includes('certificate') - ) { - error += - '. To fix enable "Ignore SSL Certificates" in Advanced settings.' + if (state.error === 'aborted') { + toaster({ + title: 'Cancel Download', + description: `Model ${state.modelId} download cancelled`, + type: 'warning', + }) + } else { + let error = state.error + if ( + typeof error?.includes === 'function' && + state.error?.includes('certificate') + ) { + error += + '. To fix enable "Ignore SSL Certificates" in Advanced settings.' + } + toaster({ + title: 'Download Failed', + description: `Model ${state.modelId} download failed: ${error}`, + type: 'error', + }) + } + } else { + // download in progress + if (state.size.total === 0) { + // this is initial state, just set the state + currentState[state.modelId] = state + set(modelDownloadStateAtom, currentState) + return } - toaster({ - title: 'Download Failed', - description: `Model ${state.modelId} download failed: ${error}`, - type: 'error', - }) - } - } else { - // download in progress - if (state.size.total === 0) { - // this is initial state, just set the state - currentState[state.modelId] = state - set(modelDownloadStateAtom, currentState) - return - } - const modelDownloadState = currentState[state.modelId] - if (!modelDownloadState) { - console.debug('setDownloadStateAtom: modelDownloadState not found') - return - } + const modelDownloadState = currentState[state.modelId] + if (!modelDownloadState) { + console.debug('setDownloadStateAtom: modelDownloadState not found') + return + } - // delete the children if the filename is matched and replace the new state - const updatedChildren: DownloadState[] = - modelDownloadState.children!.filter( - (m) => m.fileName !== state.fileName + // delete the children if the filename is matched and replace the new state + const updatedChildren: DownloadState[] = ( + modelDownloadState.children ?? [] + ).filter((m) => m.fileName !== state.fileName) + + updatedChildren.push(state) + + // re-calculate the overall progress if we have all the children download data + const isAnyChildDownloadNotReady = updatedChildren.some( + (m) => + m.size.total === 0 && + !modelDownloadState.children?.some( + (e) => e.fileName === m.fileName && e.downloadState === 'end' + ) && + modelDownloadState.children?.some((e) => e.fileName === m.fileName) ) - updatedChildren.push(state) + modelDownloadState.children = updatedChildren + if (isAnyChildDownloadNotReady) { + // just update the children + currentState[state.modelId] = modelDownloadState + set(modelDownloadStateAtom, currentState) + return + } - // re-calculate the overall progress if we have all the children download data - const isAnyChildDownloadNotReady = updatedChildren.some( - (m) => m.size.total === 0 - ) + const parentTotalSize = modelDownloadState.size.total + if (parentTotalSize === 0) { + // calculate the total size of the parent by sum all children total size + const totalSize = updatedChildren.reduce( + (acc, m) => acc + m.size.total, + 0 + ) - modelDownloadState.children = updatedChildren + modelDownloadState.size.total = totalSize + } - if (isAnyChildDownloadNotReady) { - // just update the children - currentState[state.modelId] = modelDownloadState - set(modelDownloadStateAtom, currentState) - - return - } - - const parentTotalSize = modelDownloadState.size.total - if (parentTotalSize === 0) { - // calculate the total size of the parent by sum all children total size - const totalSize = updatedChildren.reduce( - (acc, m) => acc + m.size.total, + // calculate the total transferred size by sum all children transferred size + const transferredSize = updatedChildren.reduce( + (acc, m) => acc + m.size.transferred, 0 ) - - modelDownloadState.size.total = totalSize + modelDownloadState.size.transferred = transferredSize + modelDownloadState.percent = + parentTotalSize === 0 ? 0 : transferredSize / parentTotalSize + currentState[state.modelId] = modelDownloadState } - // calculate the total transferred size by sum all children transferred size - const transferredSize = updatedChildren.reduce( - (acc, m) => acc + m.size.transferred, - 0 - ) - modelDownloadState.size.transferred = transferredSize - modelDownloadState.percent = - parentTotalSize === 0 ? 0 : transferredSize / parentTotalSize - currentState[state.modelId] = modelDownloadState + set(modelDownloadStateAtom, currentState) + } catch (e) { + console.debug('setDownloadStateAtom: state', state) + console.debug('setDownloadStateAtom: error', e) } - - set(modelDownloadStateAtom, currentState) } ) diff --git a/web/hooks/useGpuSetting.ts b/web/hooks/useGpuSetting.ts new file mode 100644 index 000000000..36f51ed57 --- /dev/null +++ b/web/hooks/useGpuSetting.ts @@ -0,0 +1,21 @@ +import { useCallback } from 'react' + +import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' + +import { extensionManager } from '@/extension' + +export default function useGpuSetting() { + const getGpuSettings = useCallback(async () => { + const gpuSetting = await extensionManager + ?.get(ExtensionTypeEnum.SystemMonitoring) + ?.getGpuSetting() + + if (!gpuSetting) { + console.debug('No GPU setting found') + return undefined + } + return gpuSetting + }, []) + + return { getGpuSettings } +} diff --git a/web/next.config.js b/web/next.config.js index a4b3e6d43..48ea0703e 100644 --- a/web/next.config.js +++ b/web/next.config.js @@ -38,6 +38,7 @@ const nextConfig = { isMac: process.platform === 'darwin', isWindows: process.platform === 'win32', isLinux: process.platform === 'linux', + PLATFORM: JSON.stringify(process.platform), }), ] return config diff --git a/web/screens/ExploreModels/ExploreModelItem/index.tsx b/web/screens/ExploreModels/ExploreModelItem/index.tsx index 9cdfbc01a..e8887e70f 100644 --- a/web/screens/ExploreModels/ExploreModelItem/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItem/index.tsx @@ -3,6 +3,8 @@ import { useState } from 'react' import { Model } from '@janhq/core' import { Badge } from '@janhq/uikit' +import { twMerge } from 'tailwind-merge' + import ExploreModelItemHeader from '@/screens/ExploreModels/ExploreModelItemHeader' type Props = { @@ -75,7 +77,16 @@ const ExploreModelItem: React.FC = ({ model }) => { Format -

{model.format}

+

+ {model.format} +

diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx index 38e7f65a6..581a628ba 100644 --- a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx @@ -152,6 +152,7 @@ const ExploreModelItemHeader: React.FC = ({ model, onClick, open }) => {
{model.name} +
@@ -172,4 +173,21 @@ const ExploreModelItemHeader: React.FC = ({ model, onClick, open }) => { ) } +type EngineBadgeProps = { + engine: string +} + +const EngineBadge: React.FC = ({ engine }) => { + switch (engine) { + case 'nitro-tensorrt-llm': + return ( +
+ TensorRT-LLM +
+ ) + default: + return null + } +} + export default ExploreModelItemHeader diff --git a/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx new file mode 100644 index 000000000..e4c8ae04f --- /dev/null +++ b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx @@ -0,0 +1,225 @@ +import { useCallback, useEffect, useState } from 'react' + +import { + Compatibility, + GpuSetting, + InstallationState, + abortDownload, + systemInformations, +} from '@janhq/core' +import { + Button, + Progress, + Tooltip, + TooltipArrow, + TooltipContent, + TooltipPortal, + TooltipTrigger, +} from '@janhq/uikit' + +import { InfoCircledIcon } from '@radix-ui/react-icons' +import { useAtomValue } from 'jotai' + +import { extensionManager } from '@/extension' +import Extension from '@/extension/Extension' +import { installingExtensionAtom } from '@/helpers/atoms/Extension.atom' + +type Props = { + item: Extension +} + +const TensorRtExtensionItem: React.FC = ({ item }) => { + const [compatibility, setCompatibility] = useState( + undefined + ) + const [installState, setInstallState] = + useState('NotRequired') + const installingExtensions = useAtomValue(installingExtensionAtom) + const [isGpuSupported, setIsGpuSupported] = useState(false) + + const isInstalling = installingExtensions.some( + (e) => e.extensionId === item.name + ) + + const progress = isInstalling + ? installingExtensions.find((e) => e.extensionId === item.name) + ?.percentage ?? -1 + : -1 + + useEffect(() => { + const getSystemInfos = async () => { + const info = await systemInformations() + if (!info) { + setIsGpuSupported(false) + return + } + + const gpuSettings: GpuSetting | undefined = info.gpuSetting + if (!gpuSettings || gpuSettings.gpus.length === 0) { + setIsGpuSupported(false) + return + } + + const arch = gpuSettings.gpus[0].arch + if (!arch) { + setIsGpuSupported(false) + return + } + + const supportedGpuArch = ['turing', 'ampere', 'ada'] + setIsGpuSupported(supportedGpuArch.includes(arch)) + } + getSystemInfos() + }, []) + + useEffect(() => { + const getExtensionInstallationState = async () => { + const extension = extensionManager.get(item.name ?? '') + if (!extension) return + + if (typeof extension?.installationState === 'function') { + const installState = await extension.installationState() + setInstallState(installState) + } + } + + getExtensionInstallationState() + }, [item.name, isInstalling]) + + useEffect(() => { + const extension = extensionManager.get(item.name ?? '') + if (!extension) return + setCompatibility(extension.compatibility()) + }, [setCompatibility, item.name]) + + const onInstallClick = useCallback(async () => { + const extension = extensionManager.get(item.name ?? '') + if (!extension) return + + await extension.install() + }, [item.name]) + + const onCancelInstallingClick = () => { + const extension = installingExtensions.find( + (e) => e.extensionId === item.name + ) + if (extension?.localPath) { + abortDownload(extension.localPath) + } + } + + return ( +
+
+
+
+ TensorRT-LLM Extension +
+

+ v{item.version} +

+
+

+ {item.description} +

+
+ {(!compatibility || compatibility['platform']?.includes(PLATFORM)) && + isGpuSupported ? ( +
+ +
+ ) : ( +
+
+ Incompatible{' '} + + + + + + + {compatibility ? ( + + Only available on{' '} + {compatibility?.platform + ?.map((e: string) => + e === 'win32' + ? 'Windows' + : e === 'linux' + ? 'Linux' + : 'MacOS' + ) + .join(', ')} + + ) : ( + + Your GPUs are not compatible with this extension + + )} + + + + +
+
+ )} +
+ ) +} + +type InstallStateProps = { + installProgress: number + installState: InstallationState + onInstallClick: () => void + onCancelClick: () => void +} + +const InstallStateIndicator: React.FC = ({ + installProgress, + installState, + onInstallClick, + onCancelClick, +}) => { + // TODO: NamH support dark mode for this + if (installProgress !== -1) { + const progress = installProgress * 100 + return ( +
+ +
+ + + {progress.toFixed(0)}% + +
+
+ ) + } + + // TODO: NamH check for dark mode here + switch (installState) { + case 'Installed': + return ( +
+ Installed +
+ ) + case 'NotInstalled': + return ( + + ) + default: + return
+ } +} + +export default TensorRtExtensionItem diff --git a/web/screens/Settings/CoreExtensions/index.tsx b/web/screens/Settings/CoreExtensions/index.tsx index 8c9f92d7a..f5b66abeb 100644 --- a/web/screens/Settings/CoreExtensions/index.tsx +++ b/web/screens/Settings/CoreExtensions/index.tsx @@ -4,13 +4,18 @@ import React, { useState, useEffect, useRef } from 'react' import { Button, ScrollArea } from '@janhq/uikit' +import Loader from '@/containers/Loader' + import { formatExtensionsName } from '@/utils/converter' +import TensorRtExtensionItem from './TensorRtExtensionItem' + import { extensionManager } from '@/extension' import Extension from '@/extension/Extension' const ExtensionCatalog = () => { const [activeExtensions, setActiveExtensions] = useState([]) + const [showLoading, setShowLoading] = useState(false) const fileInputRef = useRef(null) /** * Fetches the active extensions and their preferences from the `extensions` and `preferences` modules. @@ -63,65 +68,76 @@ const ExtensionCatalog = () => { const handleFileChange = (event: React.ChangeEvent) => { const file = event.target.files?.[0] if (file) { + setShowLoading(true) install(event) } } return ( - -
- {activeExtensions.map((item, i) => { - return ( -
-
-
-
- {formatExtensionsName(item.name ?? item.description ?? '')} -
-

- v{item.version} + <> + +

+ {activeExtensions.map((item, i) => { + // TODO: this is bad code, rewrite it + if (item.name === '@janhq/tensorrt-llm-extension') { + return + } + + return ( +
+
+
+
+ {formatExtensionsName( + item.name ?? item.description ?? '' + )} +
+

+ v{item.version} +

+
+

+ {item.description}

-

- {item.description} -

+ ) + })} + {/* Manual Installation */} +
+
+
+
+ Manual Installation +
+
+

+ Select a extension file to install (.tgz) +

- ) - })} - {/* Manual Installation */} -
-
-
-
- Manual Installation -
+
+ +
-

- Select a extension file to install (.tgz) -

-
-
- -
-
- + + {showLoading && } + ) } diff --git a/web/services/appService.ts b/web/services/appService.ts new file mode 100644 index 000000000..9327d55c3 --- /dev/null +++ b/web/services/appService.ts @@ -0,0 +1,24 @@ +import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' + +import { toaster } from '@/containers/Toast' + +import { extensionManager } from '@/extension' + +export const appService = { + systemInformations: async () => { + const gpuSetting = await extensionManager + ?.get(ExtensionTypeEnum.SystemMonitoring) + ?.getGpuSetting() + + return { + gpuSetting, + // TODO: Other system information + } + }, + showToast: (title: string, description: string) => { + toaster({ + title, + description: description, + }) + }, +} diff --git a/web/services/coreService.ts b/web/services/coreService.ts index c010c6cec..a483cc452 100644 --- a/web/services/coreService.ts +++ b/web/services/coreService.ts @@ -1,5 +1,7 @@ +import { appService } from './appService' import { EventEmitter } from './eventsService' import { restAPI } from './restService' + export const setupCoreServices = () => { if (typeof window === 'undefined') { console.debug('undefine', window) @@ -10,7 +12,10 @@ export const setupCoreServices = () => { if (!window.core) { window.core = { events: new EventEmitter(), - api: window.electronAPI ?? restAPI, + api: { + ...(window.electronAPI ? window.electronAPI : restAPI), + ...appService, + }, } } } diff --git a/web/types/index.d.ts b/web/types/index.d.ts index 833c3e2bd..ed83e0d14 100644 --- a/web/types/index.d.ts +++ b/web/types/index.d.ts @@ -11,6 +11,7 @@ declare global { declare const isMac: boolean declare const isWindows: boolean declare const isLinux: boolean + declare const PLATFORM: string interface Core { api: APIFunctions events: EventEmitter From 561b1dd94a07aec5289236410f75f6be2e055d14 Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 14 Mar 2024 07:27:24 +0000 Subject: [PATCH 11/13] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bc206f4eb..b25a917d7 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 03a0978f5345e394522f21f1638d0d62f89d2796 Mon Sep 17 00:00:00 2001 From: NamH Date: Thu, 14 Mar 2024 15:58:42 +0700 Subject: [PATCH 12/13] fix: some costmetic issues: badges corner, recommended for tensorrt models(#2346) Signed-off-by: hiro Co-authored-by: hiro --- uikit/src/badge/styles.scss | 2 +- web/containers/DropdownListSidebar/index.tsx | 2 +- web/screens/ExploreModels/ExploreModelItemHeader/index.tsx | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/uikit/src/badge/styles.scss b/uikit/src/badge/styles.scss index b777892d7..4788f65be 100644 --- a/uikit/src/badge/styles.scss +++ b/uikit/src/badge/styles.scss @@ -1,5 +1,5 @@ .badge { - @apply focus:ring-ring border-border inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-offset-2; + @apply focus:ring-ring border-border inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-offset-2; &-primary { @apply border-transparent bg-blue-100 text-blue-600; diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index ef7532063..70651a4d4 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -294,7 +294,7 @@ const DropdownListSidebar = ({ {toGibibytes(x.metadata.size)} - {x.engine == InferenceEngine.nitro && ( + {x.metadata.size && ( )}
diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx index 581a628ba..ef75f57f5 100644 --- a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx @@ -180,10 +180,11 @@ type EngineBadgeProps = { const EngineBadge: React.FC = ({ engine }) => { switch (engine) { case 'nitro-tensorrt-llm': + const title = 'TensorRT-LLM' return ( -
- TensorRT-LLM -
+ + {title} + ) default: return null From d9c3852997eb42159701fefa5265f7ba3dd74ea3 Mon Sep 17 00:00:00 2001 From: NamH Date: Thu, 14 Mar 2024 17:16:14 +0700 Subject: [PATCH 13/13] fix: ts error when declar var in case (#2348) Signed-off-by: James Co-authored-by: James --- web/screens/ExploreModels/ExploreModelItemHeader/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx index ef75f57f5..465e69fa6 100644 --- a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx +++ b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx @@ -178,9 +178,10 @@ type EngineBadgeProps = { } const EngineBadge: React.FC = ({ engine }) => { + const title = 'TensorRT-LLM' + switch (engine) { case 'nitro-tensorrt-llm': - const title = 'TensorRT-LLM' return ( {title}