Merge pull request #4465 from janhq/chore/log-engine-variant-symlink

fix: bump engine version - update engine variant symlink and copy CUDA dependencies

extensions/engine-management-extension/rolldown.config.mjs

@@ -13,7 +13,7 @@ export default defineConfig([
       NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
       API_URL: JSON.stringify('http://127.0.0.1:39291'),
       SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'),
-      CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'),
+      CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'),
       DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
       DEFAULT_REMOTE_MODELS: JSON.stringify(models),
     },
@@ -26,7 +26,7 @@ export default defineConfig([
       file: 'dist/node/index.cjs.js',
     },
     define: {
-      CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'),
+      CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'),
     },
   },
   {

extensions/engine-management-extension/src/node/index.ts

@@ -138,9 +138,11 @@ const symlinkEngines = async () => {
 
     await mkdir(path.join(symlinkEnginePath, variant), {
       recursive: true,
-    }).catch(console.error)
+    }).catch((error) => log(JSON.stringify(error)))
 
-    await symlink(targetVariantPath, symlinkVariantPath).catch(console.error)
+    await symlink(targetVariantPath, symlinkVariantPath, 'junction').catch(
+      (error) => log(JSON.stringify(error))
+    )
     console.log(`Symlink created: ${targetVariantPath} -> ${symlinkEnginePath}`)
   }
 }

extensions/inference-cortex-extension/download.bat

@@ -2,7 +2,7 @@
 set BIN_PATH=./bin
 set SHARED_PATH=./../../electron/shared
 set /p CORTEX_VERSION=<./bin/version.txt
-set ENGINE_VERSION=0.1.43
+set ENGINE_VERSION=0.1.46
 
 @REM Download cortex.llamacpp binaries
 set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
@@ -19,8 +19,8 @@ call .\node_modules\.bin\download %DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o %SHA
 call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION%
 call .\node_modules\.bin\download %DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION%
 call .\node_modules\.bin\download %DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION%
-call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %SHARED_PATH%
+call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
-call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %SHARED_PATH%
+call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
 
 move %BIN_PATH%\cortex-server-beta.exe %BIN_PATH%\cortex-server.exe
 del %BIN_PATH%\cortex-beta.exe

extensions/inference-cortex-extension/download.sh

@@ -2,10 +2,11 @@
 
 # Read CORTEX_VERSION
 CORTEX_VERSION=$(cat ./bin/version.txt)
-ENGINE_VERSION=0.1.43
+ENGINE_VERSION=0.1.46
 CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download"
 ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}"
 CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}"
+BIN_PATH=./bin
 SHARED_PATH="../../electron/shared"
 # Detect platform
 OS_TYPE=$(uname)
@@ -28,8 +29,8 @@ if [ "$OS_TYPE" == "Linux" ]; then
     download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}" 1
     download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}" 1
     download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1
-    download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1
+    download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
-    download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1
+    download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
 
 elif [ "$OS_TYPE" == "Darwin" ]; then
     # macOS downloads

extensions/inference-cortex-extension/rolldown.config.mjs

@@ -111,7 +111,7 @@ export default defineConfig([
       SETTINGS: JSON.stringify(defaultSettingJson),
       CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'),
       CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'),
-      CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'),
+      CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'),
     },
   },
   {

extensions/inference-cortex-extension/src/node/index.ts

@@ -1,5 +1,10 @@
 import path from 'path'
-import { appResourcePath, getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node'
+import {
+  appResourcePath,
+  getJanDataFolderPath,
+  log,
+  SystemInformation,
+} from '@janhq/core/node'
 import { ProcessWatchdog } from './watchdog'
 import { readdir, symlink } from 'fs/promises'
 
@@ -18,13 +23,9 @@ function run(systemInfo?: SystemInformation): Promise<any> {
     let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? ''
     let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}`
     const binPath = path.join(__dirname, '..', 'bin')
-    await createEngineSymlinks(binPath)
 
     const executablePath = path.join(binPath, binaryName)
-    const sharedPath = path.join(
+    const sharedPath = path.join(appResourcePath(), 'shared')
-      appResourcePath(),
-      'shared'
-    )
     // Execute the binary
     log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`)
 
@@ -61,23 +62,6 @@ function run(systemInfo?: SystemInformation): Promise<any> {
   })
 }
 
-/**
- * Create symlinks for the engine shared libraries
- * @param binPath 
- */
-async function createEngineSymlinks(binPath: string) {
-  const sharedPath = path.join(appResourcePath(), 'shared')
-  const sharedLibFiles = await readdir(sharedPath)
-  for (const sharedLibFile of sharedLibFiles) {
-    if (sharedLibFile.endsWith('.dll') || sharedLibFile.endsWith('.so')) {
-      const targetDllPath = path.join(sharedPath, sharedLibFile)
-      const symlinkDllPath = path.join(binPath, sharedLibFile)
-      await symlink(targetDllPath, symlinkDllPath).catch(console.error)
-      console.log(`Symlink created: ${targetDllPath} -> ${symlinkDllPath}`)
-    }
-  }
-}
-
 /**
  * Every module should have a dispose function
  * This will be called when the extension is unloaded and should clean up any resources
@@ -87,18 +71,6 @@ function dispose() {
   watchdog?.terminate()
 }
 
-function addEnvPaths(dest: string) {
-  // Add engine path to the PATH and LD_LIBRARY_PATH
-  if (process.platform === 'win32') {
-    process.env.PATH = (process.env.PATH || '').concat(path.delimiter, dest)
-  } else {
-    process.env.LD_LIBRARY_PATH = (process.env.LD_LIBRARY_PATH || '').concat(
-      path.delimiter,
-      dest
-    )
-  }
-}
-
 /**
  * Cortex process info
  */

web/containers/Layout/BottomPanel/DownloadingState/index.tsx

@@ -32,12 +32,12 @@ export default function DownloadingState() {
     <Fragment>
       {Object.values(downloadStates)?.length > 0 && (
         <Modal
-          title="Downloading model"
+          title="Downloading"
           trigger={
             <div className="flex cursor-pointer items-center gap-2">
               <Button size="small" theme="ghost">
                 <span className="font-medium">
-                  Downloading model{' '}
+                  Downloading{' '}
                   {Object.values(downloadStates).length > 1 &&
                     `1/${Object.values(downloadStates).length}`}
                 </span>

web/screens/Settings/Engines/LocalEngineSettings.tsx

@@ -11,6 +11,7 @@ import { Button, ScrollArea, Badge, Select, Progress } from '@janhq/joi'
 
 import { twMerge } from 'tailwind-merge'
 
+import { useActiveModel } from '@/hooks/useActiveModel'
 import {
   useGetDefaultEngineVariant,
   useGetInstalledEngines,
@@ -52,6 +53,7 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => {
   const [installingEngines, setInstallingEngines] = useState<
     Map<string, number>
   >(new Map())
+  const { stopModel } = useActiveModel()
 
   const isEngineUpdated =
     latestReleasedEngine &&
@@ -103,7 +105,8 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => {
   }, [defaultEngineVariant])
 
   const handleEngineUpdate = useCallback(
-    (event: { id: string; type: DownloadEvent; percent: number }) => {
+    async (event: { id: string; type: DownloadEvent; percent: number }) => {
+      await stopModel().catch(console.info)
       mutateInstalledEngines()
       mutateDefaultEngineVariant()
       // Backward compatible support - cortex.cpp returns full variant file name
@@ -138,6 +141,7 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => {
       })
     },
     [
+      stopModel,
       mutateDefaultEngineVariant,
       mutateInstalledEngines,
       setInstallingEngines,
@@ -152,7 +156,8 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => {
     }
   }, [handleEngineUpdate])
 
-  const handleChangeVariant = (e: string) => {
+  const handleChangeVariant = async (e: string) => {
+    await stopModel().catch(console.info)
     setSelectedVariants(e)
     setDefaultEngineVariant(engine, {
       variant: e,