From 2c11caf87e7f34e18b8e939e3bb23f51f7bc937b Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 28 Oct 2024 16:36:58 +0700 Subject: [PATCH] chore: shared cuda dependencies --- .husky/pre-commit | 2 +- electron/package.json | 6 ++- electron/shared/.gitkeep | 0 .../inference-cortex-extension/download.bat | 5 ++- .../inference-cortex-extension/download.sh | 4 +- .../src/node/index.ts | 41 +++++++++++-------- extensions/model-extension/src/index.ts | 3 +- 7 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 electron/shared/.gitkeep diff --git a/.husky/pre-commit b/.husky/pre-commit index 177cd4216..53c4e577e 100644 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1 +1 @@ -oxlint --fix || npm run lint --fix \ No newline at end of file +npx oxlint@latest --fix \ No newline at end of file diff --git a/electron/package.json b/electron/package.json index feaee5e16..273062139 100644 --- a/electron/package.json +++ b/electron/package.json @@ -18,7 +18,8 @@ "docs/**/*", "scripts/**/*", "icons/**/*", - "themes" + "themes", + "shared" ], "asarUnpack": [ "pre-install", @@ -26,7 +27,8 @@ "docs", "scripts", "icons", - "themes" + "themes", + "shared" ], "publish": [ { diff --git a/electron/shared/.gitkeep b/electron/shared/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index 6d917cf79..9b43459ce 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -1,5 +1,6 @@ @echo off set BIN_PATH=./bin +set SHARED_PATH=./../../electron/shared set /p CORTEX_VERSION=<./bin/version.txt @REM Download cortex.llamacpp binaries @@ -18,8 +19,8 @@ call .\node_modules\.bin\download %DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o %BIN call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o %BIN_PATH%/avx2/engines/cortex.llamacpp call .\node_modules\.bin\download %DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o %BIN_PATH%/avx512/engines/cortex.llamacpp call .\node_modules\.bin\download %DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o %BIN_PATH%/vulkan/engines/cortex.llamacpp -call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% -call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% +call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %SHARED_PATH% +call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %SHARED_PATH% @REM Loop through each folder and move DLLs (excluding engine.dll) diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index 3e809765c..be1f67f2d 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -23,8 +23,8 @@ if [ "$OS_TYPE" == "Linux" ]; then download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" -e --strip 1 -o "./bin/noavx-cuda-12-0/engines/cortex.llamacpp" 1 download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" -e --strip 1 -o "./bin/noavx-cuda-11-7/engines/cortex.llamacpp" 1 download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "./bin/vulkan/engines/cortex.llamacpp" 1 - download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "./bin" 1 - download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "./bin" 1 + download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "../../electron/shared" 1 + download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "../../electron/shared" 1 elif [ "$OS_TYPE" == "Darwin" ]; then # macOS downloads diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index c54dae4c0..a52de20bb 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -2,6 +2,7 @@ import path from 'path' import { getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node' import { executableCortexFile } from './execute' import { ProcessWatchdog } from './watchdog' +import { appResourcePath } from '@janhq/core/node' // The HOST address to use for the Nitro subprocess const LOCAL_PORT = '39291' @@ -19,9 +20,9 @@ function run(systemInfo?: SystemInformation): Promise { // If ngl is not set or equal to 0, run on CPU with correct instructions systemInfo?.gpuSetting ? { - ...systemInfo.gpuSetting, - run_mode: systemInfo.gpuSetting.run_mode, - } + ...systemInfo.gpuSetting, + run_mode: systemInfo.gpuSetting.run_mode, + } : undefined ) @@ -29,20 +30,9 @@ function run(systemInfo?: SystemInformation): Promise { log(`[CORTEX]:: Spawn cortex at path: ${executableOptions.executablePath}`) log(`[CORTEX]:: Cortex engine path: ${executableOptions.enginePath}`) - // Add engine path to the PATH and LD_LIBRARY_PATH - process.env.PATH = (process.env.PATH || '').concat( - path.delimiter, - executableOptions.enginePath, - path.delimiter, - executableOptions.binPath - ) - log(`[CORTEX] PATH: ${process.env.PATH}`) - process.env.LD_LIBRARY_PATH = (process.env.LD_LIBRARY_PATH || '').concat( - path.delimiter, - executableOptions.enginePath, - path.delimiter, - executableOptions.binPath - ) + addEnvPaths(path.join(appResourcePath(), 'shared')) + addEnvPaths(executableOptions.binPath) + addEnvPaths(executableOptions.enginePath) const dataFolderPath = getJanDataFolderPath() watchdog = new ProcessWatchdog( @@ -83,6 +73,23 @@ function dispose() { watchdog?.terminate() } +function addEnvPaths(dest: string) { + // Add engine path to the PATH and LD_LIBRARY_PATH + if (process.platform === 'win32') { + process.env.PATH = (process.env.PATH || '').concat( + path.delimiter, + dest, + ) + log(`[CORTEX] PATH: ${process.env.PATH}`) + } else { + process.env.LD_LIBRARY_PATH = (process.env.LD_LIBRARY_PATH || '').concat( + path.delimiter, + dest, + ) + log(`[CORTEX] LD_LIBRARY_PATH: ${process.env.LD_LIBRARY_PATH}`) + } +} + /** * Cortex process info */ diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 3696acd79..3e0af0172 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -13,6 +13,7 @@ import { import { CortexAPI } from './cortex' import { scanModelsFolder } from './legacy/model-json' import { downloadModel } from './legacy/download' +import { systemInformation } from '@janhq/core' declare const SETTINGS: Array @@ -65,7 +66,7 @@ export default class JanModelExtension extends ModelExtension { model.engine === InferenceEngine.nitro_tensorrt_llm || model.settings.vision_model ) { - return downloadModel(model) + return downloadModel(model, (await systemInformation()).gpuSetting) } } /**