From e41bcffcefdde2a0df8ff055b681f2777b691988 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 7 Nov 2024 10:10:05 +0700 Subject: [PATCH 1/6] fix: export PATH env to engine destination folder to have additional dlls scoped --- extensions/inference-cortex-extension/src/node/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index a52de20bb..3816605d2 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -33,6 +33,11 @@ function run(systemInfo?: SystemInformation): Promise { addEnvPaths(path.join(appResourcePath(), 'shared')) addEnvPaths(executableOptions.binPath) addEnvPaths(executableOptions.enginePath) + // Add the cortex.llamacpp path to the PATH and LD_LIBRARY_PATH + // This is required for the cortex engine to run for now since dlls are not moved to the root + addEnvPaths( + path.join(executableOptions.enginePath, 'engines', 'cortex.llamacpp') + ) const dataFolderPath = getJanDataFolderPath() watchdog = new ProcessWatchdog( From ced44973b821af829ab1cbfba136036ee72f15f8 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 7 Nov 2024 12:06:46 +0700 Subject: [PATCH 2/6] chore: queue server start and model load --- extensions/inference-cortex-extension/src/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index e099aae44..2f65c3c09 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -11,7 +11,6 @@ import { executeOnMain, systemInformation, joinPath, - dirName, LocalOAIEngine, InferenceEngine, getJanDataFolderPath, @@ -97,7 +96,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { model.settings = settings } - return await ky + return await this.queue.add(() => ky .post(`${CORTEX_API_URL}/v1/models/start`, { json: { ...extractModelLoadParams(model.settings), @@ -112,7 +111,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { .catch(async (e) => { throw (await e.response?.json()) ?? e }) - .then() + .then()) } override async unloadModel(model: Model): Promise { From 40019892b875a8ca003afcf37aa474e22472b45f Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 7 Nov 2024 13:01:34 +0700 Subject: [PATCH 3/6] chore: correct name of bin subfolders to move dll properly --- extensions/inference-cortex-extension/download.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index e4d777ea2..ecff683c3 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -7,7 +7,7 @@ set /p CORTEX_VERSION=<./bin/version.txt set VERSION=v0.1.35 set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/%VERSION%/cortex.llamacpp-0.1.35-windows-amd64 set CUDA_DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/%VERSION% -set SUBFOLDERS=win-cuda-12-0 win-cuda-11-7 win-noavx win-avx win-avx2 win-avx512 win-vulkan +set SUBFOLDERS=noavx-cuda-12-0 noavx-cuda-11-7 avx2-cuda-12-0 avx2-cuda-11-7 noavx avx avx2 avx512 vulkan call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %BIN_PATH%/avx2-cuda-12-0/engines/cortex.llamacpp From 2e9b7fdad28cce46e871aa80c33d830cf5bc0ccc Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 7 Nov 2024 13:34:09 +0700 Subject: [PATCH 4/6] chore: add import name for legacy models --- extensions/model-extension/src/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index e62e5b2ee..78c85cf0b 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -9,7 +9,7 @@ import { DownloadState, events, DownloadEvent, - OptionType + OptionType, } from '@janhq/core' import { CortexAPI } from './cortex' import { scanModelsFolder } from './legacy/model-json' @@ -189,7 +189,8 @@ export default class JanModelExtension extends ModelExtension { model.sources[0]?.url.split('/').pop() ?? model.id, ]) // Copied models - : model.sources[0].url // Symlink models + : model.sources[0].url, // Symlink models, + model.name ) ) ) From a773e169fccb9d3a9c9d23b44ca3d565cf2cb02a Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 7 Nov 2024 14:26:41 +0700 Subject: [PATCH 5/6] fix: an edge case where auto import does not work with relative model file path --- extensions/model-extension/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 78c85cf0b..7d7514f3b 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -4,6 +4,7 @@ import { InferenceEngine, joinPath, dirName, + fs, ModelManager, abortDownload, DownloadState, @@ -181,7 +182,8 @@ export default class JanModelExtension extends ModelExtension { toImportModels.map(async (model: Model & { file_path: string }) => this.importModel( model.id, - model.sources[0].url.startsWith('http') + model.sources[0].url.startsWith('http') || + !(await fs.existsSync(model.sources[0].url)) ? await joinPath([ await dirName(model.file_path), model.sources[0]?.filename ?? From 0847b32e87acbf419810cedba714354910635062 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 7 Nov 2024 14:29:27 +0700 Subject: [PATCH 6/6] fix: an edge case when start a model with relative model path --- .../inference-cortex-extension/src/index.ts | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 2f65c3c09..d070ff9a3 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -15,6 +15,7 @@ import { InferenceEngine, getJanDataFolderPath, extractModelLoadParams, + fs, } from '@janhq/core' import PQueue from 'p-queue' import ky from 'ky' @@ -96,22 +97,24 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { model.settings = settings } - return await this.queue.add(() => ky - .post(`${CORTEX_API_URL}/v1/models/start`, { - json: { - ...extractModelLoadParams(model.settings), - model: model.id, - engine: - model.engine === InferenceEngine.nitro // Legacy model cache - ? InferenceEngine.cortex_llamacpp - : model.engine, - }, - }) - .json() - .catch(async (e) => { - throw (await e.response?.json()) ?? e - }) - .then()) + return await this.queue.add(() => + ky + .post(`${CORTEX_API_URL}/v1/models/start`, { + json: { + ...extractModelLoadParams(model.settings), + model: model.id, + engine: + model.engine === InferenceEngine.nitro // Legacy model cache + ? InferenceEngine.cortex_llamacpp + : model.engine, + }, + }) + .json() + .catch(async (e) => { + throw (await e.response?.json()) ?? e + }) + .then() + ) } override async unloadModel(model: Model): Promise { @@ -159,7 +162,10 @@ export const getModelFilePath = async ( file: string ): Promise => { // Symlink to the model file - if (!model.sources[0]?.url.startsWith('http')) { + if ( + !model.sources[0]?.url.startsWith('http') && + (await fs.existsSync(model.sources[0].url)) + ) { return model.sources[0]?.url } return joinPath([await getJanDataFolderPath(), 'models', model.id, file])