diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index e4d777ea2..ecff683c3 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -7,7 +7,7 @@ set /p CORTEX_VERSION=<./bin/version.txt set VERSION=v0.1.35 set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/%VERSION%/cortex.llamacpp-0.1.35-windows-amd64 set CUDA_DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/%VERSION% -set SUBFOLDERS=win-cuda-12-0 win-cuda-11-7 win-noavx win-avx win-avx2 win-avx512 win-vulkan +set SUBFOLDERS=noavx-cuda-12-0 noavx-cuda-11-7 avx2-cuda-12-0 avx2-cuda-11-7 noavx avx avx2 avx512 vulkan call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %BIN_PATH%/avx2-cuda-12-0/engines/cortex.llamacpp diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index e099aae44..d070ff9a3 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -11,11 +11,11 @@ import { executeOnMain, systemInformation, joinPath, - dirName, LocalOAIEngine, InferenceEngine, getJanDataFolderPath, extractModelLoadParams, + fs, } from '@janhq/core' import PQueue from 'p-queue' import ky from 'ky' @@ -97,22 +97,24 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { model.settings = settings } - return await ky - .post(`${CORTEX_API_URL}/v1/models/start`, { - json: { - ...extractModelLoadParams(model.settings), - model: model.id, - engine: - model.engine === InferenceEngine.nitro // Legacy model cache - ? InferenceEngine.cortex_llamacpp - : model.engine, - }, - }) - .json() - .catch(async (e) => { - throw (await e.response?.json()) ?? e - }) - .then() + return await this.queue.add(() => + ky + .post(`${CORTEX_API_URL}/v1/models/start`, { + json: { + ...extractModelLoadParams(model.settings), + model: model.id, + engine: + model.engine === InferenceEngine.nitro // Legacy model cache + ? InferenceEngine.cortex_llamacpp + : model.engine, + }, + }) + .json() + .catch(async (e) => { + throw (await e.response?.json()) ?? e + }) + .then() + ) } override async unloadModel(model: Model): Promise { @@ -160,7 +162,10 @@ export const getModelFilePath = async ( file: string ): Promise => { // Symlink to the model file - if (!model.sources[0]?.url.startsWith('http')) { + if ( + !model.sources[0]?.url.startsWith('http') && + (await fs.existsSync(model.sources[0].url)) + ) { return model.sources[0]?.url } return joinPath([await getJanDataFolderPath(), 'models', model.id, file]) diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index a52de20bb..3816605d2 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -33,6 +33,11 @@ function run(systemInfo?: SystemInformation): Promise { addEnvPaths(path.join(appResourcePath(), 'shared')) addEnvPaths(executableOptions.binPath) addEnvPaths(executableOptions.enginePath) + // Add the cortex.llamacpp path to the PATH and LD_LIBRARY_PATH + // This is required for the cortex engine to run for now since dlls are not moved to the root + addEnvPaths( + path.join(executableOptions.enginePath, 'engines', 'cortex.llamacpp') + ) const dataFolderPath = getJanDataFolderPath() watchdog = new ProcessWatchdog( diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index e62e5b2ee..7d7514f3b 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -4,12 +4,13 @@ import { InferenceEngine, joinPath, dirName, + fs, ModelManager, abortDownload, DownloadState, events, DownloadEvent, - OptionType + OptionType, } from '@janhq/core' import { CortexAPI } from './cortex' import { scanModelsFolder } from './legacy/model-json' @@ -181,7 +182,8 @@ export default class JanModelExtension extends ModelExtension { toImportModels.map(async (model: Model & { file_path: string }) => this.importModel( model.id, - model.sources[0].url.startsWith('http') + model.sources[0].url.startsWith('http') || + !(await fs.existsSync(model.sources[0].url)) ? await joinPath([ await dirName(model.file_path), model.sources[0]?.filename ?? @@ -189,7 +191,8 @@ export default class JanModelExtension extends ModelExtension { model.sources[0]?.url.split('/').pop() ?? model.id, ]) // Copied models - : model.sources[0].url // Symlink models + : model.sources[0].url, // Symlink models, + model.name ) ) )