Merge pull request #3966 from janhq/fix/add-llamacpp-engine-path-to-export-env

fix: export PATH env to engine destination folder to have additional dlls scoped

extensions/inference-cortex-extension/download.bat

@@ -7,7 +7,7 @@ set /p CORTEX_VERSION=<./bin/version.txt
 set VERSION=v0.1.35
 set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/%VERSION%/cortex.llamacpp-0.1.35-windows-amd64
 set CUDA_DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/%VERSION%
-set SUBFOLDERS=win-cuda-12-0 win-cuda-11-7 win-noavx win-avx win-avx2 win-avx512 win-vulkan
+set SUBFOLDERS=noavx-cuda-12-0 noavx-cuda-11-7 avx2-cuda-12-0 avx2-cuda-11-7 noavx avx avx2 avx512 vulkan
 
 call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
 call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %BIN_PATH%/avx2-cuda-12-0/engines/cortex.llamacpp

extensions/inference-cortex-extension/src/index.ts

@@ -11,11 +11,11 @@ import {
   executeOnMain,
   systemInformation,
   joinPath,
-  dirName,
   LocalOAIEngine,
   InferenceEngine,
   getJanDataFolderPath,
   extractModelLoadParams,
+  fs,
 } from '@janhq/core'
 import PQueue from 'p-queue'
 import ky from 'ky'
@@ -97,22 +97,24 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
       model.settings = settings
     }
 
-    return await ky
-      .post(`${CORTEX_API_URL}/v1/models/start`, {
-        json: {
-          ...extractModelLoadParams(model.settings),
-          model: model.id,
-          engine:
-            model.engine === InferenceEngine.nitro // Legacy model cache
-              ? InferenceEngine.cortex_llamacpp
-              : model.engine,
-        },
-      })
-      .json()
-      .catch(async (e) => {
-        throw (await e.response?.json()) ?? e
-      })
-      .then()
+    return await this.queue.add(() =>
+      ky
+        .post(`${CORTEX_API_URL}/v1/models/start`, {
+          json: {
+            ...extractModelLoadParams(model.settings),
+            model: model.id,
+            engine:
+              model.engine === InferenceEngine.nitro // Legacy model cache
+                ? InferenceEngine.cortex_llamacpp
+                : model.engine,
+          },
+        })
+        .json()
+        .catch(async (e) => {
+          throw (await e.response?.json()) ?? e
+        })
+        .then()
+    )
   }
 
   override async unloadModel(model: Model): Promise<void> {
@@ -160,7 +162,10 @@ export const getModelFilePath = async (
   file: string
 ): Promise<string> => {
   // Symlink to the model file
-  if (!model.sources[0]?.url.startsWith('http')) {
+  if (
+    !model.sources[0]?.url.startsWith('http') &&
+    (await fs.existsSync(model.sources[0].url))
+  ) {
     return model.sources[0]?.url
   }
   return joinPath([await getJanDataFolderPath(), 'models', model.id, file])

extensions/inference-cortex-extension/src/node/index.ts

@@ -33,6 +33,11 @@ function run(systemInfo?: SystemInformation): Promise<any> {
     addEnvPaths(path.join(appResourcePath(), 'shared'))
     addEnvPaths(executableOptions.binPath)
     addEnvPaths(executableOptions.enginePath)
+    // Add the cortex.llamacpp path to the PATH and LD_LIBRARY_PATH
+    // This is required for the cortex engine to run for now since dlls are not moved to the root
+    addEnvPaths(
+      path.join(executableOptions.enginePath, 'engines', 'cortex.llamacpp')
+    )
 
     const dataFolderPath = getJanDataFolderPath()
     watchdog = new ProcessWatchdog(

extensions/model-extension/src/index.ts

@@ -4,12 +4,13 @@ import {
   InferenceEngine,
   joinPath,
   dirName,
+  fs,
   ModelManager,
   abortDownload,
   DownloadState,
   events,
   DownloadEvent,
-  OptionType
+  OptionType,
 } from '@janhq/core'
 import { CortexAPI } from './cortex'
 import { scanModelsFolder } from './legacy/model-json'
@@ -181,7 +182,8 @@ export default class JanModelExtension extends ModelExtension {
         toImportModels.map(async (model: Model & { file_path: string }) =>
           this.importModel(
             model.id,
-            model.sources[0].url.startsWith('http')
+            model.sources[0].url.startsWith('http') ||
+              !(await fs.existsSync(model.sources[0].url))
               ? await joinPath([
                   await dirName(model.file_path),
                   model.sources[0]?.filename ??
@@ -189,7 +191,8 @@ export default class JanModelExtension extends ModelExtension {
                     model.sources[0]?.url.split('/').pop() ??
                     model.id,
                 ]) // Copied models
-              : model.sources[0].url // Symlink models
+              : model.sources[0].url, // Symlink models,
+            model.name
           )
         )
       )