chore: validate model status

core/src/index.ts

@@ -8,7 +8,7 @@ export { core, deleteFile, invokePluginFunc } from "./core";
  * Core module exports.
  * @module
  */
-export { downloadFile, executeOnMain } from "./core";
+export { downloadFile, executeOnMain, appDataPath } from "./core";
 
 /**
  * Events module exports.

plugins/inference-plugin/nitro/version.txt

@@ -1 +1 @@
-0.1.7
+0.1.8

plugins/inference-plugin/package.json

@@ -39,7 +39,9 @@
   "dependencies": {
     "@janhq/core": "file:../../core",
     "download-cli": "^1.1.1",
+    "fetch-retry": "^5.0.6",
     "kill-port": "^2.0.1",
+    "path-browserify": "^1.0.1",
     "rxjs": "^7.8.1",
     "tcp-port-used": "^1.0.2",
     "ts-loader": "^9.5.0",
@@ -55,6 +57,7 @@
   ],
   "bundleDependencies": [
     "tcp-port-used",
-    "kill-port"
+    "kill-port",
+    "fetch-retry"
   ]
 }

plugins/inference-plugin/src/index.ts

@@ -17,6 +17,8 @@ import {
 import { InferencePlugin } from "@janhq/core/lib/plugins";
 import { requestInference } from "./helpers/sse";
 import { ulid } from "ulid";
+import { join } from "path";
+import { appDataPath } from "@janhq/core";
 
 /**
  * A class that implements the InferencePlugin interface from the @janhq/core package.
@@ -48,18 +50,19 @@ export default class JanInferencePlugin implements InferencePlugin {
 
   /**
    * Initializes the model with the specified file name.
-   * @param {string} modelFileName - The name of the model file.
+   * @param {string} modelFileName - The file name of the model file.
    * @returns {Promise<void>} A promise that resolves when the model is initialized.
    */
-  initModel(modelFileName: string): Promise<void> {
+  async initModel(modelFileName: string): Promise<void> {
-    return executeOnMain(MODULE, "initModel", modelFileName);
+    const appPath = await appDataPath();
+    return executeOnMain(MODULE, "initModel", join(appPath, modelFileName));
   }
 
   /**
    * Stops the model.
    * @returns {Promise<void>} A promise that resolves when the model is stopped.
    */
-  stopModel(): Promise<void> {
+  async stopModel(): Promise<void> {
     return executeOnMain(MODULE, "killSubprocess");
   }
 

plugins/inference-plugin/src/module.ts

@@ -1,9 +1,9 @@
 const fs = require("fs");
 const kill = require("kill-port");
 const path = require("path");
-const { app } = require("electron");
 const { spawn } = require("child_process");
 const tcpPortUsed = require("tcp-port-used");
+const fetchRetry = require("fetch-retry")(global.fetch);
 
 // The PORT to use for the Nitro subprocess
 const PORT = 3928;
@@ -11,9 +11,11 @@ const LOCAL_HOST = "127.0.0.1";
 const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`;
 const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`;
 const NITRO_HTTP_UNLOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/unloadModel`;
+const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`;
 
 // The subprocess instance for Nitro
 let subprocess = null;
+let currentModelFile = null;
 
 /**
  * The response from the initModel function.
@@ -25,74 +27,84 @@ interface InitModelResponse {
 
 /**
  * Initializes a Nitro subprocess to load a machine learning model.
- * @param fileName - The name of the machine learning model file.
+ * @param modelFile - The name of the machine learning model file.
  * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load.
  * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package
  * TODO: Should it be startModel instead?
  */
-function initModel(fileName: string): Promise<InitModelResponse> {
+function initModel(modelFile: string): Promise<InitModelResponse> {
   // 1. Check if the model file exists
+  currentModelFile = modelFile;
+
   return (
-    checkModelFileExist(fileName)
+    // 1. Check if the port is used, if used, attempt to unload model / kill nitro process
-      // 2. Check if the port is used, if used, attempt to unload model / kill nitro process
+    checkAndUnloadNitro()
-      .then(checkAndUnloadNitro)
+      // 2. Spawn the Nitro subprocess
-      // 3. Spawn the Nitro subprocess
       .then(spawnNitroProcess)
-      // 4. Wait until the port is used (Nitro http server is up)
+      // 3. Wait until the port is used (Nitro http server is up)
       .then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000))
-      // 5. Load the model into the Nitro subprocess (HTTP POST request)
+      // 4. Load the model into the Nitro subprocess (HTTP POST request)
-      .then(() => loadLLMModel(fileName))
+      .then(loadLLMModel)
-      // 6. Check if the model is loaded successfully
+      // 5. Check if the model is loaded successfully
-      .then(async (res) => {
+      .then(validateModelStatus)
-        if (res.ok) {
-          // Success - Model loaded
-          return {};
-        }
-        const json = await res.json();
-        throw new Error(`${json?.message ?? "Model loading failed."}`);
-      })
-      .catch((err) => {
-        return { error: err };
-      })
   );
 }
 
 /**
  * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request.
- * @param fileName - The name of the model file.
  * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load.
  */
-function loadLLMModel(fileName: string): Promise<Response> {
+function loadLLMModel(): Promise<Response> {
-  const llama_model_path = path.join(appPath(), fileName);
-
   const config = {
-    llama_model_path,
+    llama_model_path: currentModelFile,
     ctx_len: 2048,
     ngl: 100,
     embedding: false, // Always enable embedding mode on
   };
 
   // Load model config
-  return fetch(NITRO_HTTP_LOAD_MODEL_URL, {
+  return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
     },
     body: JSON.stringify(config),
+    retries: 3,
+    retryDelay: 500,
   });
 }
 
 /**
- * Checks if the model file exists.
+ * Validates the status of a model.
- * @param fileName - The name of the model file.
+ * @returns {Promise<InitModelResponse>} A promise that resolves to an object.
- * @returns A Promise that resolves when the model file exists, or rejects with an error message if the model file does not exist.
+ * If the model is loaded successfully, the object is empty.
+ * If the model is not loaded successfully, the object contains an error message.
  */
-function checkModelFileExist(fileName: string): Promise<string> {
+async function validateModelStatus(): Promise<InitModelResponse> {
-  return new Promise<string>(async (resolve, reject) => {
+  // Send a GET request to the validation URL.
-    if (!fileName) {
+  // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries.
-      reject("Model not found, please download again.");
+  return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, {
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    retries: 5,
+    retryDelay: 500,
+  })
+    .then(async (res: Response) => {
+      // If the response is OK, check model_loaded status.
+      if (res.ok) {
+        const body = await res.json();
+        // If the model is loaded, return an empty object.
+        // Otherwise, return an object with an error message.
+        if (body.model_loaded) {
+          return { error: undefined };
         }
-    resolve(fileName);
+      }
+      return { error: "Model is not loaded successfully" };
+    })
+    .catch((err) => {
+      return { error: `Model is not loaded successfully. ${err.message}` };
     });
 }
 
@@ -110,14 +122,6 @@ function killSubprocess(): Promise<void> {
   }
 }
 
-/**
- * Returns the path to the user data directory.
- * @returns The path to the user data directory.
- */
-function appPath() {
-  return app.getPath("userData");
-}
-
 /**
  * Check port is used or not, if used, attempt to unload model
  * If unload failed, kill the port

plugins/inference-plugin/webpack.config.js

@@ -18,7 +18,10 @@ module.exports = {
   plugins: [
     new webpack.DefinePlugin({
       MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
-      INFERENCE_URL: JSON.stringify(process.env.INFERENCE_URL || "http://127.0.0.1:3928/inferences/llamacpp/chat_completion"),
+      INFERENCE_URL: JSON.stringify(
+        process.env.INFERENCE_URL ||
+          "http://127.0.0.1:3928/inferences/llamacpp/chat_completion"
+      ),
     }),
   ],
   output: {
@@ -28,6 +31,9 @@ module.exports = {
   },
   resolve: {
     extensions: [".ts", ".js"],
+    fallback: {
+      path: require.resolve("path-browserify"),
+    },
   },
   optimization: {
     minimize: false,

web/hooks/useActiveModel.ts

@@ -45,7 +45,9 @@ export function useActiveModel() {
 
     const res = await initModel(`models/${modelId}`)
     if (res?.error) {
-      alert(res.error ?? 'Model loading failed.')
+      const errorMessage = `${res.error}`
+      console.error(errorMessage)
+      alert(errorMessage)
       setStateModel(() => ({
         state: 'start',
         loading: false,