diff --git a/core/src/index.ts b/core/src/index.ts index cba3efe92..39a69d702 100644 --- a/core/src/index.ts +++ b/core/src/index.ts @@ -8,7 +8,7 @@ export { core, deleteFile, invokePluginFunc } from "./core"; * Core module exports. * @module */ -export { downloadFile, executeOnMain } from "./core"; +export { downloadFile, executeOnMain, appDataPath } from "./core"; /** * Events module exports. diff --git a/plugins/inference-plugin/nitro/version.txt b/plugins/inference-plugin/nitro/version.txt index a1e1395ac..84aa3a7dd 100644 --- a/plugins/inference-plugin/nitro/version.txt +++ b/plugins/inference-plugin/nitro/version.txt @@ -1 +1 @@ -0.1.7 \ No newline at end of file +0.1.8 \ No newline at end of file diff --git a/plugins/inference-plugin/package.json b/plugins/inference-plugin/package.json index 95a520d02..5c52d13d0 100644 --- a/plugins/inference-plugin/package.json +++ b/plugins/inference-plugin/package.json @@ -39,7 +39,9 @@ "dependencies": { "@janhq/core": "file:../../core", "download-cli": "^1.1.1", + "fetch-retry": "^5.0.6", "kill-port": "^2.0.1", + "path-browserify": "^1.0.1", "rxjs": "^7.8.1", "tcp-port-used": "^1.0.2", "ts-loader": "^9.5.0", @@ -55,6 +57,7 @@ ], "bundleDependencies": [ "tcp-port-used", - "kill-port" + "kill-port", + "fetch-retry" ] } diff --git a/plugins/inference-plugin/src/index.ts b/plugins/inference-plugin/src/index.ts index 80857ec42..ebd44657f 100644 --- a/plugins/inference-plugin/src/index.ts +++ b/plugins/inference-plugin/src/index.ts @@ -17,6 +17,8 @@ import { import { InferencePlugin } from "@janhq/core/lib/plugins"; import { requestInference } from "./helpers/sse"; import { ulid } from "ulid"; +import { join } from "path"; +import { appDataPath } from "@janhq/core"; /** * A class that implements the InferencePlugin interface from the @janhq/core package. @@ -48,18 +50,19 @@ export default class JanInferencePlugin implements InferencePlugin { /** * Initializes the model with the specified file name. - * @param {string} modelFileName - The name of the model file. + * @param {string} modelFileName - The file name of the model file. * @returns {Promise} A promise that resolves when the model is initialized. */ - initModel(modelFileName: string): Promise { - return executeOnMain(MODULE, "initModel", modelFileName); + async initModel(modelFileName: string): Promise { + const appPath = await appDataPath(); + return executeOnMain(MODULE, "initModel", join(appPath, modelFileName)); } /** * Stops the model. * @returns {Promise} A promise that resolves when the model is stopped. */ - stopModel(): Promise { + async stopModel(): Promise { return executeOnMain(MODULE, "killSubprocess"); } diff --git a/plugins/inference-plugin/src/module.ts b/plugins/inference-plugin/src/module.ts index 5f1b3e2af..98fe43694 100644 --- a/plugins/inference-plugin/src/module.ts +++ b/plugins/inference-plugin/src/module.ts @@ -1,9 +1,9 @@ const fs = require("fs"); const kill = require("kill-port"); const path = require("path"); -const { app } = require("electron"); const { spawn } = require("child_process"); const tcpPortUsed = require("tcp-port-used"); +const fetchRetry = require("fetch-retry")(global.fetch); // The PORT to use for the Nitro subprocess const PORT = 3928; @@ -11,9 +11,11 @@ const LOCAL_HOST = "127.0.0.1"; const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; const NITRO_HTTP_UNLOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/unloadModel`; +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; // The subprocess instance for Nitro let subprocess = null; +let currentModelFile = null; /** * The response from the initModel function. @@ -25,75 +27,85 @@ interface InitModelResponse { /** * Initializes a Nitro subprocess to load a machine learning model. - * @param fileName - The name of the machine learning model file. + * @param modelFile - The name of the machine learning model file. * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package * TODO: Should it be startModel instead? */ -function initModel(fileName: string): Promise { +function initModel(modelFile: string): Promise { // 1. Check if the model file exists + currentModelFile = modelFile; + return ( - checkModelFileExist(fileName) - // 2. Check if the port is used, if used, attempt to unload model / kill nitro process - .then(checkAndUnloadNitro) - // 3. Spawn the Nitro subprocess + // 1. Check if the port is used, if used, attempt to unload model / kill nitro process + checkAndUnloadNitro() + // 2. Spawn the Nitro subprocess .then(spawnNitroProcess) - // 4. Wait until the port is used (Nitro http server is up) + // 3. Wait until the port is used (Nitro http server is up) .then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000)) - // 5. Load the model into the Nitro subprocess (HTTP POST request) - .then(() => loadLLMModel(fileName)) - // 6. Check if the model is loaded successfully - .then(async (res) => { - if (res.ok) { - // Success - Model loaded - return {}; - } - const json = await res.json(); - throw new Error(`${json?.message ?? "Model loading failed."}`); - }) - .catch((err) => { - return { error: err }; - }) + // 4. Load the model into the Nitro subprocess (HTTP POST request) + .then(loadLLMModel) + // 5. Check if the model is loaded successfully + .then(validateModelStatus) ); } /** * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @param fileName - The name of the model file. * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. */ -function loadLLMModel(fileName: string): Promise { - const llama_model_path = path.join(appPath(), fileName); - +function loadLLMModel(): Promise { const config = { - llama_model_path, + llama_model_path: currentModelFile, ctx_len: 2048, ngl: 100, embedding: false, // Always enable embedding mode on }; // Load model config - return fetch(NITRO_HTTP_LOAD_MODEL_URL, { + return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify(config), + retries: 3, + retryDelay: 500, }); } /** - * Checks if the model file exists. - * @param fileName - The name of the model file. - * @returns A Promise that resolves when the model file exists, or rejects with an error message if the model file does not exist. + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. */ -function checkModelFileExist(fileName: string): Promise { - return new Promise(async (resolve, reject) => { - if (!fileName) { - reject("Model not found, please download again."); - } - resolve(fileName); - }); +async function validateModelStatus(): Promise { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }) + .then(async (res: Response) => { + // If the response is OK, check model_loaded status. + if (res.ok) { + const body = await res.json(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return { error: undefined }; + } + } + return { error: "Model is not loaded successfully" }; + }) + .catch((err) => { + return { error: `Model is not loaded successfully. ${err.message}` }; + }); } /** @@ -110,14 +122,6 @@ function killSubprocess(): Promise { } } -/** - * Returns the path to the user data directory. - * @returns The path to the user data directory. - */ -function appPath() { - return app.getPath("userData"); -} - /** * Check port is used or not, if used, attempt to unload model * If unload failed, kill the port diff --git a/plugins/inference-plugin/webpack.config.js b/plugins/inference-plugin/webpack.config.js index f6f32a263..45be62271 100644 --- a/plugins/inference-plugin/webpack.config.js +++ b/plugins/inference-plugin/webpack.config.js @@ -18,7 +18,10 @@ module.exports = { plugins: [ new webpack.DefinePlugin({ MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - INFERENCE_URL: JSON.stringify(process.env.INFERENCE_URL || "http://127.0.0.1:3928/inferences/llamacpp/chat_completion"), + INFERENCE_URL: JSON.stringify( + process.env.INFERENCE_URL || + "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" + ), }), ], output: { @@ -28,6 +31,9 @@ module.exports = { }, resolve: { extensions: [".ts", ".js"], + fallback: { + path: require.resolve("path-browserify"), + }, }, optimization: { minimize: false, diff --git a/web/hooks/useActiveModel.ts b/web/hooks/useActiveModel.ts index d40ff0582..a6a617623 100644 --- a/web/hooks/useActiveModel.ts +++ b/web/hooks/useActiveModel.ts @@ -45,7 +45,9 @@ export function useActiveModel() { const res = await initModel(`models/${modelId}`) if (res?.error) { - alert(res.error ?? 'Model loading failed.') + const errorMessage = `${res.error}` + console.error(errorMessage) + alert(errorMessage) setStateModel(() => ({ state: 'start', loading: false,