Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: Add support for nitro 0.1.2 with load_model

Browse Source
This commit is contained in:
vuonghoainam 2023-10-19 17:44:16 -07:00
parent 25457aa594
commit d98ad60422
1 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
plugins/inference-plugin/module.ts
View File

@ -33,25 +33,16 @@ const initModel = (fileName) => {
.then(() => { .then(() => {
let binaryFolder = path.join(__dirname, "nitro"); // Current directory by default let binaryFolder = path.join(__dirname, "nitro"); // Current directory by default
// Read the existing config const config = {
const configFilePath = path.join(binaryFolder, "config", "config.json"); llama_model_path: "",
let config: any = {}; ctx_len: 2048,
if (fs.existsSync(configFilePath)) { ngl: 100,
const rawData = fs.readFileSync(configFilePath, "utf-8"); embedding: true // Always enable embedding mode on
config = JSON.parse(rawData);
}
// Update the llama_model_path
if (!config.custom_config) {
config.custom_config = {};
} }
const modelPath = path.join(app.getPath("userData"), fileName); const modelPath = path.join(app.getPath("userData"), fileName);
config.custom_config.llama_model_path = modelPath; config.llama_model_path = modelPath;
// Write the updated config back to the file
fs.writeFileSync(configFilePath, JSON.stringify(config, null, 4));
let binaryName; let binaryName;
@ -70,7 +61,7 @@ const initModel = (fileName) => {
// Execute the binary // Execute the binary
subprocess = spawn(binaryPath, [configFilePath], { cwd: binaryFolder }); subprocess = spawn(binaryPath, { cwd: binaryFolder });
// Handle subprocess output // Handle subprocess output
subprocess.stdout.on("data", (data) => { subprocess.stdout.on("data", (data) => {
@ -87,8 +78,20 @@ const initModel = (fileName) => {
}); });
}) })
.then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000)) .then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000))
.then(() => { .then((config) => {
return {}; const initModel = fetch(`http://127.0.0.1:${PORT}/inferences/llamacpp/loadmodel`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(config),
})
.then((res) => {
if (res.ok) {
return {};
}
throw new Error("Nitro: Model failed to load.");
})
}) })
.catch((err) => { .catch((err) => {
return { error: err }; return { error: err };