Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Update nitro with read/ write for engine.json

Browse Source
This commit is contained in:
hiro 2023-12-04 12:10:46 +07:00
parent 1bc5fe64f3
commit 2c648caa5f
2 changed files with 24 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
extensions/inference-nitro-extension/src/index.ts
View File

@ -26,6 +26,13 @@ import { requestInference } from "./helpers/sse";
import { ulid } from "ulid";
import { join } from "path";
interface EngineSettings {
ctx_len: number;
ngl: number;
cont_batching: boolean;
embedding: boolean;
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
@ -35,6 +42,13 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
private static readonly _homeDir = 'engines'
private static readonly _engineMetadataFileName = 'nitro.json'
private _engineSettings: EngineSettings = {
"ctx_len": 2048,
"ngl": 100,
"cont_batching": false,
"embedding": false
}
controller = new AbortController();
isCancelled = false;
/**
@ -100,21 +114,13 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
private async writeDefaultEngineSettings() {
try {
const destPath = join(JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._engineMetadataFileName)
// TODO: Check with @louis for adding new binding
// if (await fs.checkFileExists(destPath)) {
const default_engine_settings = {
"ctx_len": 2048,
"ngl": 100,
"cont_batching": false,
"embedding": false
const engine_json = join(JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._engineMetadataFileName)
if (await fs.checkFileExists(engine_json)) {
this._engineSettings = JSON.parse(await fs.readFile(engine_json))
}
else {
await fs.writeFile(engine_json, JSON.stringify(this._engineSettings, null, 2))
}
console.log(`Writing nitro engine settings to ${destPath}`)
await fs.writeFile(destPath, JSON.stringify(default_engine_settings, null, 2))
// }
// else {
// console.log(`Using existing nitro engine settings at ${destPath}`)
// }
} catch (err) {
console.error(err)
}

4
extensions/inference-nitro-extension/src/module.ts
View File

@ -36,6 +36,9 @@ interface InitModelResponse {
* TODO: Should it be startModel instead?
*/
function initModel(wrapper: any): Promise<InitModelResponse> {
if (wrapper.settings.engine !== "llamacpp") {
return
}
// 1. Check if the model file exists
currentModelFile = wrapper.modelFullPath;
log.info("Started to load model " + wrapper.modelFullPath);
@ -45,7 +48,6 @@ function initModel(wrapper: any): Promise<InitModelResponse> {
...wrapper.settings,
};
log.info(`Load model settings: ${JSON.stringify(settings, null, 2)}`);
return (