From da161cd1596742f77e096a8a6fbd0a1a7824608a Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 23 Apr 2024 15:09:48 +0700 Subject: [PATCH] fix: override cpu_threads setting from model.json (#2789) --- .../node/api/restful/helper/startStopModel.ts | 4 +- .../assistant-extension/src/node/engine.ts | 2 + .../src/node/index.ts | 3 +- .../monitoring-extension/src/node/logger.ts | 70 +++++++++---------- web/containers/ServerLogs/index.tsx | 2 +- .../Chat/ModelSetting/predefinedComponent.ts | 15 ++++ 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/core/src/node/api/restful/helper/startStopModel.ts b/core/src/node/api/restful/helper/startStopModel.ts index 3af0404e3..bcd182cb5 100644 --- a/core/src/node/api/restful/helper/startStopModel.ts +++ b/core/src/node/api/restful/helper/startStopModel.ts @@ -63,11 +63,11 @@ const runModel = async (modelId: string, settingParams?: ModelSettingParams): Pr const nitroResourceProbe = await getSystemResourceInfo() const nitroModelSettings: NitroModelSettings = { + // This is critical and requires real CPU physical core count (or performance core) + cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore), ...modelMetadata.settings, ...settingParams, llama_model_path: modelBinaryPath, - // This is critical and requires real CPU physical core count (or performance core) - cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore), ...(modelMetadata.settings.mmproj && { mmproj: join(modelFolderFullPath, modelMetadata.settings.mmproj), }), diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 17094ffbc..05a380340 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -15,6 +15,8 @@ export const readEmbeddingEngine = (engineName: string) => { const settingDirectoryPath = path.join( getJanDataFolderPath(), 'settings', + '@janhq', + // TODO: James - To be removed engineName === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension', diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts index 7d20ee8c7..fbfdb8761 100644 --- a/extensions/inference-nitro-extension/src/node/index.ts +++ b/extensions/inference-nitro-extension/src/node/index.ts @@ -131,10 +131,11 @@ async function loadModel( if (!llama_model_path) return Promise.reject('No GGUF model file found') currentSettings = { + cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore), + // model.settings can override the default settings ...params.model.settings, llama_model_path, // This is critical and requires real CPU physical core count (or performance core) - cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore), ...(params.model.settings.mmproj && { mmproj: path.isAbsolute(params.model.settings.mmproj) ? params.model.settings.mmproj diff --git a/extensions/monitoring-extension/src/node/logger.ts b/extensions/monitoring-extension/src/node/logger.ts index 29a391313..ca64ea2d9 100644 --- a/extensions/monitoring-extension/src/node/logger.ts +++ b/extensions/monitoring-extension/src/node/logger.ts @@ -67,54 +67,54 @@ export class FileLogger extends Logger { const size = maxFileSizeBytes ?? 1 * 1024 * 1024 // 1 MB const days = daysToKeep ?? 7 // 7 days const logDirectory = path.join(getJanDataFolderPath(), 'logs') - // Perform log cleaning const currentDate = new Date() - fs.readdir(logDirectory, (err, files) => { - if (err) { - console.error('Error reading log directory:', err) - return - } + if (fs.existsSync(logDirectory)) + fs.readdir(logDirectory, (err, files) => { + if (err) { + console.error('Error reading log directory:', err) + return + } - files.forEach((file) => { - const filePath = path.join(logDirectory, file) - fs.stat(filePath, (err, stats) => { - if (err) { - console.error('Error getting file stats:', err) - return - } + files.forEach((file) => { + const filePath = path.join(logDirectory, file) + fs.stat(filePath, (err, stats) => { + if (err) { + console.error('Error getting file stats:', err) + return + } - // Check size - if (stats.size > size) { - fs.unlink(filePath, (err) => { - if (err) { - console.error('Error deleting log file:', err) - return - } - console.debug( - `Deleted log file due to exceeding size limit: ${filePath}` - ) - }) - } else { - // Check age - const creationDate = new Date(stats.ctime) - const daysDifference = Math.floor( - (currentDate.getTime() - creationDate.getTime()) / - (1000 * 3600 * 24) - ) - if (daysDifference > days) { + // Check size + if (stats.size > size) { fs.unlink(filePath, (err) => { if (err) { console.error('Error deleting log file:', err) return } - console.debug(`Deleted old log file: ${filePath}`) + console.debug( + `Deleted log file due to exceeding size limit: ${filePath}` + ) }) + } else { + // Check age + const creationDate = new Date(stats.ctime) + const daysDifference = Math.floor( + (currentDate.getTime() - creationDate.getTime()) / + (1000 * 3600 * 24) + ) + if (daysDifference > days) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.debug(`Deleted old log file: ${filePath}`) + }) + } } - } + }) }) }) - }) // Schedule the next execution with doubled delays this.timeout = setTimeout( diff --git a/web/containers/ServerLogs/index.tsx b/web/containers/ServerLogs/index.tsx index f423a0873..f03088ae8 100644 --- a/web/containers/ServerLogs/index.tsx +++ b/web/containers/ServerLogs/index.tsx @@ -97,7 +97,7 @@ const ServerLogs = (props: ServerLogsProps) => {
- {logs.length > 1 ? ( + {logs.length > 0 ? (
{logs.slice(-limit).map((log, i) => { diff --git a/web/screens/Chat/ModelSetting/predefinedComponent.ts b/web/screens/Chat/ModelSetting/predefinedComponent.ts index a52214e38..652389d4a 100644 --- a/web/screens/Chat/ModelSetting/predefinedComponent.ts +++ b/web/screens/Chat/ModelSetting/predefinedComponent.ts @@ -165,6 +165,21 @@ export const presetConfiguration: Record = { requireModelReload: true, configType: 'setting', }, + cpu_threads: { + key: 'cpu_threads', + title: 'CPU Threads', + description: + 'Determines CPU inference threads, limited by hardware and OS. (Maximum determined by system)', + controllerType: 'slider', + controllerProps: { + min: 0, + max: 128, + step: 1, + value: 1, + }, + requireModelReload: true, + configType: 'setting', + }, // assistant chunk_size: { key: 'chunk_size',