chore: Update n_threads to cpu_threads

core/src/types/index.ts

@@ -275,6 +275,7 @@ export type ModelSettingParams = {
   ngl?: number;
   embedding?: boolean;
   n_parallel?: number;
+  cpu_threads: number;
   system_prompt?: string;
   user_prompt?: string;
   ai_prompt?: string;

extensions/inference-nitro-extension/src/@types/global.d.ts

@@ -12,6 +12,7 @@ declare const INFERENCE_URL: string;
 interface EngineSettings {
   ctx_len: number;
   ngl: number;
+  cpu_threads: number;
   cont_batching: boolean;
   embedding: boolean;
 }

extensions/inference-nitro-extension/src/index.ts

@@ -12,7 +12,6 @@ import {
   EventName,
   MessageRequest,
   MessageStatus,
-  ModelSettingParams,
   ExtensionType,
   ThreadContent,
   ThreadMessage,
@@ -41,6 +40,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
   private static _engineSettings: EngineSettings = {
     ctx_len: 2048,
     ngl: 100,
+    cpu_threads: 1,
     cont_batching: false,
     embedding: false,
   };

extensions/inference-nitro-extension/src/module.ts

@@ -53,7 +53,7 @@ async function initModel(wrapper: any): Promise<ModelOperationResponse> {
       llama_model_path: currentModelFile,
       ...wrapper.model.settings,
       // This is critical and requires real system information
-      n_threads: nitroResourceProbe.numCpuPhysicalCore,
+      cpu_threads: nitroResourceProbe.numCpuPhysicalCore,
     };
     log.info(`Load model settings: ${JSON.stringify(settings, null, 2)}`);
     return (

extensions/inference-openai-extension/src/index.ts

@@ -12,7 +12,6 @@ import {
   EventName,
   MessageRequest,
   MessageStatus,
-  ModelSettingParams,
   ExtensionType,
   ThreadContent,
   ThreadMessage,