diff --git a/web-app/src/containers/ModelSetting.tsx b/web-app/src/containers/ModelSetting.tsx index bcbd3dbd8..a02b9df80 100644 --- a/web-app/src/containers/ModelSetting.tsx +++ b/web-app/src/containers/ModelSetting.tsx @@ -55,11 +55,7 @@ export function ModelSetting({ if (modelConfig && modelConfig.model_path) { const result = await serviceHub .models() - .planModelLoad( - modelConfig.model_path, - undefined, - modelConfig.mmproj_path - ) + .planModelLoad(modelConfig.model_path, modelConfig.mmproj_path) // Apply the recommended settings to the model sequentially to avoid race conditions const settingsToUpdate: Array<{ @@ -78,6 +74,15 @@ export function ModelSetting({ }) } + if ( + model.settings?.no_kv_offload && + result.noOffloadKVCache !== undefined + ) { + settingsToUpdate.push({ + key: 'no_kv_offload', + value: result.noOffloadKVCache, + }) + } if ( model.settings?.no_kv_offload && result.noOffloadKVCache !== undefined diff --git a/web-app/src/services/models/default.ts b/web-app/src/services/models/default.ts index 54595d448..9809428b4 100644 --- a/web-app/src/services/models/default.ts +++ b/web-app/src/services/models/default.ts @@ -495,12 +495,14 @@ export class DefaultModelsService implements ModelsService { async planModelLoad( modelPath: string, + mmprojPath?: string, requestedCtx?: number ): Promise { try { const engine = this.getEngine('llamacpp') as AIEngine & { planModelLoad?: ( path: string, + mmprojPath?: string, requestedCtx?: number ) => Promise } @@ -514,7 +516,12 @@ export class DefaultModelsService implements ModelsService { (core) => core.joinPath ) const fullModelPath = await joinPath([janDataFolderPath, modelPath]) - return await engine.planModelLoad(fullModelPath, requestedCtx) + // mmprojPath is currently unused, but included for compatibility + return await engine.planModelLoad( + fullModelPath, + mmprojPath, + requestedCtx + ) } // Fallback if method is not available diff --git a/web-app/src/services/models/types.ts b/web-app/src/services/models/types.ts index d889e41ba..b7959227a 100644 --- a/web-app/src/services/models/types.ts +++ b/web-app/src/services/models/types.ts @@ -138,7 +138,7 @@ export interface ModelsService { validateGgufFile(filePath: string): Promise planModelLoad( modelPath: string, - requestedCtx?: number, - mmprojPath?: string + mmprojPath?: string, + requestedCtx?: number ): Promise }