Add auto unload setting to llamacpp-extension

extensions/llamacpp-extension/settings.json

@@ -9,6 +9,13 @@
       "options": []
     }
   },
+  {
+    "key": "auto_unload_models",
+    "title": "Auto-Unload Old Models",
+    "description": "Automatically unloads models that are not in use to free up memory. Ensure only one model is loaded at a time.",
+    "controllerType": "checkbox",
+    "controllerProps": { "value": true }
+  },
   {
     "key": "threads",
     "title": "Threads",
@@ -145,7 +152,7 @@
     "description": "Enable continuous batching (a.k.a dynamic batching) for concurrent requests (default: enabled).",
     "controllerType": "checkbox",
     "controllerProps": {
-      "value": true
+      "value": false
     }
   },
   {

extensions/llamacpp-extension/src/index.ts

@@ -30,6 +30,7 @@ import { invoke } from '@tauri-apps/api/core'
 
 type LlamacppConfig = {
   version_backend: string
+  auto_unload: boolean
   n_gpu_layers: number
   ctx_size: number
   threads: number
@@ -106,6 +107,7 @@ interface EmbeddingData {
 
 export default class llamacpp_extension extends AIEngine {
   provider: string = 'llamacpp'
+  autoUnload: boolean = true
   readonly providerId: string = 'llamacpp'
 
   private config: LlamacppConfig
@@ -132,7 +134,7 @@ export default class llamacpp_extension extends AIEngine {
         })
       }
     }
-
+    this.autoUnload = await this.getSetting<boolean>('auto_unload_models', true)
     this.registerSettings(settings)
 
     let config = {}