chore: update model setting include offload-mmproj

extensions/llamacpp-extension/src/index.ts

@@ -1548,6 +1548,26 @@ export default class llamacpp_extension extends AIEngine {
     }
   }
 
+  /**
+   * Check if mmproj.gguf file exists for a given model ID
+   * @param modelId - The model ID to check for mmproj.gguf
+   * @returns Promise<boolean> - true if mmproj.gguf exists, false otherwise
+   */
+  async checkMmprojExists(modelId: string): Promise<boolean> {
+    try {
+      const mmprojPath = await joinPath([
+        await this.getProviderPath(),
+        'models',
+        modelId,
+        'mmproj.gguf',
+      ])
+      return await fs.existsSync(mmprojPath)
+    } catch (e) {
+      logger.error(`Error checking mmproj.gguf for model ${modelId}:`, e)
+      return false
+    }
+  }
+
   async getDevices(): Promise<DeviceList[]> {
     const cfg = this.config
     const [version, backend] = cfg.version_backend.split('/')

web-app/src/containers/DropdownModelProvider.tsx

@@ -19,6 +19,7 @@ import { localStorageKey } from '@/constants/localStorage'
 import { useTranslation } from '@/i18n/react-i18next-compat'
 import { useFavoriteModel } from '@/hooks/useFavoriteModel'
 import { predefinedProviders } from '@/consts/providers'
+import { checkMmprojExists } from '@/services/models'
 
 type DropdownModelProviderProps = {
   model?: ThreadModel
@@ -66,6 +67,7 @@ const DropdownModelProvider = ({
     getModelBy,
     selectedProvider,
     selectedModel,
+    updateProvider,
   } = useModelProvider()
   const [displayModel, setDisplayModel] = useState<string>('')
   const { updateCurrentThreadModel } = useThreads()
@@ -79,31 +81,52 @@ const DropdownModelProvider = ({
   const searchInputRef = useRef<HTMLInputElement>(null)
 
   // Helper function to check if a model exists in providers
-  const checkModelExists = useCallback((providerName: string, modelId: string) => {
+  const checkModelExists = useCallback(
-    const provider = providers.find(
+    (providerName: string, modelId: string) => {
-      (p) => p.provider === providerName && p.active
+      const provider = providers.find(
-    )
+        (p) => p.provider === providerName && p.active
-    return provider?.models.find((m) => m.id === modelId)
+      )
-  }, [providers])
+      return provider?.models.find((m) => m.id === modelId)
+    },
+    [providers]
+  )
 
   // Initialize model provider only once
   useEffect(() => {
-    // Auto select model when existing thread is passed
+    const initializeModel = async () => {
-    if (model) {
+      // Auto select model when existing thread is passed
-      selectModelProvider(model?.provider as string, model?.id as string)
+      if (model) {
-      if (!checkModelExists(model.provider, model.id)) {
+        selectModelProvider(model?.provider as string, model?.id as string)
-        selectModelProvider('', '')
+        if (!checkModelExists(model.provider, model.id)) {
-      }
+          selectModelProvider('', '')
-    } else if (useLastUsedModel) {
+        }
-      // Try to use last used model only when explicitly requested (for new chat)
+        // Check mmproj existence for llamacpp models
-      const lastUsed = getLastUsedModel()
+        if (model?.provider === 'llamacpp') {
-      if (lastUsed && checkModelExists(lastUsed.provider, lastUsed.model)) {
+          await checkMmprojExists(
-        selectModelProvider(lastUsed.provider, lastUsed.model)
+            model.id as string,
-      } else {
+            updateProvider,
-        // Fallback to default model if last used model no longer exists
+            getProviderByName
-        selectModelProvider('', '')
+          )
+        }
+      } else if (useLastUsedModel) {
+        // Try to use last used model only when explicitly requested (for new chat)
+        const lastUsed = getLastUsedModel()
+        if (lastUsed && checkModelExists(lastUsed.provider, lastUsed.model)) {
+          selectModelProvider(lastUsed.provider, lastUsed.model)
+          if (lastUsed.provider === 'llamacpp') {
+            await checkMmprojExists(
+              lastUsed.model,
+              updateProvider,
+              getProviderByName
+            )
+          }
+        } else {
+          selectModelProvider('', '')
+        }
       }
     }
+
+    initializeModel()
   }, [
     model,
     selectModelProvider,
@@ -111,6 +134,8 @@ const DropdownModelProvider = ({
     providers,
     useLastUsedModel,
     checkModelExists,
+    updateProvider,
+    getProviderByName,
   ])
 
   // Update display model when selection changes
@@ -245,7 +270,7 @@ const DropdownModelProvider = ({
   }, [filteredItems, providers, searchValue, favoriteModels])
 
   const handleSelect = useCallback(
-    (searchableModel: SearchableModel) => {
+    async (searchableModel: SearchableModel) => {
       selectModelProvider(
         searchableModel.provider.provider,
         searchableModel.model.id
@@ -254,6 +279,16 @@ const DropdownModelProvider = ({
         id: searchableModel.model.id,
         provider: searchableModel.provider.provider,
       })
+
+      // Check mmproj existence for llamacpp models
+      if (searchableModel.provider.provider === 'llamacpp') {
+        await checkMmprojExists(
+          searchableModel.model.id,
+          updateProvider,
+          getProviderByName
+        )
+      }
+
       // Store the selected model as last used
       if (useLastUsedModel) {
         setLastUsedModel(
@@ -264,7 +299,13 @@ const DropdownModelProvider = ({
       setSearchValue('')
       setOpen(false)
     },
-    [selectModelProvider, updateCurrentThreadModel, useLastUsedModel]
+    [
+      selectModelProvider,
+      updateCurrentThreadModel,
+      useLastUsedModel,
+      updateProvider,
+      getProviderByName,
+    ]
   )
 
   const currentModel = selectedModel?.id

web-app/src/containers/ModelSetting.tsx

@@ -70,8 +70,8 @@ export function ModelSetting({
         models: updatedModels,
       })
 
-      // Call debounced stopModel only when updating ctx_len or ngl
+      // Call debounced stopModel only when updating ctx_len, ngl, chat_template, or offload_mmproj
-      if (key === 'ctx_len' || key === 'ngl' || key === 'chat_template') {
+      if (key === 'ctx_len' || key === 'ngl' || key === 'chat_template' || key === 'offload_mmproj') {
         debouncedStopModel(model.id)
       }
     }

web-app/src/services/models.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { sanitizeModelId } from '@/lib/utils'
 import {
   AIEngine,
@@ -341,8 +342,8 @@ export const startModel = async (
 /**
  * Check if model support tool use capability
  * Returned by backend engine
- * @param modelId 
+ * @param modelId
- * @returns 
+ * @returns
  */
 export const isToolSupported = async (modelId: string): Promise<boolean> => {
   const engine = getEngine()
@@ -350,3 +351,117 @@ export const isToolSupported = async (modelId: string): Promise<boolean> => {
 
   return engine.isToolSupported(modelId)
 }
+
+/**
+ * Checks if mmproj.gguf file exists for a given model ID in the llamacpp provider.
+ * Also checks if the model has offload_mmproj setting.
+ * If mmproj.gguf exists, adds offload_mmproj setting with value true.
+ * @param modelId - The model ID to check for mmproj.gguf
+ * @param updateProvider - Function to update the provider state
+ * @param getProviderByName - Function to get provider by name
+ * @returns Promise<{exists: boolean, settingsUpdated: boolean}> - exists: true if mmproj.gguf exists, settingsUpdated: true if settings were modified
+ */
+export const checkMmprojExists = async (
+  modelId: string,
+  updateProvider?: (providerName: string, data: Partial<ModelProvider>) => void,
+  getProviderByName?: (providerName: string) => ModelProvider | undefined
+): Promise<{ exists: boolean; settingsUpdated: boolean }> => {
+  let settingsUpdated = false
+
+  try {
+    const engine = getEngine('llamacpp') as AIEngine & {
+      checkMmprojExists?: (id: string) => Promise<boolean>
+    }
+    if (engine && typeof engine.checkMmprojExists === 'function') {
+      const exists = await engine.checkMmprojExists(modelId)
+
+      // If we have the store functions, use them; otherwise fall back to localStorage
+      if (updateProvider && getProviderByName) {
+        const provider = getProviderByName('llamacpp')
+        if (provider) {
+          const model = provider.models.find((m) => m.id === modelId)
+
+          if (model?.settings) {
+            const hasOffloadMmproj = 'offload_mmproj' in model.settings
+
+            // If mmproj exists, add offload_mmproj setting (only if it doesn't exist)
+            if (exists && !hasOffloadMmproj) {
+              // Create updated models array with the new setting
+              const updatedModels = provider.models.map((m) => {
+                if (m.id === modelId) {
+                  return {
+                    ...m,
+                    settings: {
+                      ...m.settings,
+                      offload_mmproj: {
+                        key: 'offload_mmproj',
+                        title: 'Offload MMProj',
+                        description:
+                          'Offload multimodal projection layers to GPU',
+                        controller_type: 'checkbox',
+                        controller_props: {
+                          value: true,
+                        },
+                      },
+                    },
+                  }
+                }
+                return m
+              })
+
+              // Update the provider with the new models array
+              updateProvider('llamacpp', { models: updatedModels })
+              settingsUpdated = true
+            }
+          }
+        }
+      } else {
+        // Fall back to localStorage approach for backwards compatibility
+        try {
+          const modelProviderData = JSON.parse(
+            localStorage.getItem('model-provider') || '{}'
+          )
+          const llamacppProvider = modelProviderData.state?.providers?.find(
+            (p: any) => p.provider === 'llamacpp'
+          )
+          const model = llamacppProvider?.models?.find(
+            (m: any) => m.id === modelId
+          )
+
+          if (model?.settings) {
+            // If mmproj exists, add offload_mmproj setting (only if it doesn't exist)
+            if (exists) {
+              if (!model.settings.offload_mmproj) {
+                model.settings.offload_mmproj = {
+                  key: 'offload_mmproj',
+                  title: 'Offload MMProj',
+                  description: 'Offload multimodal projection layers to GPU',
+                  controller_type: 'checkbox',
+                  controller_props: {
+                    value: true,
+                  },
+                }
+                // Save updated settings back to localStorage
+                localStorage.setItem(
+                  'model-provider',
+                  JSON.stringify(modelProviderData)
+                )
+                settingsUpdated = true
+              }
+            }
+          }
+        } catch (localStorageError) {
+          console.error(
+            `Error checking localStorage for model ${modelId}:`,
+            localStorageError
+          )
+        }
+      }
+
+      return { exists, settingsUpdated }
+    }
+  } catch (error) {
+    console.error(`Error checking mmproj for model ${modelId}:`, error)
+  }
+  return { exists: false, settingsUpdated }
+}