From fdc8e07f86fd29c2bbd519b5c1d8c7bb1bc85393 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Mon, 18 Aug 2025 13:46:25 +0700 Subject: [PATCH] chore: update model setting include offload-mmproj --- extensions/llamacpp-extension/src/index.ts | 20 +++ .../src/containers/DropdownModelProvider.tsx | 85 +++++++++---- web-app/src/containers/ModelSetting.tsx | 4 +- web-app/src/services/models.ts | 119 +++++++++++++++++- 4 files changed, 202 insertions(+), 26 deletions(-) diff --git a/extensions/llamacpp-extension/src/index.ts b/extensions/llamacpp-extension/src/index.ts index 3d124b37b..e4889ff2e 100644 --- a/extensions/llamacpp-extension/src/index.ts +++ b/extensions/llamacpp-extension/src/index.ts @@ -1548,6 +1548,26 @@ export default class llamacpp_extension extends AIEngine { } } + /** + * Check if mmproj.gguf file exists for a given model ID + * @param modelId - The model ID to check for mmproj.gguf + * @returns Promise - true if mmproj.gguf exists, false otherwise + */ + async checkMmprojExists(modelId: string): Promise { + try { + const mmprojPath = await joinPath([ + await this.getProviderPath(), + 'models', + modelId, + 'mmproj.gguf', + ]) + return await fs.existsSync(mmprojPath) + } catch (e) { + logger.error(`Error checking mmproj.gguf for model ${modelId}:`, e) + return false + } + } + async getDevices(): Promise { const cfg = this.config const [version, backend] = cfg.version_backend.split('/') diff --git a/web-app/src/containers/DropdownModelProvider.tsx b/web-app/src/containers/DropdownModelProvider.tsx index 1425061d9..e4db8d63f 100644 --- a/web-app/src/containers/DropdownModelProvider.tsx +++ b/web-app/src/containers/DropdownModelProvider.tsx @@ -19,6 +19,7 @@ import { localStorageKey } from '@/constants/localStorage' import { useTranslation } from '@/i18n/react-i18next-compat' import { useFavoriteModel } from '@/hooks/useFavoriteModel' import { predefinedProviders } from '@/consts/providers' +import { checkMmprojExists } from '@/services/models' type DropdownModelProviderProps = { model?: ThreadModel @@ -66,6 +67,7 @@ const DropdownModelProvider = ({ getModelBy, selectedProvider, selectedModel, + updateProvider, } = useModelProvider() const [displayModel, setDisplayModel] = useState('') const { updateCurrentThreadModel } = useThreads() @@ -79,31 +81,52 @@ const DropdownModelProvider = ({ const searchInputRef = useRef(null) // Helper function to check if a model exists in providers - const checkModelExists = useCallback((providerName: string, modelId: string) => { - const provider = providers.find( - (p) => p.provider === providerName && p.active - ) - return provider?.models.find((m) => m.id === modelId) - }, [providers]) + const checkModelExists = useCallback( + (providerName: string, modelId: string) => { + const provider = providers.find( + (p) => p.provider === providerName && p.active + ) + return provider?.models.find((m) => m.id === modelId) + }, + [providers] + ) // Initialize model provider only once useEffect(() => { - // Auto select model when existing thread is passed - if (model) { - selectModelProvider(model?.provider as string, model?.id as string) - if (!checkModelExists(model.provider, model.id)) { - selectModelProvider('', '') - } - } else if (useLastUsedModel) { - // Try to use last used model only when explicitly requested (for new chat) - const lastUsed = getLastUsedModel() - if (lastUsed && checkModelExists(lastUsed.provider, lastUsed.model)) { - selectModelProvider(lastUsed.provider, lastUsed.model) - } else { - // Fallback to default model if last used model no longer exists - selectModelProvider('', '') + const initializeModel = async () => { + // Auto select model when existing thread is passed + if (model) { + selectModelProvider(model?.provider as string, model?.id as string) + if (!checkModelExists(model.provider, model.id)) { + selectModelProvider('', '') + } + // Check mmproj existence for llamacpp models + if (model?.provider === 'llamacpp') { + await checkMmprojExists( + model.id as string, + updateProvider, + getProviderByName + ) + } + } else if (useLastUsedModel) { + // Try to use last used model only when explicitly requested (for new chat) + const lastUsed = getLastUsedModel() + if (lastUsed && checkModelExists(lastUsed.provider, lastUsed.model)) { + selectModelProvider(lastUsed.provider, lastUsed.model) + if (lastUsed.provider === 'llamacpp') { + await checkMmprojExists( + lastUsed.model, + updateProvider, + getProviderByName + ) + } + } else { + selectModelProvider('', '') + } } } + + initializeModel() }, [ model, selectModelProvider, @@ -111,6 +134,8 @@ const DropdownModelProvider = ({ providers, useLastUsedModel, checkModelExists, + updateProvider, + getProviderByName, ]) // Update display model when selection changes @@ -245,7 +270,7 @@ const DropdownModelProvider = ({ }, [filteredItems, providers, searchValue, favoriteModels]) const handleSelect = useCallback( - (searchableModel: SearchableModel) => { + async (searchableModel: SearchableModel) => { selectModelProvider( searchableModel.provider.provider, searchableModel.model.id @@ -254,6 +279,16 @@ const DropdownModelProvider = ({ id: searchableModel.model.id, provider: searchableModel.provider.provider, }) + + // Check mmproj existence for llamacpp models + if (searchableModel.provider.provider === 'llamacpp') { + await checkMmprojExists( + searchableModel.model.id, + updateProvider, + getProviderByName + ) + } + // Store the selected model as last used if (useLastUsedModel) { setLastUsedModel( @@ -264,7 +299,13 @@ const DropdownModelProvider = ({ setSearchValue('') setOpen(false) }, - [selectModelProvider, updateCurrentThreadModel, useLastUsedModel] + [ + selectModelProvider, + updateCurrentThreadModel, + useLastUsedModel, + updateProvider, + getProviderByName, + ] ) const currentModel = selectedModel?.id diff --git a/web-app/src/containers/ModelSetting.tsx b/web-app/src/containers/ModelSetting.tsx index 29d996382..b3bb55e40 100644 --- a/web-app/src/containers/ModelSetting.tsx +++ b/web-app/src/containers/ModelSetting.tsx @@ -70,8 +70,8 @@ export function ModelSetting({ models: updatedModels, }) - // Call debounced stopModel only when updating ctx_len or ngl - if (key === 'ctx_len' || key === 'ngl' || key === 'chat_template') { + // Call debounced stopModel only when updating ctx_len, ngl, chat_template, or offload_mmproj + if (key === 'ctx_len' || key === 'ngl' || key === 'chat_template' || key === 'offload_mmproj') { debouncedStopModel(model.id) } } diff --git a/web-app/src/services/models.ts b/web-app/src/services/models.ts index 4692a82ad..9829663f4 100644 --- a/web-app/src/services/models.ts +++ b/web-app/src/services/models.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { sanitizeModelId } from '@/lib/utils' import { AIEngine, @@ -341,8 +342,8 @@ export const startModel = async ( /** * Check if model support tool use capability * Returned by backend engine - * @param modelId - * @returns + * @param modelId + * @returns */ export const isToolSupported = async (modelId: string): Promise => { const engine = getEngine() @@ -350,3 +351,117 @@ export const isToolSupported = async (modelId: string): Promise => { return engine.isToolSupported(modelId) } + +/** + * Checks if mmproj.gguf file exists for a given model ID in the llamacpp provider. + * Also checks if the model has offload_mmproj setting. + * If mmproj.gguf exists, adds offload_mmproj setting with value true. + * @param modelId - The model ID to check for mmproj.gguf + * @param updateProvider - Function to update the provider state + * @param getProviderByName - Function to get provider by name + * @returns Promise<{exists: boolean, settingsUpdated: boolean}> - exists: true if mmproj.gguf exists, settingsUpdated: true if settings were modified + */ +export const checkMmprojExists = async ( + modelId: string, + updateProvider?: (providerName: string, data: Partial) => void, + getProviderByName?: (providerName: string) => ModelProvider | undefined +): Promise<{ exists: boolean; settingsUpdated: boolean }> => { + let settingsUpdated = false + + try { + const engine = getEngine('llamacpp') as AIEngine & { + checkMmprojExists?: (id: string) => Promise + } + if (engine && typeof engine.checkMmprojExists === 'function') { + const exists = await engine.checkMmprojExists(modelId) + + // If we have the store functions, use them; otherwise fall back to localStorage + if (updateProvider && getProviderByName) { + const provider = getProviderByName('llamacpp') + if (provider) { + const model = provider.models.find((m) => m.id === modelId) + + if (model?.settings) { + const hasOffloadMmproj = 'offload_mmproj' in model.settings + + // If mmproj exists, add offload_mmproj setting (only if it doesn't exist) + if (exists && !hasOffloadMmproj) { + // Create updated models array with the new setting + const updatedModels = provider.models.map((m) => { + if (m.id === modelId) { + return { + ...m, + settings: { + ...m.settings, + offload_mmproj: { + key: 'offload_mmproj', + title: 'Offload MMProj', + description: + 'Offload multimodal projection layers to GPU', + controller_type: 'checkbox', + controller_props: { + value: true, + }, + }, + }, + } + } + return m + }) + + // Update the provider with the new models array + updateProvider('llamacpp', { models: updatedModels }) + settingsUpdated = true + } + } + } + } else { + // Fall back to localStorage approach for backwards compatibility + try { + const modelProviderData = JSON.parse( + localStorage.getItem('model-provider') || '{}' + ) + const llamacppProvider = modelProviderData.state?.providers?.find( + (p: any) => p.provider === 'llamacpp' + ) + const model = llamacppProvider?.models?.find( + (m: any) => m.id === modelId + ) + + if (model?.settings) { + // If mmproj exists, add offload_mmproj setting (only if it doesn't exist) + if (exists) { + if (!model.settings.offload_mmproj) { + model.settings.offload_mmproj = { + key: 'offload_mmproj', + title: 'Offload MMProj', + description: 'Offload multimodal projection layers to GPU', + controller_type: 'checkbox', + controller_props: { + value: true, + }, + } + // Save updated settings back to localStorage + localStorage.setItem( + 'model-provider', + JSON.stringify(modelProviderData) + ) + settingsUpdated = true + } + } + } + } catch (localStorageError) { + console.error( + `Error checking localStorage for model ${modelId}:`, + localStorageError + ) + } + } + + return { exists, settingsUpdated } + } + } catch (error) { + console.error(`Error checking mmproj for model ${modelId}:`, error) + } + return { exists: false, settingsUpdated } +}