diff --git a/extensions/llamacpp-extension/src/index.ts b/extensions/llamacpp-extension/src/index.ts index 725731bd7..f5dfcdd09 100644 --- a/extensions/llamacpp-extension/src/index.ts +++ b/extensions/llamacpp-extension/src/index.ts @@ -1729,6 +1729,22 @@ export default class llamacpp_extension extends AIEngine { */ async checkMmprojExists(modelId: string): Promise { try { + const modelConfigPath = await joinPath([ + await this.getProviderPath(), + 'models', + modelId, + 'model.yml', + ]) + + const modelConfig = await invoke('read_yaml', { + path: modelConfigPath, + }) + + // If mmproj_path is not defined in YAML, return false + if (modelConfig.mmproj_path) { + return true + } + const mmprojPath = await joinPath([ await this.getProviderPath(), 'models', diff --git a/web-app/src/containers/dialogs/ImportVisionModelDialog.tsx b/web-app/src/containers/dialogs/ImportVisionModelDialog.tsx index 82f1c5cd1..840da9c8e 100644 --- a/web-app/src/containers/dialogs/ImportVisionModelDialog.tsx +++ b/web-app/src/containers/dialogs/ImportVisionModelDialog.tsx @@ -22,7 +22,7 @@ import { type ImportVisionModelDialogProps = { provider: ModelProvider trigger?: React.ReactNode - onSuccess?: () => void + onSuccess?: (importedModelName?: string) => void } export const ImportVisionModelDialog = ({ @@ -114,7 +114,7 @@ export const ImportVisionModelDialog = ({ // Reset form and close dialog resetForm() setOpen(false) - onSuccess?.() + onSuccess?.(modelName) } catch (error) { console.error('Import model error:', error) toast.error('Failed to import model', { diff --git a/web-app/src/routes/settings/providers/$providerName.tsx b/web-app/src/routes/settings/providers/$providerName.tsx index 7774b02a1..6cad024a4 100644 --- a/web-app/src/routes/settings/providers/$providerName.tsx +++ b/web-app/src/routes/settings/providers/$providerName.tsx @@ -90,9 +90,45 @@ function ProviderDetail() { !setting.controller_props.value) ) - const handleModelImportSuccess = async () => { + const handleModelImportSuccess = async (importedModelName?: string) => { // Refresh the provider to update the models list await serviceHub.providers().getProviders().then(setProviders) + + // If a model was imported and it might have vision capabilities, check and update + if (importedModelName && providerName === 'llamacpp') { + try { + const mmprojExists = await serviceHub.models().checkMmprojExists(importedModelName) + if (mmprojExists) { + // Get the updated provider after refresh + const { getProviderByName, updateProvider: updateProviderState } = useModelProvider.getState() + const llamacppProvider = getProviderByName('llamacpp') + + if (llamacppProvider) { + const modelIndex = llamacppProvider.models.findIndex( + (m: Model) => m.id === importedModelName + ) + if (modelIndex !== -1) { + const model = llamacppProvider.models[modelIndex] + const capabilities = model.capabilities || [] + + // Add 'vision' capability if not already present + if (!capabilities.includes('vision')) { + const updatedModels = [...llamacppProvider.models] + updatedModels[modelIndex] = { + ...model, + capabilities: [...capabilities, 'vision'], + } + + updateProviderState('llamacpp', { models: updatedModels }) + console.log(`Vision capability added to model after provider refresh: ${importedModelName}`) + } + } + } + } + } catch (error) { + console.error('Error checking mmproj existence after import:', error) + } + } } useEffect(() => {