Merge pull request #6388 from menloresearch/feat/import-vision-model

feat: allow user import model include mmproj file

extensions/llamacpp-extension/src/index.ts

@@ -1729,6 +1729,22 @@ export default class llamacpp_extension extends AIEngine {
    */
   async checkMmprojExists(modelId: string): Promise<boolean> {
     try {
+      const modelConfigPath = await joinPath([
+        await this.getProviderPath(),
+        'models',
+        modelId,
+        'model.yml',
+      ])
+
+      const modelConfig = await invoke<ModelConfig>('read_yaml', {
+        path: modelConfigPath,
+      })
+
+      // If mmproj_path is not defined in YAML, return false
+      if (modelConfig.mmproj_path) {
+        return true
+      }
+
       const mmprojPath = await joinPath([
         await this.getProviderPath(),
         'models',
@@ -1983,4 +1999,47 @@ export default class llamacpp_extension extends AIEngine {
       throw new Error(String(e))
     }
   }
+
+  /**
+   * Validate GGUF file and check for unsupported architectures like CLIP
+   */
+  async validateGgufFile(filePath: string): Promise<{
+    isValid: boolean
+    error?: string
+    metadata?: GgufMetadata
+  }> {
+    try {
+      logger.info(`Validating GGUF file: ${filePath}`)
+      const metadata = await readGgufMetadata(filePath)
+      
+      // Log full metadata for debugging
+      logger.info('Full GGUF metadata:', JSON.stringify(metadata, null, 2))
+      
+      // Check if architecture is 'clip' which is not supported for text generation
+      const architecture = metadata.metadata?.['general.architecture']
+      logger.info(`Model architecture: ${architecture}`)
+      
+      if (architecture === 'clip') {
+        const errorMessage = 'This model has CLIP architecture and cannot be imported as a text generation model. CLIP models are designed for vision tasks and require different handling.'
+        logger.error('CLIP architecture detected:', architecture)
+        return {
+          isValid: false,
+          error: errorMessage,
+          metadata
+        }
+      }
+      
+      logger.info('Model validation passed. Architecture:', architecture)
+      return {
+        isValid: true,
+        metadata
+      }
+    } catch (error) {
+      logger.error('Failed to validate GGUF file:', error)
+      return {
+        isValid: false,
+        error: `Failed to read model metadata: ${error instanceof Error ? error.message : 'Unknown error'}`
+      }
+    }
+  }
 }

web-app/src/containers/ChatInput.tsx

@@ -105,7 +105,9 @@ const ChatInput = ({ model, className, initialMessage }: ChatInputProps) => {
         try {
           // Only check mmproj for llamacpp provider
           if (selectedProvider === 'llamacpp') {
-            const hasLocalMmproj = await serviceHub.models().checkMmprojExists(selectedModel.id)
+            const hasLocalMmproj = await serviceHub
+              .models()
+              .checkMmprojExists(selectedModel.id)
             setHasMmproj(hasLocalMmproj)
           }
           // For non-llamacpp providers, only check vision capability

web-app/src/containers/DropdownModelProvider.tsx

@@ -139,7 +139,7 @@ const DropdownModelProvider = ({
     [getProviderByName, updateProvider, serviceHub]
   )
 
-  // Initialize model provider only once
+  // Initialize model provider - avoid race conditions with manual selections
   useEffect(() => {
     const initializeModel = async () => {
       // Auto select model when existing thread is passed
@@ -150,11 +150,13 @@ const DropdownModelProvider = ({
         }
         // Check mmproj existence for llamacpp models
         if (model?.provider === 'llamacpp') {
-          await serviceHub.models().checkMmprojExistsAndUpdateOffloadMMprojSetting(
-            model.id as string,
-            updateProvider,
-            getProviderByName
-          )
+          await serviceHub
+            .models()
+            .checkMmprojExistsAndUpdateOffloadMMprojSetting(
+              model.id as string,
+              updateProvider,
+              getProviderByName
+            )
           // Also check vision capability
           await checkAndUpdateModelVisionCapability(model.id as string)
         }
@@ -164,11 +166,13 @@ const DropdownModelProvider = ({
         if (lastUsed && checkModelExists(lastUsed.provider, lastUsed.model)) {
           selectModelProvider(lastUsed.provider, lastUsed.model)
           if (lastUsed.provider === 'llamacpp') {
-            await serviceHub.models().checkMmprojExistsAndUpdateOffloadMMprojSetting(
-              lastUsed.model,
-              updateProvider,
-              getProviderByName
-            )
+            await serviceHub
+              .models()
+              .checkMmprojExistsAndUpdateOffloadMMprojSetting(
+                lastUsed.model,
+                updateProvider,
+                getProviderByName
+              )
             // Also check vision capability
             await checkAndUpdateModelVisionCapability(lastUsed.model)
           }
@@ -186,19 +190,28 @@ const DropdownModelProvider = ({
           }
           selectModelProvider('', '')
         }
-      } else if (PlatformFeatures[PlatformFeature.WEB_AUTO_MODEL_SELECTION] && !selectedModel) {
-        // For web-only builds, always auto-select the first model from jan provider if none is selected
-        const janProvider = providers.find(
-          (p) => p.provider === 'jan' && p.active && p.models.length > 0
-        )
-        if (janProvider && janProvider.models.length > 0) {
-          const firstModel = janProvider.models[0]
-          selectModelProvider(janProvider.provider, firstModel.id)
+      } else {
+        // Get current state for web auto-selection check
+        const currentState = { selectedModel, selectedProvider }
+        if (
+          PlatformFeatures[PlatformFeature.WEB_AUTO_MODEL_SELECTION] &&
+          !currentState.selectedModel &&
+          !currentState.selectedProvider
+        ) {
+          // For web-only builds, auto-select the first model from jan provider only if nothing is selected
+          const janProvider = providers.find(
+            (p) => p.provider === 'jan' && p.active && p.models.length > 0
+          )
+          if (janProvider && janProvider.models.length > 0) {
+            const firstModel = janProvider.models[0]
+            selectModelProvider(janProvider.provider, firstModel.id)
+          }
         }
       }
     }
 
     initializeModel()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [
     model,
     selectModelProvider,
@@ -210,7 +223,7 @@ const DropdownModelProvider = ({
     getProviderByName,
     checkAndUpdateModelVisionCapability,
     serviceHub,
-    selectedModel,
+    // selectedModel and selectedProvider intentionally excluded to prevent race conditions
   ])
 
   // Update display model when selection changes
@@ -376,11 +389,13 @@ const DropdownModelProvider = ({
 
       // Check mmproj existence for llamacpp models
       if (searchableModel.provider.provider === 'llamacpp') {
-        await serviceHub.models().checkMmprojExistsAndUpdateOffloadMMprojSetting(
-          searchableModel.model.id,
-          updateProvider,
-          getProviderByName
-        )
+        await serviceHub
+          .models()
+          .checkMmprojExistsAndUpdateOffloadMMprojSetting(
+            searchableModel.model.id,
+            updateProvider,
+            getProviderByName
+          )
         // Also check vision capability
         await checkAndUpdateModelVisionCapability(searchableModel.model.id)
       }
@@ -572,7 +587,9 @@ const DropdownModelProvider = ({
                             {getProviderTitle(providerInfo.provider)}
                           </span>
                         </div>
-                        {PlatformFeatures[PlatformFeature.MODEL_PROVIDER_SETTINGS] && (
+                        {PlatformFeatures[
+                          PlatformFeature.MODEL_PROVIDER_SETTINGS
+                        ] && (
                           <div
                             className="size-6 cursor-pointer flex items-center justify-center rounded-sm hover:bg-main-view-fg/10 transition-all duration-200 ease-in-out"
                             onClick={(e) => {

web-app/src/containers/dialogs/ImportVisionModelDialog.tsx

@@ -0,0 +1,568 @@
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+  DialogFooter,
+} from '@/components/ui/dialog'
+import { Button } from '@/components/ui/button'
+import { Switch } from '@/components/ui/switch'
+import { useServiceHub } from '@/hooks/useServiceHub'
+import { useState } from 'react'
+import { toast } from 'sonner'
+import {
+  IconLoader2,
+  IconEye,
+  IconCheck,
+  IconAlertTriangle,
+} from '@tabler/icons-react'
+
+type ImportVisionModelDialogProps = {
+  provider: ModelProvider
+  trigger?: React.ReactNode
+  onSuccess?: (importedModelName?: string) => void
+}
+
+export const ImportVisionModelDialog = ({
+  provider,
+  trigger,
+  onSuccess,
+}: ImportVisionModelDialogProps) => {
+  const serviceHub = useServiceHub()
+  const [open, setOpen] = useState(false)
+  const [importing, setImporting] = useState(false)
+  const [isVisionModel, setIsVisionModel] = useState(false)
+  const [modelFile, setModelFile] = useState<string | null>(null)
+  const [mmProjFile, setMmProjFile] = useState<string | null>(null)
+  const [modelName, setModelName] = useState('')
+  const [validationError, setValidationError] = useState<string | null>(null)
+  const [isValidating, setIsValidating] = useState(false)
+  const [mmprojValidationError, setMmprojValidationError] = useState<
+    string | null
+  >(null)
+  const [isValidatingMmproj, setIsValidatingMmproj] = useState(false)
+
+  const validateGgufFile = async (
+    filePath: string,
+    fileType: 'model' | 'mmproj'
+  ): Promise<void> => {
+    if (fileType === 'model') {
+      setIsValidating(true)
+      setValidationError(null)
+    } else {
+      setIsValidatingMmproj(true)
+      setMmprojValidationError(null)
+    }
+
+    try {
+      console.log(`Reading GGUF metadata for ${fileType}:`, filePath)
+
+      // Handle validation differently for model files vs mmproj files
+      if (fileType === 'model') {
+        // For model files, use the standard validateGgufFile method
+        if (typeof serviceHub.models().validateGgufFile === 'function') {
+          const result = await serviceHub.models().validateGgufFile(filePath)
+
+          if (result.metadata) {
+            // Log full metadata for debugging
+            console.log(
+              `Full GGUF metadata for ${fileType}:`,
+              JSON.stringify(result.metadata, null, 2)
+            )
+
+            // Check architecture from metadata
+            const architecture =
+              result.metadata.metadata?.['general.architecture']
+            console.log(`${fileType} architecture:`, architecture)
+
+            // Model files should NOT be clip
+            if (architecture === 'clip') {
+              const errorMessage =
+                'This model has CLIP architecture and cannot be imported as a text generation model. CLIP models are designed for vision tasks and require different handling.'
+              setValidationError(errorMessage)
+              console.error(
+                'CLIP architecture detected in model file:',
+                architecture
+              )
+            } else {
+              console.log(
+                'Model validation passed. Architecture:',
+                architecture
+              )
+            }
+          }
+
+          if (!result.isValid) {
+            setValidationError(result.error || 'Model validation failed')
+            console.error('Model validation failed:', result.error)
+          }
+        }
+      } else {
+        // For mmproj files, we need to manually validate since validateGgufFile rejects CLIP models
+        try {
+          // Import the readGgufMetadata function directly from Tauri
+          const { invoke } = await import('@tauri-apps/api/core')
+
+          const metadata = await invoke('plugin:llamacpp|read_gguf_metadata', {
+            path: filePath,
+          })
+
+          console.log(
+            `Full GGUF metadata for ${fileType}:`,
+            JSON.stringify(metadata, null, 2)
+          )
+
+          // Check if architecture matches expected type
+          const architecture = (
+            metadata as { metadata?: Record<string, string> }
+          ).metadata?.['general.architecture']
+          console.log(`${fileType} architecture:`, architecture)
+
+          // MMProj files MUST be clip
+          if (architecture !== 'clip') {
+            const errorMessage = `This MMProj file has "${architecture}" architecture but should have "clip" architecture. MMProj files must be CLIP models for vision processing.`
+            setMmprojValidationError(errorMessage)
+            console.error(
+              'Non-CLIP architecture detected in mmproj file:',
+              architecture
+            )
+          } else {
+            console.log(
+              'MMProj validation passed. Architecture:',
+              architecture
+            )
+          }
+        } catch (directError) {
+          console.error('Failed to validate mmproj file directly:', directError)
+          const errorMessage = `Failed to read MMProj metadata: ${
+            directError instanceof Error ? directError.message : 'Unknown error'
+          }`
+          setMmprojValidationError(errorMessage)
+        }
+      }
+    } catch (error) {
+      console.error(`Failed to validate ${fileType} file:`, error)
+      const errorMessage = `Failed to read ${fileType} metadata: ${error instanceof Error ? error.message : 'Unknown error'}`
+
+      if (fileType === 'model') {
+        setValidationError(errorMessage)
+      } else {
+        setMmprojValidationError(errorMessage)
+      }
+    } finally {
+      if (fileType === 'model') {
+        setIsValidating(false)
+      } else {
+        setIsValidatingMmproj(false)
+      }
+    }
+  }
+
+  const validateModelFile = async (filePath: string): Promise<void> => {
+    await validateGgufFile(filePath, 'model')
+  }
+
+  const validateMmprojFile = async (filePath: string): Promise<void> => {
+    await validateGgufFile(filePath, 'mmproj')
+  }
+
+  const handleFileSelect = async (type: 'model' | 'mmproj') => {
+    const selectedFile = await serviceHub.dialog().open({
+      multiple: false,
+      directory: false,
+    })
+
+    if (selectedFile && typeof selectedFile === 'string') {
+      const fileName = selectedFile.split(/[\\/]/).pop() || ''
+
+      if (type === 'model') {
+        setModelFile(selectedFile)
+        // Auto-generate model name from GGUF file
+        const sanitizedName = fileName
+          .replace(/\s/g, '-')
+          .replace(/\.(gguf|GGUF)$/, '')
+          .replace(/[^a-zA-Z0-9/_.-]/g, '') // Remove any characters not allowed in model IDs
+        setModelName(sanitizedName)
+
+        // Validate the selected model file
+        await validateModelFile(selectedFile)
+      } else {
+        setMmProjFile(selectedFile)
+        // Validate the selected mmproj file
+        await validateMmprojFile(selectedFile)
+      }
+    }
+  }
+
+  const handleImport = async () => {
+    if (!modelFile) {
+      toast.error('Please select a model file')
+      return
+    }
+
+    if (isVisionModel && !mmProjFile) {
+      toast.error('Please select both model and MMPROJ files for vision models')
+      return
+    }
+
+    if (!modelName) {
+      toast.error('Unable to determine model name from file')
+      return
+    }
+
+    // Check if model already exists
+    const modelExists = provider.models.some(
+      (model) => model.name === modelName
+    )
+
+    if (modelExists) {
+      toast.error('Model already exists', {
+        description: `${modelName} already imported`,
+      })
+      return
+    }
+
+    setImporting(true)
+
+    try {
+      if (isVisionModel && mmProjFile) {
+        // Import vision model with both files - let backend calculate SHA256 and sizes
+        await serviceHub.models().pullModel(
+          modelName,
+          modelFile,
+          undefined, // modelSha256 - calculated by backend
+          undefined, // modelSize - calculated by backend
+          mmProjFile // mmprojPath
+          // mmprojSha256 and mmprojSize omitted - calculated by backend
+        )
+      } else {
+        // Import regular model - let backend calculate SHA256 and size
+        await serviceHub.models().pullModel(modelName, modelFile)
+      }
+
+      toast.success('Model imported successfully', {
+        description: `${modelName} has been imported`,
+      })
+
+      // Reset form and close dialog
+      resetForm()
+      setOpen(false)
+      onSuccess?.(modelName)
+    } catch (error) {
+      console.error('Import model error:', error)
+      toast.error('Failed to import model', {
+        description:
+          error instanceof Error ? error.message : 'Unknown error occurred',
+      })
+    } finally {
+      setImporting(false)
+    }
+  }
+
+  const resetForm = () => {
+    setModelFile(null)
+    setMmProjFile(null)
+    setModelName('')
+    setIsVisionModel(false)
+    setValidationError(null)
+    setIsValidating(false)
+    setMmprojValidationError(null)
+    setIsValidatingMmproj(false)
+  }
+
+  const handleOpenChange = (newOpen: boolean) => {
+    if (!importing) {
+      setOpen(newOpen)
+      if (!newOpen) {
+        resetForm()
+      }
+    }
+  }
+
+  return (
+    <Dialog open={open} onOpenChange={handleOpenChange}>
+      <DialogTrigger asChild>{trigger}</DialogTrigger>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle className="flex items-center gap-2">
+            Import Model
+          </DialogTitle>
+          <DialogDescription>
+            Import a GGUF model file to add it to your collection. Enable vision
+            support for models that work with images.
+          </DialogDescription>
+        </DialogHeader>
+
+        <div className="space-y-6">
+          {/* Vision Model Toggle Card */}
+          <div className="border border-main-view-fg/10 rounded-lg p-4 space-y-3 bg-main-view-fg/5">
+            <div className="flex items-start space-x-3">
+              <div className="flex-shrink-0 mt-0.5">
+                <IconEye size={20} className="text-accent" />
+              </div>
+              <div className="flex-1">
+                <h3 className="font-medium text-main-view-fg">
+                  Vision Model Support
+                </h3>
+                <p className="text-sm text-main-view-fg/70">
+                  Enable if your model supports image understanding (requires
+                  MMPROJ file)
+                </p>
+              </div>
+              <Switch
+                id="vision-model"
+                checked={isVisionModel}
+                onCheckedChange={(checked) => {
+                  setIsVisionModel(checked)
+                  if (!checked) {
+                    setMmProjFile(null)
+                    setMmprojValidationError(null)
+                    setIsValidatingMmproj(false)
+                  }
+                }}
+                className="mt-1"
+              />
+            </div>
+          </div>
+
+          {/* Model Name Preview */}
+          {modelName && (
+            <div className="bg-main-view-fg/5 rounded-lg p-3">
+              <div className="flex items-center gap-2">
+                <span className="text-sm font-medium text-main-view-fg/80">
+                  Model will be saved as:
+                </span>
+              </div>
+              <p className="text-base font-mono mt-1 text-main-view-fg">
+                {modelName}
+              </p>
+            </div>
+          )}
+
+          {/* File Selection Area */}
+          <div className="space-y-4">
+            {/* Model File Selection */}
+            <div className="border border-main-view-fg/10 rounded-lg p-4 space-y-3 bg-main-view-fg/5">
+              <div className="flex items-center gap-2">
+                <h3 className="font-medium text-main-view-fg">
+                  Model File (GGUF)
+                </h3>
+                <span className="text-xs bg-main-view-fg/10 text-main-view-fg/70 px-2 py-1 rounded">
+                  Required
+                </span>
+              </div>
+
+              {modelFile ? (
+                <div className="space-y-2">
+                  <div className="bg-accent/10 border border-accent/20 rounded-lg p-3">
+                    <div className="flex items-center justify-between">
+                      <div className="flex items-center gap-2">
+                        {isValidating ? (
+                          <IconLoader2
+                            size={16}
+                            className="text-accent animate-spin"
+                          />
+                        ) : validationError ? (
+                          <IconAlertTriangle
+                            size={16}
+                            className="text-destructive"
+                          />
+                        ) : (
+                          <IconCheck size={16} className="text-accent" />
+                        )}
+                        <span className="text-sm font-medium text-main-view-fg">
+                          {modelFile.split(/[\\/]/).pop()}
+                        </span>
+                      </div>
+                      <Button
+                        variant="link"
+                        size="sm"
+                        onClick={() => handleFileSelect('model')}
+                        disabled={importing || isValidating}
+                        className="text-accent hover:text-accent/80"
+                      >
+                        Change
+                      </Button>
+                    </div>
+                  </div>
+
+                  {/* Validation Error Display */}
+                  {validationError && (
+                    <div className="bg-destructive/10 border border-destructive/20 rounded-lg p-3">
+                      <div className="flex items-start gap-2">
+                        <IconAlertTriangle
+                          size={16}
+                          className="text-destructive mt-0.5 flex-shrink-0"
+                        />
+                        <div>
+                          <p className="text-sm font-medium text-destructive">
+                            Model Validation Error
+                          </p>
+                          <p className="text-sm text-destructive/90 mt-1">
+                            {validationError}
+                          </p>
+                        </div>
+                      </div>
+                    </div>
+                  )}
+
+                  {/* Validation Loading State */}
+                  {isValidating && (
+                    <div className="bg-blue-50 border border-blue-200 rounded-lg p-3">
+                      <div className="flex items-center gap-2">
+                        <IconLoader2
+                          size={16}
+                          className="text-blue-500 animate-spin"
+                        />
+                        <p className="text-sm text-blue-700">
+                          Validating model file...
+                        </p>
+                      </div>
+                    </div>
+                  )}
+                </div>
+              ) : (
+                <Button
+                  type="button"
+                  variant="link"
+                  onClick={() => handleFileSelect('model')}
+                  disabled={importing}
+                  className="w-full h-12 border border-dashed border-main-view-fg/10 bg-main-view text-main-view-fg/50 hover:text-main-view-fg"
+                >
+                  Select GGUF File
+                </Button>
+              )}
+            </div>
+
+            {/* MMPROJ File Selection - only show if vision model is enabled */}
+            {isVisionModel && (
+              <div className="border border-main-view-fg/10 rounded-lg p-4 space-y-3 bg-main-view-fg/5">
+                <div className="flex items-center gap-2">
+                  <h3 className="font-medium text-main-view-fg">MMPROJ File</h3>
+                  <span className="text-xs bg-accent/10 text-accent px-2 py-1 rounded">
+                    Required for Vision
+                  </span>
+                </div>
+
+                {mmProjFile ? (
+                  <div className="space-y-2">
+                    <div className="bg-accent/10 border border-accent/20 rounded-lg p-3">
+                      <div className="flex items-center justify-between">
+                        <div className="flex items-center gap-2">
+                          {isValidatingMmproj ? (
+                            <IconLoader2
+                              size={16}
+                              className="text-accent animate-spin"
+                            />
+                          ) : mmprojValidationError ? (
+                            <IconAlertTriangle
+                              size={16}
+                              className="text-destructive"
+                            />
+                          ) : (
+                            <IconCheck size={16} className="text-accent" />
+                          )}
+                          <span className="text-sm font-medium text-main-view-fg">
+                            {mmProjFile.split(/[\\/]/).pop()}
+                          </span>
+                        </div>
+                        <Button
+                          variant="link"
+                          size="sm"
+                          onClick={() => handleFileSelect('mmproj')}
+                          disabled={importing || isValidatingMmproj}
+                          className="text-accent hover:text-accent/80"
+                        >
+                          Change
+                        </Button>
+                      </div>
+                    </div>
+
+                    {/* MMProj Validation Error Display */}
+                    {mmprojValidationError && (
+                      <div className="bg-destructive/10 border border-destructive/20 rounded-lg p-3">
+                        <div className="flex items-start gap-2">
+                          <IconAlertTriangle
+                            size={16}
+                            className="text-destructive mt-0.5 flex-shrink-0"
+                          />
+                          <div>
+                            <p className="text-sm font-medium text-destructive">
+                              MMProj Validation Error
+                            </p>
+                            <p className="text-sm text-destructive/90 mt-1">
+                              {mmprojValidationError}
+                            </p>
+                          </div>
+                        </div>
+                      </div>
+                    )}
+
+                    {/* MMProj Validation Loading State */}
+                    {isValidatingMmproj && (
+                      <div className="bg-blue-50 border border-blue-200 rounded-lg p-3">
+                        <div className="flex items-center gap-2">
+                          <IconLoader2
+                            size={16}
+                            className="text-blue-500 animate-spin"
+                          />
+                          <p className="text-sm text-blue-700">
+                            Validating MMProj file...
+                          </p>
+                        </div>
+                      </div>
+                    )}
+                  </div>
+                ) : (
+                  <Button
+                    type="button"
+                    variant="link"
+                    onClick={() => handleFileSelect('mmproj')}
+                    disabled={importing}
+                    className="w-full h-12 border border-dashed border-main-view-fg/10 bg-main-view text-main-view-fg/50 hover:text-main-view-fg"
+                  >
+                    Select MMPROJ File
+                  </Button>
+                )}
+              </div>
+            )}
+          </div>
+        </div>
+
+        <DialogFooter className="flex gap-2 pt-4">
+          <Button
+            variant="link"
+            onClick={() => handleOpenChange(false)}
+            disabled={importing}
+            className="flex-1"
+          >
+            Cancel
+          </Button>
+          <Button
+            onClick={handleImport}
+            disabled={
+              importing ||
+              !modelFile ||
+              !modelName ||
+              (isVisionModel && !mmProjFile) ||
+              validationError !== null ||
+              isValidating ||
+              mmprojValidationError !== null ||
+              isValidatingMmproj
+            }
+            className="flex-1"
+          >
+            {importing && <IconLoader2 className="mr-2 h-4 w-4 animate-spin" />}
+            {importing ? (
+              'Importing...'
+            ) : (
+              <>Import {isVisionModel ? 'Vision ' : ''}Model</>
+            )}
+          </Button>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  )
+}

web-app/src/routes/settings/providers/$providerName.tsx

@@ -15,6 +15,7 @@ import { DynamicControllerSetting } from '@/containers/dynamicControllerSetting'
 import { RenderMarkdown } from '@/containers/RenderMarkdown'
 import { DialogEditModel } from '@/containers/dialogs/EditModel'
 import { DialogAddModel } from '@/containers/dialogs/AddModel'
+import { ImportVisionModelDialog } from '@/containers/dialogs/ImportVisionModelDialog'
 import { ModelSetting } from '@/containers/ModelSetting'
 import { DialogDeleteModel } from '@/containers/dialogs/DeleteModel'
 import { FavoriteModelAction } from '@/containers/FavoriteModelAction'
@@ -73,7 +74,6 @@ function ProviderDetail() {
   const [activeModels, setActiveModels] = useState<string[]>([])
   const [loadingModels, setLoadingModels] = useState<string[]>([])
   const [refreshingModels, setRefreshingModels] = useState(false)
-  const [importingModel, setImportingModel] = useState(false)
   const { providerName } = useParams({ from: Route.id })
   const { getProviderByName, setProviders, updateProvider } = useModelProvider()
   const provider = getProviderByName(providerName)
@@ -90,67 +90,60 @@ function ProviderDetail() {
           !setting.controller_props.value)
     )
 
-  const handleImportModel = async () => {
-    if (!provider) {
-      return
-    }
-
-    setImportingModel(true)
-    const selectedFile = await serviceHub.dialog().open({
-      multiple: false,
-      directory: false,
-    })
-    // If the dialog returns a file path, extract just the file name
-    const fileName =
-      typeof selectedFile === 'string'
-        ? selectedFile.split(/[\\/]/).pop()?.replace(/\s/g, '-')
-        : undefined
-
-    if (selectedFile && fileName) {
-      // Check if model already exists
-      const modelExists = provider.models.some(
-        (model) => model.name === fileName
-      )
-
-      if (modelExists) {
-        toast.error('Model already exists', {
-          description: `${fileName} already imported`,
-        })
-        setImportingModel(false)
-        return
-      }
-
+  const handleModelImportSuccess = async (importedModelName?: string) => {
+    // Refresh the provider to update the models list
+    await serviceHub.providers().getProviders().then(setProviders)
+    
+    // If a model was imported and it might have vision capabilities, check and update
+    if (importedModelName && providerName === 'llamacpp') {
       try {
-        await serviceHub.models().pullModel(fileName, typeof selectedFile === 'string' ? selectedFile : selectedFile?.[0])
-        // Refresh the provider to update the models list
-        await serviceHub.providers().getProviders().then(setProviders)
-        toast.success(t('providers:import'), {
-          id: `import-model-${provider.provider}`,
-          description: t('providers:importModelSuccess', {
-            provider: fileName,
-          }),
-        })
+        const mmprojExists = await serviceHub.models().checkMmprojExists(importedModelName)
+        if (mmprojExists) {
+          // Get the updated provider after refresh
+          const { getProviderByName, updateProvider: updateProviderState } = useModelProvider.getState()
+          const llamacppProvider = getProviderByName('llamacpp')
+          
+          if (llamacppProvider) {
+            const modelIndex = llamacppProvider.models.findIndex(
+              (m: Model) => m.id === importedModelName
+            )
+            if (modelIndex !== -1) {
+              const model = llamacppProvider.models[modelIndex]
+              const capabilities = model.capabilities || []
+
+              // Add 'vision' capability if not already present
+              if (!capabilities.includes('vision')) {
+                const updatedModels = [...llamacppProvider.models]
+                updatedModels[modelIndex] = {
+                  ...model,
+                  capabilities: [...capabilities, 'vision'],
+                }
+
+                updateProviderState('llamacpp', { models: updatedModels })
+                console.log(`Vision capability added to model after provider refresh: ${importedModelName}`)
+              }
+            }
+          }
+        }
       } catch (error) {
-        console.error(t('providers:importModelError'), error)
-        toast.error(t('providers:importModelError'), {
-          description:
-            error instanceof Error ? error.message : 'Unknown error occurred',
-        })
-      } finally {
-        setImportingModel(false)
+        console.error('Error checking mmproj existence after import:', error)
       }
-    } else {
-      setImportingModel(false)
     }
   }
 
   useEffect(() => {
     // Initial data fetch
-    serviceHub.models().getActiveModels().then((models) => setActiveModels(models || []))
+    serviceHub
+      .models()
+      .getActiveModels()
+      .then((models) => setActiveModels(models || []))
 
     // Set up interval for real-time updates
     const intervalId = setInterval(() => {
-      serviceHub.models().getActiveModels().then((models) => setActiveModels(models || []))
+      serviceHub
+        .models()
+        .getActiveModels()
+        .then((models) => setActiveModels(models || []))
     }, 5000)
 
     return () => clearInterval(intervalId)
@@ -199,7 +192,9 @@ function ProviderDetail() {
 
     setRefreshingModels(true)
     try {
-      const modelIds = await serviceHub.providers().fetchModelsFromProvider(provider)
+      const modelIds = await serviceHub
+        .providers()
+        .fetchModelsFromProvider(provider)
 
       // Create new models from the fetched IDs
       const newModels: Model[] = modelIds.map((id) => ({
@@ -255,10 +250,15 @@ function ProviderDetail() {
     setLoadingModels((prev) => [...prev, modelId])
     if (provider)
       // Original: startModel(provider, modelId).then(() => { setActiveModels((prevModels) => [...prevModels, modelId]) })
-      serviceHub.models().startModel(provider, modelId)
+      serviceHub
+        .models()
+        .startModel(provider, modelId)
         .then(() => {
           // Refresh active models after starting
-          serviceHub.models().getActiveModels().then((models) => setActiveModels(models || []))
+          serviceHub
+            .models()
+            .getActiveModels()
+            .then((models) => setActiveModels(models || []))
         })
         .catch((error) => {
           console.error('Error starting model:', error)
@@ -276,10 +276,15 @@ function ProviderDetail() {
 
   const handleStopModel = (modelId: string) => {
     // Original: stopModel(modelId).then(() => { setActiveModels((prevModels) => prevModels.filter((model) => model !== modelId)) })
-    serviceHub.models().stopModel(modelId)
+    serviceHub
+      .models()
+      .stopModel(modelId)
       .then(() => {
         // Refresh active models after stopping
-        serviceHub.models().getActiveModels().then((models) => setActiveModels(models || []))
+        serviceHub
+          .models()
+          .getActiveModels()
+          .then((models) => setActiveModels(models || []))
       })
       .catch((error) => {
         console.error('Error stopping model:', error)
@@ -434,10 +439,12 @@ function ProviderDetail() {
                                   }
                                 }
 
-                                serviceHub.providers().updateSettings(
-                                  providerName,
-                                  updateObj.settings ?? []
-                                )
+                                serviceHub
+                                  .providers()
+                                  .updateSettings(
+                                    providerName,
+                                    updateObj.settings ?? []
+                                  )
                                 updateProvider(providerName, {
                                   ...provider,
                                   ...updateObj,
@@ -553,32 +560,28 @@ function ProviderDetail() {
                           </>
                         )}
                         {provider && provider.provider === 'llamacpp' && (
-                          <Button
-                            variant="link"
-                            size="sm"
-                            className="hover:no-underline"
-                            disabled={importingModel}
-                            onClick={handleImportModel}
-                          >
-                            <div className="cursor-pointer flex items-center justify-center rounded hover:bg-main-view-fg/15 bg-main-view-fg/10 transition-all duration-200 ease-in-out p-1.5 py-1 gap-1 -mr-2">
-                              {importingModel ? (
-                                <IconLoader
-                                  size={18}
-                                  className="text-main-view-fg/50 animate-spin"
-                                />
-                              ) : (
-                                <IconFolderPlus
-                                  size={18}
-                                  className="text-main-view-fg/50"
-                                />
-                              )}
-                              <span className="text-main-view-fg/70">
-                                {importingModel
-                                  ? 'Importing...'
-                                  : t('providers:import')}
-                              </span>
-                            </div>
-                          </Button>
+                          <ImportVisionModelDialog
+                            provider={provider}
+                            onSuccess={handleModelImportSuccess}
+                            trigger={
+                              <Button
+                                variant="link"
+                                size="sm"
+                                className="hover:no-underline !outline-none focus:outline-none active:outline-none"
+                                asChild
+                              >
+                                <div className="cursor-pointer flex items-center justify-center rounded hover:bg-main-view-fg/15 bg-main-view-fg/10 transition-all duration-200 ease-in-out p-1.5 py-1 gap-1 -mr-2">
+                                  <IconFolderPlus
+                                    size={18}
+                                    className="text-main-view-fg/50"
+                                  />
+                                  <span className="text-main-view-fg/70">
+                                    {t('providers:import')}
+                                  </span>
+                                </div>
+                              </Button>
+                            }
+                          />
                         )}
                       </div>
                     </div>

web-app/src/services/models/default.ts

@@ -11,7 +11,13 @@ import {
   modelInfo,
 } from '@janhq/core'
 import { Model as CoreModel } from '@janhq/core'
-import type { ModelsService, ModelCatalog, HuggingFaceRepo, CatalogModel } from './types'
+import type {
+  ModelsService,
+  ModelCatalog,
+  HuggingFaceRepo,
+  CatalogModel,
+  ModelValidationResult,
+} from './types'
 
 // TODO: Replace this with the actual provider later
 const defaultProvider = 'llamacpp'
@@ -151,7 +157,9 @@ export class DefaultModelsService implements ModelsService {
 
   async updateModel(model: Partial<CoreModel>): Promise<void> {
     if (model.settings)
-      this.getEngine()?.updateSettings(model.settings as SettingComponentProps[])
+      this.getEngine()?.updateSettings(
+        model.settings as SettingComponentProps[]
+      )
   }
 
   async pullModel(
@@ -266,7 +274,10 @@ export class DefaultModelsService implements ModelsService {
     if (models) await Promise.all(models.map((model) => this.stopModel(model)))
   }
 
-  async startModel(provider: ProviderObject, model: string): Promise<SessionInfo | undefined> {
+  async startModel(
+    provider: ProviderObject,
+    model: string
+  ): Promise<SessionInfo | undefined> {
     const engine = this.getEngine(provider.provider)
     if (!engine) return undefined
 
@@ -312,7 +323,10 @@ export class DefaultModelsService implements ModelsService {
 
   async checkMmprojExistsAndUpdateOffloadMMprojSetting(
     modelId: string,
-    updateProvider?: (providerName: string, data: Partial<ModelProvider>) => void,
+    updateProvider?: (
+      providerName: string,
+      data: Partial<ModelProvider>
+    ) => void,
     getProviderByName?: (providerName: string) => ModelProvider | undefined
   ): Promise<{ exists: boolean; settingsUpdated: boolean }> {
     let settingsUpdated = false
@@ -374,7 +388,8 @@ export class DefaultModelsService implements ModelsService {
               (p: { provider: string }) => p.provider === 'llamacpp'
             )
             const model = llamacppProvider?.models?.find(
-              (m: { id: string; settings?: Record<string, unknown> }) => m.id === modelId
+              (m: { id: string; settings?: Record<string, unknown> }) =>
+                m.id === modelId
             )
 
             if (model?.settings) {
@@ -429,7 +444,10 @@ export class DefaultModelsService implements ModelsService {
     return false
   }
 
-  async isModelSupported(modelPath: string, ctxSize?: number): Promise<'RED' | 'YELLOW' | 'GREEN' | 'GREY'> {
+  async isModelSupported(
+    modelPath: string,
+    ctxSize?: number
+  ): Promise<'RED' | 'YELLOW' | 'GREEN' | 'GREY'> {
     try {
       const engine = this.getEngine('llamacpp') as AIEngine & {
         isModelSupported?: (
@@ -448,4 +466,29 @@ export class DefaultModelsService implements ModelsService {
       return 'GREY' // Error state, assume not supported
     }
   }
-}
+
+  async validateGgufFile(filePath: string): Promise<ModelValidationResult> {
+    try {
+      const engine = this.getEngine('llamacpp') as AIEngine & {
+        validateGgufFile?: (path: string) => Promise<ModelValidationResult>
+      }
+
+      if (engine && typeof engine.validateGgufFile === 'function') {
+        return await engine.validateGgufFile(filePath)
+      }
+
+      // If the specific method isn't available, we can fallback to a basic check
+      console.warn('validateGgufFile method not available in llamacpp engine')
+      return {
+        isValid: true, // Assume valid for now
+        error: 'Validation method not available',
+      }
+    } catch (error) {
+      console.error(`Error validating GGUF file ${filePath}:`, error)
+      return {
+        isValid: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      }
+    }
+  }
+}

web-app/src/services/models/types.ts

@@ -69,6 +69,18 @@ export interface HuggingFaceRepo {
   readme?: string
 }
 
+export interface GgufMetadata {
+  version: number
+  tensor_count: number
+  metadata: Record<string, string>
+}
+
+export interface ModelValidationResult {
+  isValid: boolean
+  error?: string
+  metadata?: GgufMetadata
+}
+
 export interface ModelsService {
   fetchModels(): Promise<modelInfo[]>
   fetchModelCatalog(): Promise<ModelCatalog>
@@ -104,4 +116,5 @@ export interface ModelsService {
   ): Promise<{ exists: boolean; settingsUpdated: boolean }>
   checkMmprojExists(modelId: string): Promise<boolean>
   isModelSupported(modelPath: string, ctxSize?: number): Promise<'RED' | 'YELLOW' | 'GREEN' | 'GREY'>
+  validateGgufFile(filePath: string): Promise<ModelValidationResult>
 }