enhancement: Add support for mmproj models (#6150)

web-app/src/routes/hub/index.tsx

@@ -288,7 +288,8 @@ function Hub() {
       const handleDownload = () => {
         // Immediately set local downloading state
         addLocalDownloadingModel(modelId)
-        pullModel(modelId, modelUrl)
+        const mmprojPath = model.mmproj_models?.[0]?.path
+        pullModel(modelId, modelUrl, mmprojPath)
       }
 
       return (
@@ -749,7 +750,10 @@ function Hub() {
                                                   )
                                                   pullModel(
                                                     variant.model_id,
-                                                    variant.path
+                                                    variant.path,
+                                                    filteredModels[
+                                                      virtualItem.index
+                                                    ].mmproj_models?.[0]?.path
                                                   )
                                                 }}
                                               >

web-app/src/services/models.ts

@@ -12,6 +12,12 @@ export interface ModelQuant {
   file_size: string
 }
 
+export interface MMProjModel {
+  model_id: string
+  path: string
+  file_size: string
+}
+
 export interface CatalogModel {
   model_name: string
   description: string
@@ -19,6 +25,7 @@ export interface CatalogModel {
   downloads: number
   num_quants: number
   quants: ModelQuant[]
+  mmproj_models?: MMProjModel[]
   created_at?: string
   readme?: string
 }
@@ -201,9 +208,14 @@ export const updateModel = async (
  * @param model The model to pull.
  * @returns A promise that resolves when the model download task is created.
  */
-export const pullModel = async (id: string, modelPath: string) => {
+export const pullModel = async (
+  id: string,
+  modelPath: string,
+  mmprojPath?: string
+) => {
   return getEngine()?.import(id, {
     modelPath,
+    mmprojPath,
   })
 }