fix: migrate new models (#1034)

* fix: migrate new models

* fix: filter out invalid model files

* chore: migrate models

core/src/api/index.ts

@@ -42,6 +42,7 @@ export enum ExtensionRoute {
 export enum FileSystemRoute {
   appendFile = 'appendFile',
   copyFile = 'copyFile',
+  syncFile = 'syncFile',
   deleteFile = 'deleteFile',
   exists = 'exists',
   getResourcePath = 'getResourcePath',

core/src/fs.ts

@@ -63,6 +63,8 @@ const appendFile: (path: string, data: string) => Promise<any> = (path, data) =>
 const copyFile: (src: string, dest: string) => Promise<any> = (src, dest) =>
   global.core.api?.copyFile(src, dest)
 
+const syncFile: (src: string, dest: string) => Promise<any> = (src, dest) =>
+  global.core.api?.syncFile(src, dest)
 /**
  * Reads a file line by line.
  * @param {string} path - The path of the file to read.
@@ -83,4 +85,5 @@ export const fs = {
   appendFile,
   readLineByLine,
   copyFile,
+  syncFile,
 }

electron/handlers/fs.ts

@@ -5,6 +5,7 @@ import { join } from 'path'
 import readline from 'readline'
 import { userSpacePath } from './../utils/path'
 import { FileSystemRoute } from '@janhq/core'
+const reflect = require('@alumna/reflect')
 
 /**
  * Handles file system operations.
@@ -175,12 +176,32 @@ export function handleFsIPCs() {
     }
   )
 
+  ipcMain.handle(
+    FileSystemRoute.syncFile,
+    async (_event, src: string, dest: string) => {
+      console.debug(`Copying file from ${src} to ${dest}`)
+
+      return reflect({
+        src,
+        dest,
+        recursive: true,
+        delete: false,
+        overwrite: true,
+        errorOnExist: false,
+      })
+    }
+  )
+
   ipcMain.handle(
     FileSystemRoute.copyFile,
     async (_event, src: string, dest: string) => {
       console.debug(`Copying file from ${src} to ${dest}`)
 
-      return fse.copySync(src, dest, { overwrite: false })
+      return fse.copySync(src, dest, {
+        overwrite: false,
+        recursive: true,
+        errorOnExist: false,
+      })
     }
   )
 

electron/package.json

@@ -67,6 +67,7 @@
     "build:publish:linux": "tsc -p . && electron-builder -p onTagOrDraft -l deb"
   },
   "dependencies": {
+    "@alumna/reflect": "^1.1.3",
     "@janhq/core": "link:./core",
     "@npmcli/arborist": "^7.1.0",
     "@types/request": "^2.48.12",

extensions/model-extension/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@janhq/model-extension",
-  "version": "1.0.13",
+  "version": "1.0.14",
-  "description": "This extension provides model downloads and controls the model lifecycle",
+  "description": "Model Management Extension provides model exploration and seamless downloads",
   "main": "dist/index.js",
   "module": "dist/module.js",
   "author": "Jan <service@jan.ai>",

extensions/model-extension/src/@types/global.d.ts

@@ -1,2 +1,3 @@
-declare const PLUGIN_NAME: string
+declare const EXTENSION_NAME: string
 declare const MODULE_PATH: string
+declare const VERSION: stringå

extensions/model-extension/src/index.ts

@@ -41,15 +41,14 @@ export default class JanModelExtension implements ModelExtension {
 
   private async copyModelsToHomeDir() {
     try {
-      // list all of the files under the home directory
+      if (localStorage.getItem(`${EXTENSION_NAME}-version`) === VERSION) {
-      const files = await fs.listFiles('')
+        console.debug('Model already migrated')
-
-      if (files.includes(JanModelExtension._homeDir)) {
-        // ignore if the model is already downloaded
-        console.debug('Model already downloaded')
         return
       }
 
+      // Get available models
+      const readyModels = (await this.getDownloadedModels()).map((e) => e.id)
+
       // copy models folder from resources to home directory
       const resourePath = await getResourcePath()
       const srcPath = join(resourePath, 'models')
@@ -57,7 +56,25 @@ export default class JanModelExtension implements ModelExtension {
       const userSpace = await getUserSpace()
       const destPath = join(userSpace, JanModelExtension._homeDir)
 
-      await fs.copyFile(srcPath, destPath)
+      await fs.syncFile(srcPath, destPath)
+
+      console.debug('Finished syncing models')
+
+      const reconfigureModels = (await this.getConfiguredModels()).filter((e) =>
+        readyModels.includes(e.id)
+      )
+      console.debug(
+        'Finished updating downloaded models'
+      )
+
+      // update back the status
+      await Promise.all(
+        reconfigureModels.map(async (model) => this.saveModel(model))
+      )
+
+      // Finished migration
+
+      localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION)
     } catch (err) {
       console.error(err)
     }
@@ -193,12 +210,18 @@ export default class JanModelExtension implements ModelExtension {
       const results = await Promise.allSettled(readJsonPromises)
       const modelData = results.map((result) => {
         if (result.status === 'fulfilled') {
-          return JSON.parse(result.value) as Model
+          try {
+            return JSON.parse(result.value) as Model
+          } catch {
+            console.debug(`Unable to parse model metadata: ${result.value}`)
+            return undefined
+          }
         } else {
           console.error(result.reason)
+          return undefined
         }
       })
-      return modelData
+      return modelData.filter((e) => !!e)
     } catch (err) {
       console.error(err)
       return []

extensions/model-extension/webpack.config.js

@@ -17,8 +17,9 @@ module.exports = {
   },
   plugins: [
     new webpack.DefinePlugin({
-      PLUGIN_NAME: JSON.stringify(packageJson.name),
+      EXTENSION_NAME: JSON.stringify(packageJson.name),
       MODULE_PATH: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
+      VERSION: JSON.stringify(packageJson.version),
     }),
   ],
   output: {