feat: add quick ask (#2197)

* feat: add quick ask

Signed-off-by: James <james@jan.ai>

---------

Signed-off-by: James <james@jan.ai>
Co-authored-by: James <james@jan.ai>
Co-authored-by: Louis <louis@jan.ai>

.gitignore

@@ -14,6 +14,7 @@ electron/renderer
 electron/models
 electron/docs
 electron/engines
+electron/playwright-report
 server/pre-install
 package-lock.json
 

core/package.json

@@ -45,19 +45,20 @@
     "start": "rollup -c rollup.config.ts -w"
   },
   "devDependencies": {
-    "jest": "^25.4.0",
+    "jest": "^29.7.0",
-    "@types/jest": "^29.5.11",
+    "@types/jest": "^29.5.12",
     "@types/node": "^12.0.2",
-    "eslint-plugin-jest": "^23.8.2",
+    "eslint-plugin-jest": "^27.9.0",
+    "eslint": "8.57.0",
     "rollup": "^2.38.5",
     "rollup-plugin-commonjs": "^9.1.8",
     "rollup-plugin-json": "^3.1.0",
     "rollup-plugin-node-resolve": "^5.2.0",
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rollup-plugin-typescript2": "^0.36.0",
-    "ts-jest": "^26.1.1",
+    "ts-jest": "^29.1.2",
     "tslib": "^2.6.2",
-    "typescript": "^5.2.2",
+    "typescript": "^5.3.3",
     "rimraf": "^3.0.2"
   }
 }

core/src/api/index.ts

@@ -9,6 +9,14 @@ export enum NativeRoute {
   selectDirectory = 'selectDirectory',
   selectModelFiles = 'selectModelFiles',
   relaunch = 'relaunch',
+
+  hideQuickAskWindow = 'hideQuickAskWindow',
+  sendQuickAskInput = 'sendQuickAskInput',
+
+  hideMainWindow = 'hideMainWindow',
+  showMainWindow = 'showMainWindow',
+
+  quickAskSizeUpdated = 'quickAskSizeUpdated',
 }
 
 /**
@@ -31,6 +39,9 @@ export enum AppEvent {
   onAppUpdateDownloadUpdate = 'onAppUpdateDownloadUpdate',
   onAppUpdateDownloadError = 'onAppUpdateDownloadError',
   onAppUpdateDownloadSuccess = 'onAppUpdateDownloadSuccess',
+
+  onUserSubmitQuickAsk = 'onUserSubmitQuickAsk',
+  onSelectedText = 'onSelectedText',
 }
 
 export enum DownloadRoute {

core/src/node/api/restful/helper/startStopModel.ts

@@ -41,7 +41,7 @@ const runModel = async (modelId: string, settingParams?: ModelSettingParams): Pr
   const modelFolderFullPath = join(janDataFolderPath, 'models', modelId)
 
   if (!fs.existsSync(modelFolderFullPath)) {
-    throw `Model not found: ${modelId}`
+    throw new Error(`Model not found: ${modelId}`)
   }
 
   const files: string[] = fs.readdirSync(modelFolderFullPath)
@@ -53,7 +53,7 @@ const runModel = async (modelId: string, settingParams?: ModelSettingParams): Pr
   const modelMetadata: Model = JSON.parse(fs.readFileSync(modelMetadataPath, 'utf-8'))
 
   if (!ggufBinFile) {
-    throw 'No GGUF model file found'
+    throw new Error('No GGUF model file found')
   }
   const modelBinaryPath = join(modelFolderFullPath, ggufBinFile)
 
@@ -76,7 +76,7 @@ const runModel = async (modelId: string, settingParams?: ModelSettingParams): Pr
     const promptTemplate = modelMetadata.settings.prompt_template
     const prompt = promptTemplateConverter(promptTemplate)
     if (prompt?.error) {
-      return Promise.reject(prompt.error)
+      throw new Error(prompt.error)
     }
     nitroModelSettings.system_prompt = prompt.system_prompt
     nitroModelSettings.user_prompt = prompt.user_prompt

core/src/types/miscellaneous/appUpdate.ts

@@ -0,0 +1,7 @@
+export type AppUpdateInfo = {
+  total: number
+  delta: number
+  transferred: number
+  percent: number
+  bytesPerSecond: number
+}

core/src/types/miscellaneous/index.ts

@@ -1,2 +1,3 @@
 export * from './systemResourceInfo'
 export * from './promptTemplate'
+export * from './appUpdate'

core/src/types/model/modelEntity.ts

@@ -138,3 +138,7 @@ export type ModelRuntimeParams = {
   presence_penalty?: number
   engine?: string
 }
+
+export type ModelInitFailed = Model & {
+  error: Error
+}

electron/handlers/common.ts

@@ -1,25 +1,20 @@
 import { Handler, RequestHandler } from '@janhq/core/node'
 import { ipcMain } from 'electron'
-import { WindowManager } from '../managers/window'
+import { windowManager } from '../managers/window'
 
 export function injectHandler() {
   const ipcWrapper: Handler = (
     route: string,
     listener: (...args: any[]) => any
-  ) => {
+  ) =>
-    return ipcMain.handle(route, async (event, ...args: any[]) => {
+    ipcMain.handle(route, async (_event, ...args: any[]) => {
       return listener(...args)
     })
-  }
 
   const handler = new RequestHandler(
     ipcWrapper,
-    (channel: string, args: any) => {
+    (channel: string, args: any) =>
-      return WindowManager.instance.currentWindow?.webContents.send(
+      windowManager.mainWindow?.webContents.send(channel, args)
-        channel,
-        args
-      )
-    }
   )
   handler.handle()
 }

electron/handlers/native.ts

@@ -1,13 +1,13 @@
 import { app, ipcMain, dialog, shell } from 'electron'
 import { join } from 'path'
-import { WindowManager } from '../managers/window'
+import { windowManager } from '../managers/window'
 import {
   ModuleManager,
   getJanDataFolderPath,
   getJanExtensionsPath,
   init,
 } from '@janhq/core/node'
-import { NativeRoute } from '@janhq/core'
+import { AppEvent, NativeRoute } from '@janhq/core'
 
 export function handleAppIPCs() {
   /**
@@ -62,12 +62,12 @@ export function handleAppIPCs() {
         // Path to install extension to
         extensionsPath: getJanExtensionsPath(),
       })
-      WindowManager.instance.currentWindow?.reload()
+      windowManager.mainWindow?.reload()
     }
   })
 
   ipcMain.handle(NativeRoute.selectDirectory, async () => {
-    const mainWindow = WindowManager.instance.currentWindow
+    const mainWindow = windowManager.mainWindow
     if (!mainWindow) {
       console.error('No main window found')
       return
@@ -85,7 +85,7 @@ export function handleAppIPCs() {
   })
 
   ipcMain.handle(NativeRoute.selectModelFiles, async () => {
-    const mainWindow = WindowManager.instance.currentWindow
+    const mainWindow = windowManager.mainWindow
     if (!mainWindow) {
       console.error('No main window found')
       return
@@ -101,4 +101,35 @@ export function handleAppIPCs() {
 
     return filePaths
   })
+
+  ipcMain.handle(
+    NativeRoute.hideQuickAskWindow,
+    async (): Promise<void> => windowManager.hideQuickAskWindow()
+  )
+
+  ipcMain.handle(
+    NativeRoute.sendQuickAskInput,
+    async (_event, input: string): Promise<void> => {
+      windowManager.mainWindow?.webContents.send(
+        AppEvent.onUserSubmitQuickAsk,
+        input
+      )
+    }
+  )
+
+  ipcMain.handle(
+    NativeRoute.hideMainWindow,
+    async (): Promise<void> => windowManager.hideMainWindow()
+  )
+
+  ipcMain.handle(
+    NativeRoute.showMainWindow,
+    async (): Promise<void> => windowManager.showMainWindow()
+  )
+
+  ipcMain.handle(
+    NativeRoute.quickAskSizeUpdated,
+    async (_event, heightOffset: number): Promise<void> =>
+      windowManager.expandQuickAskWindow(heightOffset)
+  )
 }

electron/handlers/update.ts

@@ -1,6 +1,11 @@
 import { app, dialog } from 'electron'
-import { WindowManager } from './../managers/window'
+import { windowManager } from './../managers/window'
-import { autoUpdater } from 'electron-updater'
+import {
+  ProgressInfo,
+  UpdateDownloadedEvent,
+  UpdateInfo,
+  autoUpdater,
+} from 'electron-updater'
 import { AppEvent } from '@janhq/core'
 
 export let waitingToInstallVersion: string | undefined = undefined
@@ -11,7 +16,7 @@ export function handleAppUpdates() {
     return
   }
   /* New Update Available */
-  autoUpdater.on('update-available', async (_info: any) => {
+  autoUpdater.on('update-available', async (_info: UpdateInfo) => {
     const action = await dialog.showMessageBox({
       title: 'Update Available',
       message: 'Would you like to download and install it now?',
@@ -21,8 +26,8 @@ export function handleAppUpdates() {
   })
 
   /* App Update Completion Message */
-  autoUpdater.on('update-downloaded', async (_info: any) => {
+  autoUpdater.on('update-downloaded', async (_info: UpdateDownloadedEvent) => {
-    WindowManager.instance.currentWindow?.webContents.send(
+    windowManager.mainWindow?.webContents.send(
       AppEvent.onAppUpdateDownloadSuccess,
       {}
     )
@@ -37,23 +42,24 @@ export function handleAppUpdates() {
   })
 
   /* App Update Error */
-  autoUpdater.on('error', (info: any) => {
+  autoUpdater.on('error', (info: Error) => {
-    WindowManager.instance.currentWindow?.webContents.send(
+    windowManager.mainWindow?.webContents.send(
       AppEvent.onAppUpdateDownloadError,
       { failedToInstallVersion: waitingToInstallVersion, info }
     )
   })
 
   /* App Update Progress */
-  autoUpdater.on('download-progress', (progress: any) => {
+  autoUpdater.on('download-progress', (progress: ProgressInfo) => {
     console.debug('app update progress: ', progress.percent)
-    WindowManager.instance.currentWindow?.webContents.send(
+    windowManager.mainWindow?.webContents.send(
       AppEvent.onAppUpdateDownloadUpdate,
       {
-        percent: progress.percent,
+        ...progress,
       }
     )
   })
+
   autoUpdater.autoDownload = false
   autoUpdater.autoInstallOnAppQuit = true
   if (process.env.CI !== 'e2e') {

electron/main.ts

@@ -1,9 +1,10 @@
-import { app, BrowserWindow, shell } from 'electron'
+import { app, BrowserWindow, globalShortcut, Menu, Tray } from 'electron'
+
 import { join } from 'path'
 /**
  * Managers
  **/
-import { WindowManager } from './managers/window'
+import { windowManager } from './managers/window'
 import { log } from '@janhq/core/node'
 
 /**
@@ -25,6 +26,18 @@ import { setupCore } from './utils/setup'
 import { setupReactDevTool } from './utils/dev'
 import { cleanLogs } from './utils/log'
 
+import { registerShortcut } from './utils/selectedText'
+
+const preloadPath = join(__dirname, 'preload.js')
+const rendererPath = join(__dirname, '..', 'renderer')
+const quickAskPath = join(rendererPath, 'search.html')
+const mainPath = join(rendererPath, 'index.html')
+
+const mainUrl = 'http://localhost:3000'
+const quickAskUrl = `${mainUrl}/search`
+
+const quickAskHotKey = 'CommandOrControl+J'
+
 app
   .whenReady()
   .then(setupReactDevTool)
@@ -35,7 +48,36 @@ app
   .then(setupMenu)
   .then(handleIPCs)
   .then(handleAppUpdates)
+  .then(createQuickAskWindow)
   .then(createMainWindow)
+  .then(() => {
+    if (!app.isPackaged) {
+      windowManager.mainWindow?.webContents.openDevTools()
+    }
+  })
+  .then(() => {
+    const iconPath = join(app.getAppPath(), 'icons', 'icon-tray.png')
+    const tray = new Tray(iconPath)
+    tray.setToolTip(app.getName())
+
+    const contextMenu = Menu.buildFromTemplate([
+      {
+        label: 'Open Jan',
+        type: 'normal',
+        click: () => windowManager.showMainWindow(),
+      },
+      {
+        label: 'Open Quick Ask',
+        type: 'normal',
+        click: () => windowManager.showQuickAskWindow(),
+      },
+      { label: 'Quit', type: 'normal', click: () => app.quit() },
+    ])
+    tray.setContextMenu(contextMenu)
+  })
+  .then(() => {
+    log(`Version: ${app.getVersion()}`)
+  })
   .then(() => {
     app.on('activate', () => {
       if (!BrowserWindow.getAllWindows().length) {
@@ -45,45 +87,42 @@ app
   })
   .then(() => cleanLogs())
 
-app.once('window-all-closed', () => {
+app.on('ready', () => {
-  cleanUpAndQuit()
+  registerGlobalShortcuts()
 })
 
 app.once('quit', () => {
   cleanUpAndQuit()
 })
 
+function createQuickAskWindow() {
+  const startUrl = app.isPackaged ? `file://${quickAskPath}` : quickAskUrl
+  windowManager.createQuickAskWindow(preloadPath, startUrl)
+}
+
 function createMainWindow() {
-  /* Create main window */
+  const startUrl = app.isPackaged ? `file://${mainPath}` : mainUrl
-  const mainWindow = WindowManager.instance.createWindow({
+  windowManager.createMainWindow(preloadPath, startUrl)
-    webPreferences: {
+}
-      nodeIntegration: true,
+
-      preload: join(__dirname, 'preload.js'),
+function registerGlobalShortcuts() {
-      webSecurity: false,
+  // TODO: Toggle below line when build production
-    },
+  // const ret = globalShortcut.register(quickAskHotKey, () => {
+  // const selectedText = ''
+  const ret = registerShortcut(quickAskHotKey, (selectedText: string) => {
+    if (!windowManager.isQuickAskWindowVisible()) {
+      windowManager.showQuickAskWindow()
+      windowManager.sendQuickAskSelectedText(selectedText)
+    } else {
+      windowManager.hideQuickAskWindow()
+    }
   })
 
-  const startURL = app.isPackaged
+  if (!ret) {
-    ? `file://${join(__dirname, '..', 'renderer', 'index.html')}`
+    console.error('Global shortcut registration failed')
-    : 'http://localhost:3000'
+  } else {
-
+    console.log('Global shortcut registered successfully')
-  /* Load frontend app to the window */
+  }
-  mainWindow.loadURL(startURL)
-
-  mainWindow.once('ready-to-show', () => mainWindow?.show())
-  mainWindow.on('closed', () => {
-    if (process.platform !== 'darwin') app.quit()
-  })
-
-  /* Open external links in the default browser */
-  mainWindow.webContents.setWindowOpenHandler(({ url }) => {
-    shell.openExternal(url)
-    return { action: 'deny' }
-  })
-
-  /* Enable dev tools for development */
-  if (!app.isPackaged) mainWindow.webContents.openDevTools()
-  log(`Version: ${app.getVersion()}`)
 }
 
 /**

electron/managers/mainWindowConfig.ts

@@ -0,0 +1,16 @@
+const DEFAULT_WIDTH = 1200
+const DEFAULT_HEIGHT = 800
+
+export const mainWindowConfig: Electron.BrowserWindowConstructorOptions = {
+  width: DEFAULT_WIDTH,
+  minWidth: DEFAULT_WIDTH,
+  height: DEFAULT_HEIGHT,
+  skipTaskbar: true,
+  show: true,
+  trafficLightPosition: {
+    x: 10,
+    y: 15,
+  },
+  titleBarStyle: 'hiddenInset',
+  vibrancy: 'sidebar',
+}

electron/managers/quickAskWindowConfig.ts

@@ -0,0 +1,13 @@
+const DEFAULT_WIDTH = 556
+
+const DEFAULT_HEIGHT = 60
+
+export const quickAskWindowConfig: Electron.BrowserWindowConstructorOptions = {
+  width: DEFAULT_WIDTH,
+  height: DEFAULT_HEIGHT,
+  skipTaskbar: true,
+  resizable: false,
+  transparent: true,
+  frame: false,
+  type: 'panel',
+}

electron/managers/window.ts

@@ -1,37 +1,123 @@
-import { BrowserWindow } from 'electron'
+import { BrowserWindow, app, shell } from 'electron'
+import { quickAskWindowConfig } from './quickAskWindowConfig'
+import { AppEvent } from '@janhq/core'
+import { mainWindowConfig } from './mainWindowConfig'
 
 /**
  * Manages the current window instance.
  */
-export class WindowManager {
+// TODO: refactor this
-  public static instance: WindowManager = new WindowManager()
+let isAppQuitting = false
-  public currentWindow?: BrowserWindow
+class WindowManager {
-
+  public mainWindow?: BrowserWindow
-  constructor() {
+  private _quickAskWindow: BrowserWindow | undefined = undefined
-    if (WindowManager.instance) {
+  private _quickAskWindowVisible = false
-      return WindowManager.instance
+  private _mainWindowVisible = false
-    }
-  }
 
   /**
    * Creates a new window instance.
    * @param {Electron.BrowserWindowConstructorOptions} options - The options to create the window with.
    * @returns The created window instance.
    */
-  createWindow(options?: Electron.BrowserWindowConstructorOptions | undefined) {
+  createMainWindow(preloadPath: string, startUrl: string) {
-    this.currentWindow = new BrowserWindow({
+    this.mainWindow = new BrowserWindow({
-      width: 1200,
+      ...mainWindowConfig,
-      minWidth: 1200,
+      webPreferences: {
-      height: 800,
+        nodeIntegration: true,
-      show: true,
+        preload: preloadPath,
-      trafficLightPosition: {
+        webSecurity: false,
-        x: 10,
-        y: 15,
       },
-      titleBarStyle: 'hiddenInset',
-      vibrancy: 'sidebar',
-      ...options,
     })
-    return this.currentWindow
+
+    /* Load frontend app to the window */
+    this.mainWindow.loadURL(startUrl)
+
+    /* Open external links in the default browser */
+    this.mainWindow.webContents.setWindowOpenHandler(({ url }) => {
+      shell.openExternal(url)
+      return { action: 'deny' }
+    })
+
+    app.on('before-quit', function () {
+      isAppQuitting = true
+    })
+
+    windowManager.mainWindow?.on('close', function (evt) {
+      if (!isAppQuitting) {
+        evt.preventDefault()
+        windowManager.hideMainWindow()
+      }
+    })
+  }
+
+  createQuickAskWindow(preloadPath: string, startUrl: string): void {
+    this._quickAskWindow = new BrowserWindow({
+      ...quickAskWindowConfig,
+      webPreferences: {
+        nodeIntegration: true,
+        preload: preloadPath,
+        webSecurity: false,
+      },
+    })
+
+    this._quickAskWindow.loadURL(startUrl)
+    this._quickAskWindow.on('blur', () => {
+      this.hideQuickAskWindow()
+    })
+    this.hideQuickAskWindow()
+  }
+
+  isMainWindowVisible(): boolean {
+    return this._mainWindowVisible
+  }
+
+  hideMainWindow(): void {
+    this.mainWindow?.hide()
+    this._mainWindowVisible = false
+    // Only macos
+    if (process.platform === 'darwin') app.dock.hide()
+  }
+
+  showMainWindow(): void {
+    this.mainWindow?.show()
+    this._mainWindowVisible = true
+    // Only macos
+    if (process.platform === 'darwin') app.dock.show()
+  }
+
+  hideQuickAskWindow(): void {
+    this._quickAskWindow?.hide()
+    this._quickAskWindowVisible = false
+  }
+
+  showQuickAskWindow(): void {
+    this._quickAskWindow?.show()
+    this._quickAskWindowVisible = true
+  }
+
+  isQuickAskWindowVisible(): boolean {
+    return this._quickAskWindowVisible
+  }
+
+  expandQuickAskWindow(heightOffset: number): void {
+    const width = quickAskWindowConfig.width!
+    const height = quickAskWindowConfig.height! + heightOffset
+    this._quickAskWindow?.setSize(width, height, true)
+  }
+
+  sendQuickAskSelectedText(selectedText: string): void {
+    this._quickAskWindow?.webContents.send(
+      AppEvent.onSelectedText,
+      selectedText
+    )
+  }
+
+  cleanUp(): void {
+    this.mainWindow?.destroy()
+    this._quickAskWindow?.destroy()
+    this._quickAskWindowVisible = false
+    this._mainWindowVisible = false
   }
 }
+
+export const windowManager = new WindowManager()

electron/package.json

@@ -16,13 +16,15 @@
       "pre-install",
       "models/**/*",
       "docs/**/*",
-      "scripts/**/*"
+      "scripts/**/*",
+      "icons/**/*"
     ],
     "asarUnpack": [
       "pre-install",
       "models",
       "docs",
-      "scripts"
+      "scripts",
+      "icons"
     ],
     "publish": [
       {
@@ -39,6 +41,7 @@
       "notarize": {
         "teamId": "F8AH6NHVY5"
       },
+     
       "icon": "icons/icon.png"
     },
     "linux": {
@@ -81,7 +84,6 @@
     "@janhq/core": "link:./core",
     "@janhq/server": "link:./server",
     "@npmcli/arborist": "^7.1.0",
-    "@uiball/loaders": "^1.3.0",
     "electron-store": "^8.1.0",
     "electron-updater": "^6.1.7",
     "fs-extra": "^11.2.0",
@@ -90,7 +92,7 @@
     "request": "^2.88.2",
     "request-progress": "^3.0.0",
     "ulid": "^2.3.0",
-    "use-debounce": "^9.0.4"
+    "@hurdlegroup/robotjs": "^0.11.4"
   },
   "devDependencies": {
     "@electron/notarize": "^2.1.0",
@@ -101,13 +103,15 @@
     "@typescript-eslint/eslint-plugin": "^6.7.3",
     "@typescript-eslint/parser": "^6.7.3",
     "electron": "28.0.0",
-    "electron-builder": "^24.9.1",
+    "electron-builder": "^24.13.3",
+    "electron-builder-squirrel-windows": "^24.13.3",
     "electron-devtools-installer": "^3.2.0",
     "electron-playwright-helpers": "^1.6.0",
-    "eslint-plugin-react": "^7.33.2",
+    "eslint": "8.57.0",
+    "eslint-plugin-react": "^7.34.0",
     "rimraf": "^5.0.5",
     "run-script-os": "^1.1.6",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.3"
   },
   "installConfig": {
     "hoistingLimits": "workspaces"

electron/utils/clean.ts

@@ -1,12 +1,12 @@
 import { ModuleManager } from '@janhq/core/node'
-import { WindowManager } from './../managers/window'
+import { windowManager } from './../managers/window'
 import { dispose } from './disposable'
 import { app } from 'electron'
 
 export function cleanUpAndQuit() {
   if (!ModuleManager.instance.cleaningResource) {
     ModuleManager.instance.cleaningResource = true
-    WindowManager.instance.currentWindow?.destroy()
+    windowManager.cleanUp()
     dispose(ModuleManager.instance.requiredModules)
     ModuleManager.instance.clearImportedModules()
     app.quit()

electron/utils/selectedText.ts

@@ -0,0 +1,39 @@
+import { clipboard, globalShortcut } from "electron";
+import { keyTap, keys } from "@hurdlegroup/robotjs";
+
+/**
+ * Gets selected text by synthesizing the keyboard shortcut
+ * "CommandOrControl+c" then reading text from the clipboard
+ */
+export const getSelectedText = async () => {
+  const currentClipboardContent = clipboard.readText(); // preserve clipboard content
+  clipboard.clear();
+  keyTap("c" as keys, process.platform === "darwin" ? "command" : "control");
+  await new Promise((resolve) => setTimeout(resolve, 200)); // add a delay before checking clipboard
+  const selectedText = clipboard.readText();
+  clipboard.writeText(currentClipboardContent);
+  return selectedText;
+};
+
+/**
+ * Registers a global shortcut of `accelerator`. The `callback` is called
+ * with the selected text when the registered shorcut is pressed by the user
+ *
+ * Returns `true` if the shortcut was registered successfully
+ */
+export const registerShortcut = (
+  accelerator: Electron.Accelerator,
+  callback: (selectedText: string) => void
+) => {
+  return globalShortcut.register(accelerator, async () => {
+    callback(await getSelectedText());
+  });
+};
+
+/**
+ * Unregisters a global shortcut of `accelerator` and
+ * is equivalent to electron.globalShortcut.unregister
+ */
+export const unregisterShortcut = (accelerator: Electron.Accelerator) => {
+  globalShortcut.unregister(accelerator);
+};

extensions/assistant-extension/package.json

@@ -26,7 +26,7 @@
     "rollup-plugin-define": "^1.0.1",
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rollup-plugin-typescript2": "^0.36.0",
-    "typescript": "^5.2.2",
+    "typescript": "^5.3.3",
     "run-script-os": "^1.1.6"
   },
   "dependencies": {

extensions/inference-nitro-extension/package.json

@@ -35,7 +35,7 @@
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rollup-plugin-typescript2": "^0.36.0",
     "run-script-os": "^1.1.6",
-    "typescript": "^5.2.2",
+    "typescript": "^5.3.3",
     "@types/os-utils": "^0.0.4",
     "@rollup/plugin-replace": "^5.0.5"
   },

server/package.json

@@ -39,8 +39,8 @@
     "@types/tcp-port-used": "^1.0.4",
     "@typescript-eslint/eslint-plugin": "^6.7.3",
     "@typescript-eslint/parser": "^6.7.3",
-    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react": "^7.34.0",
     "run-script-os": "^1.1.6",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.3"
   }
 }

uikit/package.json

@@ -52,6 +52,6 @@
     "tailwind-merge": "^2.0.0",
     "terser": "^5.24.0",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.3"
   }
 }

web/app/search/SelectedText.tsx

@@ -0,0 +1,47 @@
+import React, { useCallback, useEffect, useRef } from 'react'
+
+import { useAtom } from 'jotai'
+import { X } from 'lucide-react'
+
+import { selectedTextAtom } from '@/containers/Providers/Jotai'
+
+const SelectedText = ({ onCleared }: { onCleared?: () => void }) => {
+  const [text, setText] = useAtom(selectedTextAtom)
+  const containerRef = useRef<HTMLDivElement>(null)
+
+  useEffect(() => {
+    if (text.trim().length === 0) {
+      window.core?.api?.quickAskSizeUpdated(0)
+    } else {
+      window.core?.api?.quickAskSizeUpdated(
+        (containerRef.current?.offsetHeight ?? 0) + 14
+      )
+    }
+  })
+
+  const onClearClicked = useCallback(() => {
+    setText('')
+    onCleared?.()
+  }, [setText, onCleared])
+
+  const shouldShowSelectedText = text.trim().length > 0
+
+  return shouldShowSelectedText ? (
+    <div
+      ref={containerRef}
+      className="relative rounded-lg border-[1px] border-[#0000000F] bg-[#0000000A] p-[10px]"
+    >
+      <div
+        className="absolute right-1 top-1 flex h-6 w-6 items-center justify-center rounded-full border-[1px] border-[#0000000F] bg-white drop-shadow"
+        onClick={onClearClicked}
+      >
+        <X size={16} />
+      </div>
+      <p className="font-semibold text-[#00000099]">{text}</p>
+    </div>
+  ) : (
+    <div />
+  )
+}
+
+export default SelectedText

web/app/search/UserInput.tsx

@@ -0,0 +1,86 @@
+import React, { useState, useRef, useEffect } from 'react'
+
+import { Button } from '@janhq/uikit'
+import { useAtomValue } from 'jotai'
+
+import { Send } from 'lucide-react'
+
+import LogoMark from '@/containers/Brand/Logo/Mark'
+
+import { selectedTextAtom } from '@/containers/Providers/Jotai'
+
+import SelectedText from './SelectedText'
+
+const UserInput: React.FC = () => {
+  const [inputValue, setInputValue] = useState('')
+  const inputRef = useRef<HTMLInputElement>(null)
+  const formRef = useRef<HTMLFormElement>(null)
+  const selectedText = useAtomValue(selectedTextAtom)
+
+  useEffect(() => {
+    if (inputRef.current) {
+      inputRef.current.focus()
+    }
+
+    const onKeyDown = (e: KeyboardEvent) => {
+      if (e.key === 'Escape') {
+        window.core?.api?.hideQuickAskWindow()
+      }
+    }
+
+    document.addEventListener('keydown', onKeyDown)
+
+    return () => {
+      document.removeEventListener('keydown', onKeyDown)
+    }
+  }, [])
+
+  const handleChange = (
+    event:
+      | React.ChangeEvent<HTMLInputElement>
+      | React.ChangeEvent<HTMLTextAreaElement>
+  ) => {
+    const { value } = event.target
+    setInputValue(value)
+  }
+
+  const onSubmit = (e: React.FormEvent) => {
+    e.preventDefault()
+    if (inputValue.trim() !== '') {
+      const fullText = `${inputValue} ${selectedText}`.trim()
+      window.core?.api?.sendQuickAskInput(fullText)
+      setInputValue('')
+      window.core?.api?.hideQuickAskWindow()
+      window.core?.api?.showMainWindow()
+    }
+  }
+
+  return (
+    <div className="flex flex-col space-y-3 p-3">
+      <form
+        ref={formRef}
+        className="flex h-full w-full items-center justify-center"
+        onSubmit={onSubmit}
+      >
+        <div className="flex h-full w-full items-center gap-4">
+          <LogoMark width={28} height={28} className="mx-auto" />
+          <input
+            ref={inputRef}
+            className="flex-1 bg-transparent font-bold text-black focus:outline-none"
+            type="text"
+            value={inputValue}
+            onChange={handleChange}
+            placeholder="Ask me anything"
+          />
+          <Button onClick={onSubmit}>
+            <Send size={16} />
+          </Button>
+        </div>
+      </form>
+
+      <SelectedText onCleared={() => inputRef?.current?.focus()} />
+    </div>
+  )
+}
+
+export default UserInput

web/app/search/page.tsx

@@ -0,0 +1,13 @@
+'use client'
+
+import UserInput from './UserInput'
+
+const Search: React.FC = () => {
+  return (
+    <div className="h-screen w-screen overflow-hidden bg-white">
+      <UserInput />
+    </div>
+  )
+}
+
+export default Search

web/containers/Providers/AppUpdateListener.tsx

@@ -1,6 +1,6 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
 import { Fragment, PropsWithChildren, useEffect } from 'react'
 
+import { AppUpdateInfo } from '@janhq/core'
 import { useSetAtom } from 'jotai'
 
 import { appDownloadProgress, updateVersionError } from './Jotai'
@@ -12,13 +12,14 @@ const AppUpdateListener = ({ children }: PropsWithChildren) => {
   useEffect(() => {
     if (window && window.electronAPI) {
       window.electronAPI.onAppUpdateDownloadUpdate(
-        (_event: string, progress: any) => {
+        (_event: string, appUpdateInfo: AppUpdateInfo) => {
-          setProgress(progress.percent)
+          setProgress(appUpdateInfo.percent)
-          console.debug('app update progress:', progress.percent)
+          console.debug('app update progress:', appUpdateInfo.percent)
         }
       )
 
       window.electronAPI.onAppUpdateDownloadError(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         (_event: string, error: any) => {
           console.error('Download error: ', error)
           setProgress(-1)
@@ -33,8 +34,7 @@ const AppUpdateListener = ({ children }: PropsWithChildren) => {
         setProgress(-1)
       })
     }
-    return () => {}
+  }, [setProgress, setUpdateVersionError])
-  }, [setProgress])
 
   return <Fragment>{children}</Fragment>
 }

web/containers/Providers/ClipboardListener.tsx

@@ -0,0 +1,17 @@
+import { Fragment, PropsWithChildren } from 'react'
+
+import { useSetAtom } from 'jotai'
+
+import { selectedTextAtom } from './Jotai'
+
+const ClipboardListener = ({ children }: PropsWithChildren) => {
+  const setSelectedText = useSetAtom(selectedTextAtom)
+
+  window?.electronAPI?.onSelectedText((_event: string, text: string) => {
+    setSelectedText(text)
+  })
+
+  return <Fragment>{children}</Fragment>
+}
+
+export default ClipboardListener

web/containers/Providers/DataLoader.tsx

@@ -2,7 +2,7 @@
 
 import { Fragment, ReactNode, useEffect } from 'react'
 
-import { AppConfiguration } from '@janhq/core/.'
+import { AppConfiguration } from '@janhq/core'
 import { useSetAtom } from 'jotai'
 
 import useAssistants from '@/hooks/useAssistants'

web/containers/Providers/EventHandler.tsx

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
 import { Fragment, ReactNode, useCallback, useEffect, useRef } from 'react'
 
 import {
@@ -15,6 +14,7 @@ import {
   MessageRequestType,
   ModelEvent,
   Thread,
+  ModelInitFailed,
 } from '@janhq/core'
 import { useAtomValue, useSetAtom } from 'jotai'
 import { ulid } from 'ulid'
@@ -113,15 +113,14 @@ export default function EventHandler({ children }: { children: ReactNode }) {
   }, [setActiveModel, setStateModel])
 
   const onModelInitFailed = useCallback(
-    (res: any) => {
+    (res: ModelInitFailed) => {
-      const errorMessage = res?.error ?? res
+      console.error('Failed to load model: ', res.error.message)
-      console.error('Failed to load model: ', errorMessage)
       setStateModel(() => ({
         state: 'start',
         loading: false,
-        model: res.modelId,
+        model: res.id,
       }))
-      setLoadModelError(errorMessage)
+      setLoadModelError(res.error.message)
       setQueuedMessage(false)
     },
     [setStateModel, setQueuedMessage, setLoadModelError]
@@ -245,7 +244,7 @@ export default function EventHandler({ children }: { children: ReactNode }) {
 
       if (!threadMessages || threadMessages.length === 0) return
 
-      const summarizeFirstPrompt = `Summarize this text "${threadMessages[0].content[0].text.value}" for a conversation title in less than 10 words`
+      const summarizeFirstPrompt = `Summarize in a 5-word Title. Give the title only. "${threadMessages[0].content[0].text.value}"`
       // Prompt: Given this query from user {query}, return to me the summary in 5 words as the title
       const msgId = ulid()
       const messages: ChatCompletionMessage[] = [

web/containers/Providers/EventListener.tsx

@@ -8,9 +8,11 @@ import { useSetAtom } from 'jotai'
 import { setDownloadStateAtom } from '@/hooks/useDownloadState'
 
 import AppUpdateListener from './AppUpdateListener'
+import ClipboardListener from './ClipboardListener'
 import EventHandler from './EventHandler'
 
 import ModelImportListener from './ModelImportListener'
+import QuickAskListener from './QuickAskListener'
 
 const EventListenerWrapper = ({ children }: PropsWithChildren) => {
   const setDownloadState = useSetAtom(setDownloadStateAtom)
@@ -55,9 +57,13 @@ const EventListenerWrapper = ({ children }: PropsWithChildren) => {
 
   return (
     <AppUpdateListener>
-      <ModelImportListener>
+      <ClipboardListener>
-        <EventHandler>{children}</EventHandler>
+        <ModelImportListener>
-      </ModelImportListener>
+          <QuickAskListener>
+            <EventHandler>{children}</EventHandler>
+          </QuickAskListener>
+        </ModelImportListener>
+      </ClipboardListener>
     </AppUpdateListener>
   )
 }

web/containers/Providers/Jotai.tsx

@@ -15,6 +15,8 @@ export const appDownloadProgress = atom<number>(-1)
 export const updateVersionError = atom<string | undefined>(undefined)
 export const searchAtom = atom<string>('')
 
+export const selectedTextAtom = atom('')
+
 export default function JotaiWrapper({ children }: Props) {
   return <Provider>{children}</Provider>
 }

web/containers/Providers/KeyListener.tsx

@@ -24,6 +24,10 @@ export default function KeyListener({ children }: Props) {
 
   useEffect(() => {
     const onKeyDown = (e: KeyboardEvent) => {
+      if (e.key === 'Escape') {
+        window.core?.api?.hideMainWindow()
+      }
+
       const prefixKey = isMac ? e.metaKey : e.ctrlKey
 
       if (e.key === 'b' && prefixKey) {

web/containers/Providers/QuickAskListener.tsx

@@ -0,0 +1,39 @@
+import { Fragment, ReactNode, useRef } from 'react'
+
+import { useSetAtom } from 'jotai'
+
+import { MainViewState } from '@/constants/screens'
+
+import useSendChatMessage from '@/hooks/useSendChatMessage'
+
+import { showRightSideBarAtom } from '@/screens/Chat/Sidebar'
+
+import { showLeftSideBarAtom } from './KeyListener'
+
+import { mainViewStateAtom } from '@/helpers/atoms/App.atom'
+
+type Props = {
+  children: ReactNode
+}
+
+const QuickAskListener: React.FC<Props> = ({ children }) => {
+  const { sendChatMessage } = useSendChatMessage()
+  const setShowRightSideBar = useSetAtom(showRightSideBarAtom)
+  const setShowLeftSideBar = useSetAtom(showLeftSideBarAtom)
+  const setMainState = useSetAtom(mainViewStateAtom)
+
+  const previousMessage = useRef('')
+
+  window.electronAPI.onUserSubmitQuickAsk((_event: string, input: string) => {
+    if (previousMessage.current === input) return
+    setMainState(MainViewState.Thread)
+    setShowRightSideBar(false)
+    setShowLeftSideBar(false)
+    sendChatMessage(input)
+    previousMessage.current = input
+  })
+
+  return <Fragment>{children}</Fragment>
+}
+
+export default QuickAskListener

web/helpers/atoms/Assistant.atom.ts

@@ -1,4 +1,4 @@
-import { Assistant } from '@janhq/core/.'
+import { Assistant } from '@janhq/core'
 import { atom } from 'jotai'
 
 export const assistantsAtom = atom<Assistant[]>([])

web/hooks/useActiveModel.ts

@@ -1,3 +1,5 @@
+import { useEffect, useRef } from 'react'
+
 import { events, Model, ModelEvent } from '@janhq/core'
 import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai'
 
@@ -24,6 +26,12 @@ export function useActiveModel() {
   const downloadedModels = useAtomValue(downloadedModelsAtom)
   const setLoadModelError = useSetAtom(loadModelErrorAtom)
 
+  const downloadedModelsRef = useRef<Model[]>([])
+
+  useEffect(() => {
+    downloadedModelsRef.current = downloadedModels
+  }, [downloadedModels])
+
   const startModel = async (modelId: string) => {
     if (
       (activeModel && activeModel.id === modelId) ||
@@ -39,7 +47,7 @@ export function useActiveModel() {
 
     setStateModel({ state: 'start', loading: true, model: modelId })
 
-    let model = downloadedModels.find((e) => e.id === modelId)
+    let model = downloadedModelsRef?.current.find((e) => e.id === modelId)
 
     if (!model) {
       toaster({

web/hooks/useSendChatMessage.ts

@@ -63,9 +63,8 @@ export default function useSendChatMessage() {
   const setEditPrompt = useSetAtom(editPromptAtom)
 
   const currentMessages = useAtomValue(getCurrentChatMessagesAtom)
-  const { activeModel } = useActiveModel()
   const selectedModel = useAtomValue(selectedModelAtom)
-  const { startModel } = useActiveModel()
+  const { activeModel, startModel } = useActiveModel()
   const setQueuedMessage = useSetAtom(queuedMessageAtom)
   const loadModelFailed = useAtomValue(loadModelErrorAtom)
 
@@ -78,6 +77,7 @@ export default function useSendChatMessage() {
   const setReloadModel = useSetAtom(reloadModelAtom)
   const [fileUpload, setFileUpload] = useAtom(fileUploadAtom)
   const setIsGeneratingResponse = useSetAtom(isGeneratingResponseAtom)
+  const activeThreadRef = useRef<Thread | undefined>()
 
   useEffect(() => {
     modelRef.current = activeModel
@@ -87,15 +87,19 @@ export default function useSendChatMessage() {
     loadModelFailedRef.current = loadModelFailed
   }, [loadModelFailed])
 
+  useEffect(() => {
+    activeThreadRef.current = activeThread
+  }, [activeThread])
+
   const resendChatMessage = async (currentMessage: ThreadMessage) => {
-    if (!activeThread) {
+    if (!activeThreadRef.current) {
       console.error('No active thread')
       return
     }
     setIsGeneratingResponse(true)
-    updateThreadWaiting(activeThread.id, true)
+    updateThreadWaiting(activeThreadRef.current.id, true)
     const messages: ChatCompletionMessage[] = [
-      activeThread.assistants[0]?.instructions,
+      activeThreadRef.current.assistants[0]?.instructions,
     ]
       .filter((e) => e && e.trim() !== '')
       .map<ChatCompletionMessage>((instructions) => {
@@ -123,13 +127,14 @@ export default function useSendChatMessage() {
       id: ulid(),
       type: MessageRequestType.Thread,
       messages: messages,
-      threadId: activeThread.id,
+      threadId: activeThreadRef.current.id,
-      model: activeThread.assistants[0].model ?? selectedModel,
+      model: activeThreadRef.current.assistants[0].model ?? selectedModel,
     }
 
-    const modelId = selectedModel?.id ?? activeThread.assistants[0].model.id
+    const modelId =
+      selectedModel?.id ?? activeThreadRef.current.assistants[0].model.id
 
-    if (activeModel?.id !== modelId) {
+    if (modelRef.current?.id !== modelId) {
       setQueuedMessage(true)
       startModel(modelId)
       await waitForModelStarting(modelId)
@@ -139,11 +144,11 @@ export default function useSendChatMessage() {
     if (currentMessage.role !== ChatCompletionRole.User) {
       // Delete last response before regenerating
       deleteMessage(currentMessage.id ?? '')
-      if (activeThread) {
+      if (activeThreadRef.current) {
         await extensionManager
           .get<ConversationalExtension>(ExtensionTypeEnum.Conversational)
           ?.writeMessages(
-            activeThread.id,
+            activeThreadRef.current.id,
             currentMessages.filter((msg) => msg.id !== currentMessage.id)
           )
       }
@@ -154,7 +159,7 @@ export default function useSendChatMessage() {
   const sendChatMessage = async (message: string) => {
     if (!message || message.trim().length === 0) return
 
-    if (!activeThread) {
+    if (!activeThreadRef.current) {
       console.error('No active thread')
       return
     }
@@ -165,7 +170,7 @@ export default function useSendChatMessage() {
     const runtimeParams = toRuntimeParams(activeModelParams)
     const settingParams = toSettingParams(activeModelParams)
 
-    updateThreadWaiting(activeThread.id, true)
+    updateThreadWaiting(activeThreadRef.current.id, true)
     const prompt = message.trim()
     setCurrentPrompt('')
     setEditPrompt('')
@@ -187,7 +192,7 @@ export default function useSendChatMessage() {
     }
 
     const messages: ChatCompletionMessage[] = [
-      activeThread.assistants[0]?.instructions,
+      activeThreadRef.current.assistants[0]?.instructions,
     ]
       .filter((e) => e && e.trim() !== '')
       .map<ChatCompletionMessage>((instructions) => {
@@ -218,7 +223,7 @@ export default function useSendChatMessage() {
                         ? {
                             type: ChatCompletionMessageContentType.Doc,
                             doc_url: {
-                              url: `threads/${activeThread.id}/files/${msgId}.pdf`,
+                              url: `threads/${activeThreadRef.current.id}/files/${msgId}.pdf`,
                             },
                           }
                         : null,
@@ -236,13 +241,14 @@ export default function useSendChatMessage() {
           ])
       )
 
-    let modelRequest = selectedModel ?? activeThread.assistants[0].model
+    let modelRequest =
+      selectedModel ?? activeThreadRef.current.assistants[0].model
     if (runtimeParams.stream == null) {
       runtimeParams.stream = true
     }
     // Add middleware to the model request with tool retrieval enabled
     if (
-      activeThread.assistants[0].tools?.some(
+      activeThreadRef.current.assistants[0].tools?.some(
         (tool: AssistantTool) => tool.type === 'retrieval' && tool.enabled
       )
     ) {
@@ -260,14 +266,14 @@ export default function useSendChatMessage() {
     const messageRequest: MessageRequest = {
       id: msgId,
       type: MessageRequestType.Thread,
-      threadId: activeThread.id,
+      threadId: activeThreadRef.current.id,
       messages,
       model: {
         ...modelRequest,
         settings: settingParams,
         parameters: runtimeParams,
       },
-      thread: activeThread,
+      thread: activeThreadRef.current,
     }
 
     const timestamp = Date.now()
@@ -307,7 +313,7 @@ export default function useSendChatMessage() {
 
     const threadMessage: ThreadMessage = {
       id: msgId,
-      thread_id: activeThread.id,
+      thread_id: activeThreadRef.current.id,
       role: ChatCompletionRole.User,
       status: MessageStatus.Ready,
       created: timestamp,
@@ -322,10 +328,10 @@ export default function useSendChatMessage() {
     }
 
     const updatedThread: Thread = {
-      ...activeThread,
+      ...activeThreadRef.current,
       updated: timestamp,
       metadata: {
-        ...(activeThread.metadata ?? {}),
+        ...(activeThreadRef.current.metadata ?? {}),
         lastMessage: prompt,
       },
     }
@@ -337,9 +343,10 @@ export default function useSendChatMessage() {
       .get<ConversationalExtension>(ExtensionTypeEnum.Conversational)
       ?.addNewMessage(threadMessage)
 
-    const modelId = selectedModel?.id ?? activeThread.assistants[0].model.id
+    const modelId =
+      selectedModel?.id ?? activeThreadRef.current.assistants[0].model.id
 
-    if (activeModel?.id !== modelId) {
+    if (modelRef.current?.id !== modelId) {
       setQueuedMessage(true)
       startModel(modelId)
       await waitForModelStarting(modelId)

web/package.json

@@ -37,6 +37,7 @@
     "react-dropzone": "^14.2.3",
     "react-hook-form": "^7.47.0",
     "react-hot-toast": "^2.4.1",
+    "csstype": "^3.0.10",
     "react-icons": "^4.12.0",
     "react-scroll-to-bottom": "^4.2.0",
     "react-toastify": "^9.1.3",
@@ -66,11 +67,11 @@
     "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-import": "^2.28.1",
     "eslint-plugin-prettier": "^5.0.1",
-    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react": "^7.34.0",
     "eslint-plugin-react-hooks": "^4.6.0",
     "prettier": "^3.0.3",
     "prettier-plugin-tailwindcss": "^0.5.6",
     "rimraf": "^5.0.5",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.3"
   }
 }

web/screens/Settings/ImportingModelModal/ImportingModelItem.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useMemo } from 'react'
 
-import { ImportingModel } from '@janhq/core/.'
+import { ImportingModel } from '@janhq/core'
 import { useSetAtom } from 'jotai'
 
 import { AlertCircle } from 'lucide-react'