Merge branch 'dev' into docs-pena-team

Makefile

@@ -53,15 +53,17 @@ build: check-file-counts
 clean:
 ifeq ($(OS),Windows_NT)
 	powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out -Recurse -Directory | Remove-Item -Recurse -Force"
+	powershell -Command "Get-ChildItem -Path . -Include package-lock.json -Recurse -File | Remove-Item -Recurse -Force"
 	powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz"
 	powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz"
-	rmdir /s /q "%USERPROFILE%\jan\extensions"
+	powershell -Command "if (Test-Path \"$($env:USERPROFILE)\jan\extensions\") { Remove-Item -Path \"$($env:USERPROFILE)\jan\extensions\" -Recurse -Force }"
 else ifeq ($(shell uname -s),Linux)
 	find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 	find . -name ".next" -type d -exec rm -rf '{}' +
 	find . -name "dist" -type d -exec rm -rf '{}' +
 	find . -name "build" -type d -exec rm -rf '{}' +
 	find . -name "out" -type d -exec rm -rf '{}' +
+	find . -name "packake-lock.json" -type f -exec rm -rf '{}' +
 	rm -rf ./pre-install/*.tgz
 	rm -rf ./electron/pre-install/*.tgz
 	rm -rf "~/jan/extensions"
@@ -72,6 +74,7 @@ else
 	find . -name "dist" -type d -exec rm -rf '{}' +
 	find . -name "build" -type d -exec rm -rf '{}' +
 	find . -name "out" -type d -exec rm -rf '{}' +
+	find . -name "packake-lock.json" -type f -exec rm -rf '{}' +
 	rm -rf ./pre-install/*.tgz
 	rm -rf ./electron/pre-install/*.tgz
 	rm -rf ~/jan/extensions

core/package.json

@@ -8,8 +8,8 @@
   ],
   "homepage": "https://jan.ai",
   "license": "AGPL-3.0",
-  "main": "dist/core.umd.js",
+  "main": "dist/core.es5.js",
-  "module": "dist/core.es5.js",
+  "module": "dist/core.cjs.js",
   "typings": "dist/types/index.d.ts",
   "files": [
     "dist",
@@ -17,8 +17,7 @@
   ],
   "author": "Jan <service@jan.ai>",
   "exports": {
-    ".": "./dist/core.umd.js",
+    ".": "./dist/core.es5.js",
-    "./sdk": "./dist/core.umd.js",
     "./node": "./dist/node/index.cjs.js"
   },
   "typesVersions": {
@@ -27,10 +26,6 @@
         "./dist/core.es5.js.map",
         "./dist/types/index.d.ts"
       ],
-      "sdk": [
-        "./dist/core.es5.js.map",
-        "./dist/types/index.d.ts"
-      ],
       "node": [
         "./dist/node/index.cjs.js.map",
         "./dist/types/node/index.d.ts"
@@ -45,6 +40,7 @@
     "start": "rollup -c rollup.config.ts -w"
   },
   "devDependencies": {
+    "@rollup/plugin-replace": "^5.0.5",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.11.4",
     "eslint": "8.57.0",
@@ -63,6 +59,6 @@
   },
   "dependencies": {
     "rxjs": "^7.8.1",
-    "ulid": "^2.3.0"
+    "ulidx": "^2.3.0"
   }
 }

core/rollup.config.ts

@@ -3,17 +3,16 @@ import commonjs from 'rollup-plugin-commonjs'
 import sourceMaps from 'rollup-plugin-sourcemaps'
 import typescript from 'rollup-plugin-typescript2'
 import json from 'rollup-plugin-json'
+import replace from '@rollup/plugin-replace'
 
 const pkg = require('./package.json')
 
-const libraryName = 'core'
-
 export default [
   {
     input: `src/index.ts`,
     output: [
-      { file: pkg.main, name: libraryName, format: 'umd', sourcemap: true },
+      // { file: pkg.main, name: libraryName, format: 'umd', sourcemap: true },
-      { file: pkg.module, format: 'es', sourcemap: true },
+      { file: pkg.main, format: 'es', sourcemap: true },
     ],
     // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
     external: ['path'],
@@ -30,7 +29,13 @@ export default [
       // Allow node_modules resolution, so you can use 'external' to control
       // which external modules to include in the bundle
       // https://github.com/rollup/rollup-plugin-node-resolve#usage
-      resolve(),
+      replace({
+        'node:crypto': 'crypto',
+        'delimiters': ['"', '"'],
+      }),
+      resolve({
+        browser: true,
+      }),
 
       // Resolve source maps to the original source
       sourceMaps(),
@@ -46,7 +51,7 @@ export default [
       'pacote',
       '@types/pacote',
       '@npmcli/arborist',
-      'ulid',
+      'ulidx',
       'node-fetch',
       'fs',
       'request',
@@ -64,7 +69,7 @@ export default [
       // Allow json resolution
       json(),
       // Compile TypeScript files
-      typescript({ useTsconfigDeclarationDir: true, exclude: ['src/*.ts', 'src/extensions/**'] }),
+      typescript({ useTsconfigDeclarationDir: true }),
       // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
       commonjs(),
       // Allow node_modules resolution, so you can use 'external' to control

core/src/core.ts

@@ -13,7 +13,7 @@ const executeOnMain: (extension: string, method: string, ...args: any[]) => Prom
   extension,
   method,
   ...args
-) => global.core?.api?.invokeExtensionFunc(extension, method, ...args)
+) => globalThis.core?.api?.invokeExtensionFunc(extension, method, ...args)
 
 /**
  * Downloads a file from a URL and saves it to the local file system.
@@ -26,7 +26,7 @@ const executeOnMain: (extension: string, method: string, ...args: any[]) => Prom
 const downloadFile: (downloadRequest: DownloadRequest, network?: NetworkConfig) => Promise<any> = (
   downloadRequest,
   network
-) => global.core?.api?.downloadFile(downloadRequest, network)
+) => globalThis.core?.api?.downloadFile(downloadRequest, network)
 
 /**
  * Aborts the download of a specific file.
@@ -34,14 +34,14 @@ const downloadFile: (downloadRequest: DownloadRequest, network?: NetworkConfig)
  * @returns {Promise<any>} A promise that resolves when the download has been aborted.
  */
 const abortDownload: (fileName: string) => Promise<any> = (fileName) =>
-  global.core.api?.abortDownload(fileName)
+  globalThis.core.api?.abortDownload(fileName)
 
 /**
  * Gets Jan's data folder path.
  *
  * @returns {Promise<string>} A Promise that resolves with Jan's data folder path.
  */
-const getJanDataFolderPath = (): Promise<string> => global.core.api?.getJanDataFolderPath()
+const getJanDataFolderPath = (): Promise<string> => globalThis.core.api?.getJanDataFolderPath()
 
 /**
  * Opens the file explorer at a specific path.
@@ -49,21 +49,21 @@ const getJanDataFolderPath = (): Promise<string> => global.core.api?.getJanDataF
  * @returns {Promise<any>} A promise that resolves when the file explorer is opened.
  */
 const openFileExplorer: (path: string) => Promise<any> = (path) =>
-  global.core.api?.openFileExplorer(path)
+  globalThis.core.api?.openFileExplorer(path)
 
 /**
  * Joins multiple paths together.
  * @param paths - The paths to join.
  * @returns {Promise<string>} A promise that resolves with the joined path.
  */
-const joinPath: (paths: string[]) => Promise<string> = (paths) => global.core.api?.joinPath(paths)
+const joinPath: (paths: string[]) => Promise<string> = (paths) => globalThis.core.api?.joinPath(paths)
 
 /**
  * Retrive the basename from an url.
  * @param path - The path to retrieve.
  * @returns {Promise<string>} A promise that resolves with the basename.
  */
-const baseName: (paths: string) => Promise<string> = (path) => global.core.api?.baseName(path)
+const baseName: (paths: string) => Promise<string> = (path) => globalThis.core.api?.baseName(path)
 
 /**
  * Opens an external URL in the default web browser.
@@ -72,20 +72,20 @@ const baseName: (paths: string) => Promise<string> = (path) => global.core.api?.
  * @returns {Promise<any>} - A promise that resolves when the URL has been successfully opened.
  */
 const openExternalUrl: (url: string) => Promise<any> = (url) =>
-  global.core.api?.openExternalUrl(url)
+  globalThis.core.api?.openExternalUrl(url)
 
 /**
  * Gets the resource path of the application.
  *
  * @returns {Promise<string>} - A promise that resolves with the resource path.
  */
-const getResourcePath: () => Promise<string> = () => global.core.api?.getResourcePath()
+const getResourcePath: () => Promise<string> = () => globalThis.core.api?.getResourcePath()
 
 /**
  * Gets the user's home path.
  * @returns return user's home path
  */
-const getUserHomePath = (): Promise<string> => global.core.api?.getUserHomePath()
+const getUserHomePath = (): Promise<string> => globalThis.core.api?.getUserHomePath()
 
 /**
  * Log to file from browser processes.
@@ -93,7 +93,7 @@ const getUserHomePath = (): Promise<string> => global.core.api?.getUserHomePath(
  * @param message - Message to log.
  */
 const log: (message: string, fileName?: string) => void = (message, fileName) =>
-  global.core.api?.log(message, fileName)
+  globalThis.core.api?.log(message, fileName)
 
 /**
  * Check whether the path is a subdirectory of another path.
@@ -104,14 +104,14 @@ const log: (message: string, fileName?: string) => void = (message, fileName) =>
  * @returns {Promise<boolean>} - A promise that resolves with a boolean indicating whether the path is a subdirectory.
  */
 const isSubdirectory: (from: string, to: string) => Promise<boolean> = (from: string, to: string) =>
-  global.core.api?.isSubdirectory(from, to)
+  globalThis.core.api?.isSubdirectory(from, to)
 
 /**
  * Get system information
  * @returns {Promise<any>} - A promise that resolves with the system information.
  */
 const systemInformation: () => Promise<SystemInformation> = () =>
-  global.core.api?.systemInformation()
+  globalThis.core.api?.systemInformation()
 
 /**
  * Show toast message from browser processes.
@@ -120,7 +120,7 @@ const systemInformation: () => Promise<SystemInformation> = () =>
  * @returns
  */
 const showToast: (title: string, message: string) => void = (title, message) =>
-  global.core.api?.showToast(title, message)
+  globalThis.core.api?.showToast(title, message)
 /**
  * Register extension point function type definition
  */

core/src/events.ts

@@ -5,7 +5,7 @@
  * @param handler The handler function to call when the event is observed.
  */
 const on: (eventName: string, handler: Function) => void = (eventName, handler) => {
-  global.core?.events?.on(eventName, handler)
+  globalThis.core?.events?.on(eventName, handler)
 }
 
 /**
@@ -15,7 +15,7 @@ const on: (eventName: string, handler: Function) => void = (eventName, handler)
  * @param handler The handler function to call when the event is observed.
  */
 const off: (eventName: string, handler: Function) => void = (eventName, handler) => {
-  global.core?.events?.off(eventName, handler)
+  globalThis.core?.events?.off(eventName, handler)
 }
 
 /**
@@ -25,7 +25,7 @@ const off: (eventName: string, handler: Function) => void = (eventName, handler)
  * @param object The object to pass to the event callback.
  */
 const emit: (eventName: string, object: any) => void = (eventName, object) => {
-  global.core?.events?.emit(eventName, object)
+  globalThis.core?.events?.emit(eventName, object)
 }
 
 export const events = {

core/src/extensions/ai-engines/OAIEngine.ts

@@ -1,5 +1,5 @@
 import { requestInference } from './helpers/sse'
-import { ulid } from 'ulid'
+import { ulid } from 'ulidx'
 import { AIEngine } from './AIEngine'
 import {
   ChatCompletionRole,

core/src/fs.ts

@@ -4,7 +4,7 @@ import { FileStat } from './types'
  * Writes data to a file at the specified path.
  * @returns {Promise<any>} A Promise that resolves when the file is written successfully.
  */
-const writeFileSync = (...args: any[]) => global.core.api?.writeFileSync(...args)
+const writeFileSync = (...args: any[]) => globalThis.core.api?.writeFileSync(...args)
 
 /**
  * Writes blob data to a file at the specified path.
@@ -13,52 +13,52 @@ const writeFileSync = (...args: any[]) => global.core.api?.writeFileSync(...args
  * @returns
  */
 const writeBlob: (path: string, data: string) => Promise<any> = (path, data) =>
-  global.core.api?.writeBlob(path, data)
+  globalThis.core.api?.writeBlob(path, data)
 
 /**
  * Reads the contents of a file at the specified path.
  * @returns {Promise<any>} A Promise that resolves with the contents of the file.
  */
-const readFileSync = (...args: any[]) => global.core.api?.readFileSync(...args)
+const readFileSync = (...args: any[]) => globalThis.core.api?.readFileSync(...args)
 /**
  * Check whether the file exists
  * @param {string} path
  * @returns {boolean} A boolean indicating whether the path is a file.
  */
-const existsSync = (...args: any[]) => global.core.api?.existsSync(...args)
+const existsSync = (...args: any[]) => globalThis.core.api?.existsSync(...args)
 /**
  * List the directory files
  * @returns {Promise<any>} A Promise that resolves with the contents of the directory.
  */
-const readdirSync = (...args: any[]) => global.core.api?.readdirSync(...args)
+const readdirSync = (...args: any[]) => globalThis.core.api?.readdirSync(...args)
 /**
  * Creates a directory at the specified path.
  * @returns {Promise<any>} A Promise that resolves when the directory is created successfully.
  */
-const mkdirSync = (...args: any[]) => global.core.api?.mkdirSync(...args)
+const mkdirSync = (...args: any[]) => globalThis.core.api?.mkdirSync(...args)
 
-const mkdir = (...args: any[]) => global.core.api?.mkdir(...args)
+const mkdir = (...args: any[]) => globalThis.core.api?.mkdir(...args)
 
 /**
  * Removes a directory at the specified path.
  * @returns {Promise<any>} A Promise that resolves when the directory is removed successfully.
  */
 const rmdirSync = (...args: any[]) =>
-  global.core.api?.rmdirSync(...args, { recursive: true, force: true })
+  globalThis.core.api?.rmdirSync(...args, { recursive: true, force: true })
 
-const rm = (path: string) => global.core.api?.rm(path)
+const rm = (path: string) => globalThis.core.api?.rm(path)
 
 /**
  * Deletes a file from the local file system.
  * @param {string} path - The path of the file to delete.
  * @returns {Promise<any>} A Promise that resolves when the file is deleted.
  */
-const unlinkSync = (...args: any[]) => global.core.api?.unlinkSync(...args)
+const unlinkSync = (...args: any[]) => globalThis.core.api?.unlinkSync(...args)
 
 /**
  * Appends data to a file at the specified path.
  */
-const appendFileSync = (...args: any[]) => global.core.api?.appendFileSync(...args)
+const appendFileSync = (...args: any[]) => globalThis.core.api?.appendFileSync(...args)
 
 /**
  * Synchronizes a file from a source path to a destination path.
@@ -67,15 +67,15 @@ const appendFileSync = (...args: any[]) => global.core.api?.appendFileSync(...ar
  * @returns {Promise<any>} - A promise that resolves when the file has been successfully synchronized.
  */
 const syncFile: (src: string, dest: string) => Promise<any> = (src, dest) =>
-  global.core.api?.syncFile(src, dest)
+  globalThis.core.api?.syncFile(src, dest)
 
 /**
  * Copy file sync.
  */
-const copyFileSync = (...args: any[]) => global.core.api?.copyFileSync(...args)
+const copyFileSync = (...args: any[]) => globalThis.core.api?.copyFileSync(...args)
 
 const copyFile: (src: string, dest: string) => Promise<void> = (src, dest) =>
-  global.core.api?.copyFile(src, dest)
+  globalThis.core.api?.copyFile(src, dest)
 
 /**
  * Gets the file's stats.
@@ -87,7 +87,7 @@ const copyFile: (src: string, dest: string) => Promise<void> = (src, dest) =>
 const fileStat: (path: string, outsideJanDataFolder?: boolean) => Promise<FileStat | undefined> = (
   path,
   outsideJanDataFolder
-) => global.core.api?.fileStat(path, outsideJanDataFolder)
+) => globalThis.core.api?.fileStat(path, outsideJanDataFolder)
 
 // TODO: Export `dummy` fs functions automatically
 // Currently adding these manually

core/src/node/api/restful/helper/builder.ts

@@ -216,7 +216,7 @@ export const createMessage = async (threadId: string, message: any) => {
   const threadMessagesFileName = 'messages.jsonl'
 
   try {
-    const { ulid } = require('ulid')
+    const { ulid } = require('ulidx')
     const msgId = ulid()
     const createdAt = Date.now()
     const threadMessage: ThreadMessage = {

core/src/node/index.ts

@@ -4,3 +4,5 @@ export * from './extension/manager'
 export * from './extension/store'
 export * from './api'
 export * from './helper'
+export * from './../types'
+export * from './../api'

docs/blog/2024-03-19-TensorRT-LLM.md

@@ -1,117 +1,145 @@
 ---
-title: Jan now supports TensorRT-LLM
+title: Benchmarking TensorRT-LLM vs. llama.cpp
-description: Jan has added for Nvidia's TensorRT-LLM, a hardware-optimized LLM inference engine that runs very fast on Nvidia GPUs
+description: Jan has added support for the TensorRT-LLM Inference Engine, as an alternative to llama.cpp. We provide a performance benchmark that shows the head-to-head comparison of the two Inference Engine and model formats, with TensorRT-LLM providing better performance but consumes significantly more VRAM and RAM. 
-tags: [Nvidia, TensorRT-LLM]
+tags: [Nvidia, TensorRT-LLM, llama.cpp, 3090, 4090, "inference engine"]
-keywords: [Nvidia, TensorRT-LLM]
+unlisted: true
 ---
 
-Jan now supports [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) as an alternative inference engine. TensorRT-LLM is a hardware-optimized LLM inference engine that compiles models to [run extremely fast on Nvidia GPUs](https://blogs.nvidia.com/blog/tensorrt-llm-windows-stable-diffusion-rtx/).
+Jan has added support [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) as an alternative to the default [llama.cpp](https://github.com/ggerganov/llama.cpp) inference engine. TensorRT-LLM allows Nvidia GPU owners to run blazing fast LLM inference as a hardware-optimized LLM inference engine that compiles models to [run extremely fast on Nvidia GPUs](https://blogs.nvidia.com/blog/tensorrt-llm-windows-stable-diffusion-rtx/). 
 
-- [TensorRT-LLM Extension](/guides/providers/tensorrt-llm) is available in [0.4.9 release](https://github.com/janhq/jan/releases/tag/v0.4.9)
+You can follow our [TensorRT-LLM Guide](/guides/providers/tensorrt-llm) to try it out today. We've also added a few TensorRT-LLM models to Jan's Model Hub for download:
-- Currently available only for Windows
 
-We've made a few TensorRT-LLM models TensorRT-LLM models available in the Jan Hub for download:
-
-- TinyLlama-1.1b 
 - Mistral 7b
-- TinyJensen-1.1b, which is trained on Jensen Huang's 👀
+- TinyLlama-1.1b
+- TinyJensen-1.1b 😂
 
-## What is TensorRT-LLM?
+:::tip
 
-Please read our [TensorRT-LLM Guide](/guides/providers/tensorrt-llm). 
+TensorRT-LLM support is available in [v0.4.9](https://github.com/janhq/jan/releases/tag/v0.4.9), but should be considered an experimental feature. 
 
-TensorRT-LLM is mainly used in datacenter-grade GPUs to achieve [10,000 tokens/s](https://nvidia.github.io/TensorRT-LLM/blogs/H100vsA100.html) type speeds.
+Please report bugs on [Github](https://github.com/janhq/jan) or on our Discord's [#tensorrt-llm](https://discord.com/channels/1107178041848909847/1201832734704795688) channel.  
+
+:::
 
 ## Performance Benchmarks
 
 
-We were curious to see how this would perform on consumer-grade GPUs, as most of Jan's users use consumer-grade GPUs.
+We were really curious to see how TensorRT-LLM would perform vs. llama.cpp on consumer-grade GPUs. TensorRT-LLM has previously been shown by Nvidia to reach performance of up to [10,000 tokens/s](https://nvidia.github.io/TensorRT-LLM/blogs/H100vsA100.html) on datacenter-grade GPUs. As most of Jan's users are proud card carrying members of the [GPU Poor](https://www.semianalysis.com/p/google-gemini-eats-the-world-gemini#the-gpu-poor), we wanted to see how the two inference engine performed on the same hardware.  
 
-- We’ve done a comparison of how TensorRT-LLM does vs. llama.cpp, our default inference engine.
+:::info
 
-| NVIDIA GPU | Architecture | VRAM Used (GB) | CUDA Cores | Tensor Cores | Memory Bus Width (bit) | Memory Bandwidth (GB/s) |
+An interesting aside: Jan actually started out in June 2023 building on [FastTransformer](https://github.com/NVIDIA/FasterTransformer), the precursor library to TensorRT-LLM. TensorRT-LLM was released in September 2023, making it a very young library. We're excited to see it's roadmap develop!
-| ---------- | ------------ | -------------- | ---------- | ------------ | ---------------------- | ----------------------- |
-| RTX 4090   | Ada          | 24             | 16,384     | 512          | 384                    | ~1000                   |
-| RTX 3090   | Ampere       | 24             | 10,496     | 328          | 384                    | 935.8                   |
-| RTX 4060   | Ada          | 8              | 3,072      | 96           | 128                    | 272                     |
 
-> We test using batch_size 1 and input length 2048, output length 512 as it’s the common use case people all use. We run 5 times and get the Average.
+:::
 
-> We use Windows task manager and Linux NVIDIA-SMI/ Htop to get CPU/ Memory/ NVIDIA GPU metrics per process.
+### Test Setup
 
-> We turn off all user application and only open Jan app with Nitro tensorrt-llm or NVIDIA benchmark script in python
+We picked 3 hardware platforms to run the test on, based on Jan's userbase's self-reported common hardware platforms. 
 
-### RTX 4090 on Windows PC
+| NVIDIA GPU                | VRAM Used (GB) | CUDA Cores | Tensor Cores | Memory Bus Width (bit) | Memory Bandwidth (GB/s) |
+| ------------------------- | -------------- | ---------- | ------------ | ---------------------- | ----------------------- |
+| RTX 4090 Desktop (Ada)    | 24             | 16,384     | 512          | 384                    | ~1000                   |
+| RTX 3090 Desktop (Ampere) | 24             | 10,496     | 328          | 384                    | 935.8                   |
+| RTX 4060 Laptop (Ada)     | 8              | 3,072      | 96           | 128                    | 272                     |
+
+:::warning[Low-spec Machines?]
+
+We didn't bother including low-spec machines: TensorRT-LLM is meant for performance, and simply doesn't work on lower grade Nvidia GPUs, or computers without GPUs.
+
+TensorRT-LLM provides blazing fast performance at the cost of [memory usage](https://nvidia.github.io/TensorRT-LLM/memory.html). This means that the performance improvements only show up in higher-range GPUs with larger VRAMs. 
+
+We've found that [llama.cpp](https://github.com/ggerganov/llama.cpp) does an incredible job of democratizing inference to the [GPU Poor](https://www.semianalysis.com/p/google-gemini-eats-the-world-gemini#the-gpu-poor) with CPU-only or lower-range GPUs. Huge shout outs to the [llama.cpp maintainers](https://github.com/ggerganov/llama.cpp/graphs/contributors) and the [ggml.ai](https://ggml.ai/) team.   
+
+:::
+
+We chose the popular Mistral 7b model to run on both GGUF and TensorRT-LLM, picking comparable quantizations. 
+
+#### llama.cpp Setup
+- For llama.cpp, we used `Mistral-7b-q4_k_m`
+- [ ] Fill in `ngl` params, GPU offload etc
+
+#### TensorRT-LLM Setup
+- For TensorRT-LLM, we used `Mistral-7b-int4 AWQ`
+- We ran TensorRT-LLM with `free_gpu_memory_fraction` to test it with the lowest VRAM consumption (performance may be affected)
+- Note: We picked AWQ for TensorRT-LLM as a handicap as AWQ supposedly sacrifices performance for quality
+
+#### Experiment Setup
+We ran the experiment using a standardized inference request in a sandboxed environment on the same machine:
+- We ran tests 5 times for each inference engine, on a baremetal PC with no other applications open
+- Each inference request was of `batch_size` 1 and `input_len` 2048, `output_len` 512 as a realistic test case
+- CPU and Memory usage were obtained from.... Windows Task Manager 😱
+- GPU usage was obtained from `nvtop`, `htop`, and `nvidia-smi`
+
+## Results
+
+Our biggest takeaway: TensorRT-LLM is faster than llama.cpp on 4090s and 3090s with larger VRAMs. However, on smaller GPUs (e.g. Laptop 4060 GPUs), 
+
+|              | 4090 Desktop | 3090 Desktop | 4060 Laptop |
+| ------------ | ------------ | ------------ | ----------- |
+| TensorRT-LLM | ✅ 159t/s     | ✅ 140.27t/s  | ❌ 19t/s     |
+| llama.cpp    | 101.3t/s     | 90t/s        | 22t/s       |
+
+### RTX-4090 Desktop
+
+:::info[Hardware Details]
 
 - CPU: Intel 13th series
 - GPU: NVIDIA GPU 4090 (Ampere - sm 86)
-- RAM: 120GB
+- RAM: 32GB
-- OS: Windows
+- OS: Windows 11 Pro on Proxmox
 
-#### TinyLlama-1.1b q4
+:::
 
-| Metrics              | GGUF (using the GPU) | TensorRT-LLM |
+Nvidia's RTX-4090 is their top-of-the-line consumer GPU, and retails for [approximately $2,000](https://www.amazon.com/rtx-4090/s?k=rtx+4090). 
-| -------------------- | -------------------- | ------------ |
-| Throughput (token/s) | 104                  | ✅ 131       |
-| VRAM Used (GB)       | 2.1                  | 😱 21.5      |
-| RAM Used (GB)        | 0.3                  | 😱 15        |
-| Disk Size (GB)       | 4.07                 | 4.07         |
 
 #### Mistral-7b int4
 
-| Metrics              | GGUF (using the GPU) | TensorRT-LLM |
+| Metrics              | GGUF (using GPU) | TensorRT-LLM | Difference     |
-| -------------------- | -------------------- | ------------ |
+| -------------------- | -------------------- | ------------ | -------------- |
-| Throughput (token/s) | 80                   | ✅ 97.9      |
+| Throughput (token/s) | 101.3                | 159          | ✅ 57% faster   |
-| VRAM Used (GB)       | 2.1                  | 😱 23.5      |
+| VRAM Used (GB)       | 5.5                  | 6.3          | 🤔 14% more    |
-| RAM Used (GB)        | 0.3                  | 😱 15        |
+| RAM Used (GB)        | 0.54                 | 0.42         | 🤯 20% less    |
-| Disk Size (GB)       | 4.07                 | 4.07         |
+| Disk Size (GB)       | 4.07                 | 3.66         | 🤯 10% smaller |
 
-### RTX 3090 on Windows PC
+
+### RTX-3090 Desktop
+
+:::info[Hardware Details]
 
 - CPU: Intel 13th series
 - GPU: NVIDIA GPU 3090 (Ampere - sm 86)
 - RAM: 64GB
 - OS: Windows
 
-#### TinyLlama-1.1b q4
+:::
-
-| Metrics              | GGUF (using the GPU) | TensorRT-LLM |
-| -------------------- | -------------------- | ------------ |
-| Throughput (token/s) | 131.28               | ✅ 194       |
-| VRAM Used (GB)       | 2.1                  | 😱 21.5      |
-| RAM Used (GB)        | 0.3                  | 😱 15        |
-| Disk Size (GB)       | 4.07                 | 4.07         |
 
 #### Mistral-7b int4
 
-| Metrics              | GGUF (using the GPU) | TensorRT-LLM |
+| Metrics              | GGUF (using GPU) | TensorRT-LLM | Difference   |
-| -------------------- | -------------------- | ------------ |
+| -------------------- | -------------------- | ------------ | ------------ |
-| Throughput (token/s) | 88                   | ✅ 137       |
+| Throughput (token/s) | 90                   | ✅ 140.27     | ✅ 55% faster |
-| VRAM Used (GB)       | 6.0                  | 😱 23.8      |
+| VRAM Used (GB)       | 6.0                  | 6.8          | 🤔 13% more  |
-| RAM Used (GB)        | 0.3                  | 😱 25        |
+| RAM Used (GB)        | 0.54                 | 0.42         | 🤯 22% less  |
-| Disk Size (GB)       | 4.07                 | 4.07         |
+| Disk Size (GB)       | 4.07                 | 3.66         | 🤯 10% less  |
 
-### RTX 4060 on Windows Laptop
+### RTX-4060 Laptop
+
+- [ ] Dan to re-run perf tests and fill in details
+
+:::info[Hardware Details]
 
 - Manufacturer: Acer Nitro 16 Phenix
 - CPU: Ryzen 7000
 - RAM: 16GB
 - GPU: NVIDIA Laptop GPU 4060 (Ada)
 
-#### TinyLlama-1.1b q4
+:::
-
-| Metrics              | GGUF (using the GPU) | TensorRT-LLM |
-| -------------------- | -------------------- | ------------ |
-| Throughput (token/s) | 65                   | ❌ 41        |
-| VRAM Used (GB)       | 2.1                  | 😱 7.6       |
-| RAM Used (GB)        | 0.3                  | 😱 7.2       |
-| Disk Size (GB)       | 4.07                 | 4.07 GB      |
 
 #### Mistral-7b int4
 
-| Metrics              | GGUF (using the GPU) | TensorRT-LLM |
+| Metrics              | GGUF (using the GPU) | TensorRT-LLM | Difference |
-| -------------------- | -------------------- | ------------ |
+| -------------------- | -------------------- | ------------ | ---------- |
-| Throughput (token/s) | 22                   | ❌ 19        |
+| Throughput (token/s) | 22                   | ❌ 19         |            |
-| VRAM Used (GB)       | 2.1                  | 😱 7.7       |
+| VRAM Used (GB)       | 2.1                  | 7.7          |            |
-| RAM Used (GB)        | 0.3                  | 😱 13.5      |
+| RAM Used (GB)        | 0.3                  | 13.5         |            |
-| Disk Size (GB)       | 4.07                 | 4.07         |
+| Disk Size (GB)       | 4.07                 | 4.07         |            |

electron/handlers/native.ts

@@ -6,8 +6,8 @@ import {
   getJanDataFolderPath,
   getJanExtensionsPath,
   init,
+  AppEvent, NativeRoute,
 } from '@janhq/core/node'
-import { AppEvent, NativeRoute } from '@janhq/core'
 
 export function handleAppIPCs() {
   /**

electron/handlers/update.ts

@@ -6,7 +6,7 @@ import {
   UpdateInfo,
   autoUpdater,
 } from 'electron-updater'
-import { AppEvent } from '@janhq/core'
+import { AppEvent } from '@janhq/core/node'
 import { trayManager } from '../managers/tray'
 
 export let waitingToInstallVersion: string | undefined = undefined

electron/managers/window.ts

@@ -1,8 +1,7 @@
 import { BrowserWindow, app, shell } from 'electron'
 import { quickAskWindowConfig } from './quickAskWindowConfig'
-import { AppEvent } from '@janhq/core'
 import { mainWindowConfig } from './mainWindowConfig'
-import { getAppConfigurations } from '@janhq/core/node'
+import { getAppConfigurations, AppEvent } from '@janhq/core/node'
 
 /**
  * Manages the current window instance.

electron/package.json

@@ -90,7 +90,7 @@
     "pacote": "^17.0.4",
     "request": "^2.88.2",
     "request-progress": "^3.0.0",
-    "ulid": "^2.3.0",
+    "ulidx": "^2.3.0",
     "@nut-tree/nut-js": "^4.0.0"
   },
   "devDependencies": {

electron/preload.ts

@@ -3,7 +3,7 @@
  * @module preload
  */
 
-import { APIEvents, APIRoutes } from '@janhq/core'
+import { APIEvents, APIRoutes } from '@janhq/core/node'
 import { contextBridge, ipcRenderer } from 'electron'
 
 const interfaces: { [key: string]: (...args: any[]) => any } = {}

electron/tests/config/fixtures.ts

@@ -22,6 +22,8 @@ export let appInfo: ElectronAppInfo
 export const TIMEOUT = parseInt(process.env.TEST_TIMEOUT || Constants.TIMEOUT)
 
 export async function setupElectron() {
+  console.log(`TEST TIMEOUT: ${TIMEOUT}`)
+
   process.env.CI = 'e2e'
 
   const latestBuild = findLatestBuild('dist')
@@ -101,11 +103,11 @@ export const test = base.extend<
     },
     { auto: true },
   ],
+  
 })
 
-test.setTimeout(TIMEOUT)
-
 test.beforeAll(async () => {
+  test.setTimeout(TIMEOUT)
   await setupElectron()
   await page.waitForSelector('img[alt="Jan - Logo"]', {
     state: 'visible',

extensions/assistant-extension/package.json

@@ -9,9 +9,9 @@
   "scripts": {
     "clean:modules": "rimraf node_modules/pdf-parse/test && cd node_modules/pdf-parse/lib/pdf.js && rimraf v1.9.426 v1.10.88 v2.0.550",
     "build": "yarn clean:modules && tsc --module commonjs && rollup -c rollup.config.ts",
-    "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:linux": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install",
-    "build:publish:darwin": "rimraf *.tgz --glob && npm run build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:darwin": "rimraf *.tgz --glob && yarn build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install",
-    "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:win32": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install",
     "build:publish": "run-script-os"
   },
   "devDependencies": {

extensions/assistant-extension/rollup.config.ts

@@ -36,6 +36,7 @@ export default [
       // https://github.com/rollup/rollup-plugin-node-resolve#usage
       resolve({
         extensions: ['.js', '.ts', '.svelte'],
+        browser: true
       }),
 
       // Resolve source maps to the original source

extensions/assistant-extension/src/index.ts

@@ -10,7 +10,6 @@ import {
   executeOnMain,
   AssistantExtension,
   AssistantEvent,
-  ChatCompletionMessageContentType,
 } from '@janhq/core'
 
 export default class JanAssistantExtension extends AssistantExtension {

extensions/conversational-extension/package.json

@@ -7,7 +7,7 @@
   "license": "MIT",
   "scripts": {
     "build": "tsc -b . && webpack --config webpack.config.js",
-    "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install"
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install"
   },
   "exports": {
     ".": "./dist/index.js",

extensions/conversational-extension/webpack.config.js

@@ -24,7 +24,7 @@ module.exports = {
     extensions: ['.ts', '.js'],
     fallback: {
       path: require.resolve('path-browserify'),
-    },
+    }
   },
   // Do not minify the output, otherwise it breaks the class registration
   optimization: {

extensions/huggingface-extension/src/node/index.ts

@@ -1,8 +1,7 @@
 import { PythonShell } from 'python-shell'
 import { spawn, ChildProcess } from 'child_process'
 import { resolve as presolve, join as pjoin } from 'path'
-import type { Quantization } from '@janhq/core'
+import { log, Quantization } from '@janhq/core/node'
-import { log } from '@janhq/core/node'
 import { statSync } from 'fs'
 export { renameSync } from 'fs'
 

extensions/inference-nitro-extension/package.json

@@ -12,9 +12,9 @@
     "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
     "downloadnitro:win32": "download.bat",
     "downloadnitro": "run-script-os",
-    "build:publish:darwin": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:darwin": "rimraf *.tgz --glob && yarn build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
-    "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:win32": "rimraf *.tgz --glob && yarn build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
-    "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" &&  npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:linux": "rimraf *.tgz --glob && yarn build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
     "build:publish": "run-script-os"
   },
   "exports": {
@@ -45,7 +45,7 @@
     "path-browserify": "^1.0.1",
     "rxjs": "^7.8.1",
     "tcp-port-used": "^1.0.2",
-    "ulid": "^2.3.0"
+    "ulidx": "^2.3.0"
   },
   "engines": {
     "node": ">=18.0.0"

extensions/inference-nitro-extension/rollup.config.ts

@@ -43,6 +43,7 @@ export default [
       // https://github.com/rollup/rollup-plugin-node-resolve#usage
       resolve({
         extensions: ['.js', '.ts', '.svelte'],
+        browser: true,
       }),
 
       // Resolve source maps to the original source

extensions/inference-nitro-extension/src/index.ts

@@ -29,7 +29,7 @@ import {
   getJanDataFolderPath,
 } from '@janhq/core'
 import { requestInference } from './helpers/sse'
-import { ulid } from 'ulid'
+import { ulid } from 'ulidx'
 
 /**
  * A class that implements the InferenceExtension interface from the @janhq/core package.

extensions/inference-nitro-extension/src/node/index.ts

@@ -3,13 +3,14 @@ import path from 'path'
 import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
 import tcpPortUsed from 'tcp-port-used'
 import fetchRT from 'fetch-retry'
-import { log, getSystemResourceInfo } from '@janhq/core/node'
 import {
+  log,
+  getSystemResourceInfo,
   Model,
   InferenceEngine,
   ModelSettingParams,
   PromptTemplate,
-} from '@janhq/core'
+} from '@janhq/core/node'
 import { executableNitroFile } from './execute'
 
 // Polyfill fetch with retry

extensions/inference-openai-extension/package.json

@@ -8,7 +8,7 @@
   "license": "AGPL-3.0",
   "scripts": {
     "build": "tsc -b . && webpack --config webpack.config.js",
-    "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install"
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install"
   },
   "exports": {
     ".": "./dist/index.js",
@@ -25,7 +25,7 @@
     "@janhq/core": "file:../../core",
     "fetch-retry": "^5.0.6",
     "path-browserify": "^1.0.1",
-    "ulid": "^2.3.0"
+    "ulidx": "^2.3.0"
   },
   "engines": {
     "node": ">=18.0.0"

extensions/inference-openai-extension/src/index.ts

@@ -25,7 +25,7 @@ import {
   joinPath,
 } from '@janhq/core'
 import { requestInference } from './helpers/sse'
-import { ulid } from 'ulid'
+import { ulid } from 'ulidx'
 import { join } from 'path'
 
 /**

extensions/inference-triton-trtllm-extension/package.json

@@ -8,7 +8,7 @@
   "license": "AGPL-3.0",
   "scripts": {
     "build": "tsc -b . && webpack --config webpack.config.js",
-    "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install"
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install"
   },
   "exports": {
     ".": "./dist/index.js",
@@ -17,16 +17,17 @@
   "devDependencies": {
     "cpx": "^1.5.0",
     "rimraf": "^3.0.2",
+    "ts-loader": "^9.5.0",
+    "typescript": "5.3.3",
     "webpack": "^5.88.2",
-    "webpack-cli": "^5.1.4",
+    "webpack-cli": "^5.1.4"
-    "ts-loader": "^9.5.0"
   },
   "dependencies": {
     "@janhq/core": "file:../../core",
     "fetch-retry": "^5.0.6",
     "path-browserify": "^1.0.1",
-    "ulid": "^2.3.0",
+    "rxjs": "^7.8.1",
-    "rxjs": "^7.8.1"
+    "ulidx": "^2.3.0"
   },
   "engines": {
     "node": ">=18.0.0"

extensions/inference-triton-trtllm-extension/src/index.ts

@@ -22,7 +22,7 @@ import {
   ModelEvent,
 } from '@janhq/core'
 import { requestInference } from './helpers/sse'
-import { ulid } from 'ulid'
+import { ulid } from 'ulidx'
 import { join } from 'path'
 import { EngineSettings } from './@types/global'
 

extensions/model-extension/package.json

@@ -7,15 +7,23 @@
   "author": "Jan <service@jan.ai>",
   "license": "AGPL-3.0",
   "scripts": {
-    "build": "tsc -b . && webpack --config webpack.config.js",
+    "build": "rollup -c rollup.config.ts",
-    "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install"
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install"
   },
   "devDependencies": {
     "cpx": "^1.5.0",
     "rimraf": "^3.0.2",
-    "webpack": "^5.88.2",
+    "ts-loader": "^9.5.0",
-    "webpack-cli": "^5.1.4",
+    "typescript": "5.3.3",
-    "ts-loader": "^9.5.0"
+    "@rollup/plugin-commonjs": "^25.0.7",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^15.2.3",
+    "@rollup/plugin-replace": "^5.0.5",
+    "@types/pdf-parse": "^1.1.4",
+    "rollup": "^2.38.5",
+    "rollup-plugin-define": "^1.0.1",
+    "rollup-plugin-sourcemaps": "^0.6.3",
+    "rollup-plugin-typescript2": "^0.36.0"
   },
   "files": [
     "dist/*",

extensions/model-extension/rollup.config.ts

@@ -0,0 +1,48 @@
+import resolve from '@rollup/plugin-node-resolve'
+import commonjs from '@rollup/plugin-commonjs'
+import sourceMaps from 'rollup-plugin-sourcemaps'
+import typescript from 'rollup-plugin-typescript2'
+import json from '@rollup/plugin-json'
+import replace from '@rollup/plugin-replace'
+
+const packageJson = require('./package.json')
+
+const pkg = require('./package.json')
+
+export default [
+  {
+    input: `src/index.ts`,
+    output: [{ file: pkg.main, format: 'es', sourcemap: true }],
+    // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+    external: [],
+    watch: {
+      include: 'src/**',
+    },
+    plugins: [
+      replace({
+        EXTENSION_NAME: JSON.stringify(packageJson.name),
+        MODULE_PATH: JSON.stringify(
+          `${packageJson.name}/${packageJson.module}`
+        ),
+        VERSION: JSON.stringify(packageJson.version),
+      }),
+      // Allow json resolution
+      json(),
+      //     Compile TypeScript files
+      typescript({ useTsconfigDeclarationDir: true }),
+      // Compile TypeScript files
+      // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+      // commonjs(),
+      // Allow node_modules resolution, so you can use 'external' to control
+      // which external modules to include in the bundle
+      // https://github.com/rollup/rollup-plugin-node-resolve#usage
+      resolve({
+        extensions: ['.js', '.ts', '.svelte'],
+        browser: true,
+      }),
+
+      // Resolve source maps to the original source
+      sourceMaps(),
+    ],
+  },
+]

extensions/model-extension/webpack.config.js

@@ -1,40 +0,0 @@
-const path = require('path')
-const webpack = require('webpack')
-const packageJson = require('./package.json')
-
-module.exports = {
-  experiments: { outputModule: true },
-  entry: './src/index.ts', // Adjust the entry point to match your project's main file
-  mode: 'production',
-  module: {
-    rules: [
-      {
-        test: /\.tsx?$/,
-        use: 'ts-loader',
-        exclude: /node_modules/,
-      },
-    ],
-  },
-  plugins: [
-    new webpack.DefinePlugin({
-      EXTENSION_NAME: JSON.stringify(packageJson.name),
-      MODULE_PATH: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
-      VERSION: JSON.stringify(packageJson.version),
-    }),
-  ],
-  output: {
-    filename: 'index.js', // Adjust the output file name as needed
-    path: path.resolve(__dirname, 'dist'),
-    library: { type: 'module' }, // Specify ESM output format
-  },
-  resolve: {
-    extensions: ['.ts', '.js'],
-    fallback: {
-      path: require.resolve('path-browserify'),
-    },
-  },
-  optimization: {
-    minimize: false,
-  },
-  // Add loaders and other configuration as needed for your project
-}

extensions/monitoring-extension/package.json

@@ -12,7 +12,7 @@
     "download-artifacts:darwin": "echo 'No artifacts to download for darwin'",
     "download-artifacts:win32": "download.bat",
     "download-artifacts:linux": "download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo",
-    "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install"
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install"
   },
   "exports": {
     ".": "./dist/index.js",

extensions/monitoring-extension/src/node/index.ts

@@ -4,8 +4,9 @@ import {
   OperatingSystemInfo,
   ResourceInfo,
   SupportedPlatforms,
-} from '@janhq/core'
+  getJanDataFolderPath,
-import { getJanDataFolderPath, log } from '@janhq/core/node'
+  log,
+} from '@janhq/core/node'
 import { mem, cpu } from 'node-os-utils'
 import { exec } from 'child_process'
 import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs'

extensions/tensorrt-llm-extension/package.json

@@ -22,9 +22,9 @@
   "provider": "nitro-tensorrt-llm",
   "scripts": {
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
-    "build:publish:win32": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:win32": "rimraf *.tgz --glob && yarn build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
-    "build:publish:linux": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" &&  npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:linux": "rimraf *.tgz --glob && yarn build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
-    "build:publish:darwin": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" &&  npm pack && cpx *.tgz ../../pre-install",
+    "build:publish:darwin": "rimraf *.tgz --glob && yarn build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
     "build:publish": "run-script-os"
   },
   "exports": {
@@ -57,7 +57,7 @@
     "path-browserify": "^1.0.1",
     "rxjs": "^7.8.1",
     "tcp-port-used": "^1.0.2",
-    "ulid": "^2.3.0"
+    "ulidx": "^2.3.0"
   },
   "engines": {
     "node": ">=18.0.0"

extensions/tensorrt-llm-extension/src/node/index.ts

@@ -2,10 +2,13 @@ import path from 'path'
 import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
 import tcpPortUsed from 'tcp-port-used'
 import fetchRT from 'fetch-retry'
-import { log, getJanDataFolderPath } from '@janhq/core/node'
+import {
+  log,
+  getJanDataFolderPath,
+  SystemInformation,
+  PromptTemplate,
+} from '@janhq/core/node'
 import decompress from 'decompress'
-import { SystemInformation } from '@janhq/core'
-import { PromptTemplate } from '@janhq/core'
 
 // Polyfill fetch with retry
 const fetchRetry = fetchRT(fetch)

package.json

@@ -34,9 +34,9 @@
     "build:web": "yarn workspace jan-web build && cpx \"web/out/**\" \"electron/renderer/\"",
     "build:electron": "yarn copy:assets && yarn workspace jan build",
     "build:electron:test": "yarn workspace jan build:test",
-    "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; npm install; npm run build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"",
+    "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; yarn; yarn build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"",
-    "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'",
+    "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'",
-    "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'",
+    "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {}; yarn; yarn build:publish'",
     "build:extensions:server": "yarn workspace build:extensions ",
     "build:extensions": "run-script-os",
     "build:test": "yarn copy:assets && yarn build:web && yarn workspace jan build:test",

web/containers/Providers/EventHandler.tsx

@@ -17,7 +17,7 @@ import {
   ModelInitFailed,
 } from '@janhq/core'
 import { useAtomValue, useSetAtom } from 'jotai'
-import { ulid } from 'ulid'
+import { ulid } from 'ulidx'
 
 import {
   activeModelAtom,

web/hooks/useSendChatMessage.ts

@@ -21,7 +21,7 @@ import {
 } from '@janhq/core'
 import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai'
 
-import { ulid } from 'ulid'
+import { ulid } from 'ulidx'
 
 import { selectedModelAtom } from '@/containers/DropdownListSidebar'
 import {

web/package.json

@@ -44,7 +44,7 @@
     "sass": "^1.69.4",
     "tailwind-merge": "^2.0.0",
     "tailwindcss": "3.3.5",
-    "ulid": "^2.3.0",
+    "ulidx": "^2.3.0",
     "uuid": "^9.0.1",
     "zod": "^3.22.4"
   },