Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: prettier fix (#2019)

Browse Source
This commit is contained in:
Louis 2024-02-15 08:38:05 +07:00 committed by GitHub
parent b11f51d80f
commit 3412a23654
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
65 changed files with 995 additions and 992 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.devcontainer/devcontainer.json
View File

@ -1,4 +1,4 @@
{ {
"name": "jan", "name": "jan",
"image": "node:20" "image": "node:20"
} }

2
core/jest.config.js
View File

@ -4,4 +4,4 @@ module.exports = {
moduleNameMapper: { moduleNameMapper: {
'@/(.*)': '<rootDir>/src/$1', '@/(.*)': '<rootDir>/src/$1',
}, },
} }

2
core/rollup.config.ts
View File

@ -55,7 +55,7 @@ export default [
'http', 'http',
'os', 'os',
'util', 'util',
'child_process' 'child_process',
], ],
watch: { watch: {
include: 'src/node/**', include: 'src/node/**',

5
core/src/api/index.ts
View File

@ -125,8 +125,5 @@ export const CoreRoutes = [
...Object.values(FileManagerRoute), ...Object.values(FileManagerRoute),
] ]
export const APIRoutes = [ export const APIRoutes = [...CoreRoutes, ...Object.values(NativeRoute)]
...CoreRoutes,
...Object.values(NativeRoute),
]
export const APIEvents = [...Object.values(AppEvent), ...Object.values(DownloadEvent)] export const APIEvents = [...Object.values(AppEvent), ...Object.values(DownloadEvent)]

18
core/src/extension.ts
View File

@ -1,13 +1,13 @@
export enum ExtensionTypeEnum { export enum ExtensionTypeEnum {
Assistant = "assistant", Assistant = 'assistant',
Conversational = "conversational", Conversational = 'conversational',
Inference = "inference", Inference = 'inference',
Model = "model", Model = 'model',
SystemMonitoring = "systemMonitoring", SystemMonitoring = 'systemMonitoring',
} }
export interface ExtensionType { export interface ExtensionType {
type(): ExtensionTypeEnum | undefined; type(): ExtensionTypeEnum | undefined
} }
/** /**
* Represents a base extension. * Represents a base extension.
@ -20,16 +20,16 @@ export abstract class BaseExtension implements ExtensionType {
* Undefined means its not extending any known extension by the application. * Undefined means its not extending any known extension by the application.
*/ */
type(): ExtensionTypeEnum | undefined { type(): ExtensionTypeEnum | undefined {
return undefined; return undefined
} }
/** /**
* Called when the extension is loaded. * Called when the extension is loaded.
* Any initialization logic for the extension should be put here. * Any initialization logic for the extension should be put here.
*/ */
abstract onLoad(): void; abstract onLoad(): void
/** /**
* Called when the extension is unloaded. * Called when the extension is unloaded.
* Any cleanup logic for the extension should be put here. * Any cleanup logic for the extension should be put here.
*/ */
abstract onUnload(): void; abstract onUnload(): void
} }

12
core/src/extensions/assistant.ts
View File

@ -1,5 +1,5 @@
import { Assistant, AssistantInterface } from "../index"; import { Assistant, AssistantInterface } from '../index'
import { BaseExtension, ExtensionTypeEnum } from "../extension"; import { BaseExtension, ExtensionTypeEnum } from '../extension'
/** /**
* Assistant extension for managing assistants. * Assistant extension for managing assistants.
@ -10,10 +10,10 @@ export abstract class AssistantExtension extends BaseExtension implements Assist
* Assistant extension type. * Assistant extension type.
*/ */
type(): ExtensionTypeEnum | undefined { type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Assistant; return ExtensionTypeEnum.Assistant
} }
abstract createAssistant(assistant: Assistant): Promise<void>; abstract createAssistant(assistant: Assistant): Promise<void>
abstract deleteAssistant(assistant: Assistant): Promise<void>; abstract deleteAssistant(assistant: Assistant): Promise<void>
abstract getAssistants(): Promise<Assistant[]>; abstract getAssistants(): Promise<Assistant[]>
} }

2
core/src/extensions/conversational.ts
View File

@ -14,7 +14,7 @@ export abstract class ConversationalExtension
* Conversation extension type. * Conversation extension type.
*/ */
type(): ExtensionTypeEnum | undefined { type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Conversational; return ExtensionTypeEnum.Conversational
} }
abstract getThreads(): Promise<Thread[]> abstract getThreads(): Promise<Thread[]>

10
core/src/extensions/index.ts
View File

@ -2,24 +2,24 @@
* Conversational extension. Persists and retrieves conversations. * Conversational extension. Persists and retrieves conversations.
* @module * @module
*/ */
export { ConversationalExtension } from "./conversational"; export { ConversationalExtension } from './conversational'
/** /**
* Inference extension. Start, stop and inference models. * Inference extension. Start, stop and inference models.
*/ */
export { InferenceExtension } from "./inference"; export { InferenceExtension } from './inference'
/** /**
* Monitoring extension for system monitoring. * Monitoring extension for system monitoring.
*/ */
export { MonitoringExtension } from "./monitoring"; export { MonitoringExtension } from './monitoring'
/** /**
* Assistant extension for managing assistants. * Assistant extension for managing assistants.
*/ */
export { AssistantExtension } from "./assistant"; export { AssistantExtension } from './assistant'
/** /**
* Model extension for managing models. * Model extension for managing models.
*/ */
export { ModelExtension } from "./model"; export { ModelExtension } from './model'

8
core/src/extensions/inference.ts
View File

@ -1,5 +1,5 @@
import { InferenceInterface, MessageRequest, ThreadMessage } from "../index"; import { InferenceInterface, MessageRequest, ThreadMessage } from '../index'
import { BaseExtension, ExtensionTypeEnum } from "../extension"; import { BaseExtension, ExtensionTypeEnum } from '../extension'
/** /**
* Inference extension. Start, stop and inference models. * Inference extension. Start, stop and inference models.
@ -9,8 +9,8 @@ export abstract class InferenceExtension extends BaseExtension implements Infere
* Inference extension type. * Inference extension type.
*/ */
type(): ExtensionTypeEnum | undefined { type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Inference; return ExtensionTypeEnum.Inference
} }
abstract inference(data: MessageRequest): Promise<ThreadMessage>; abstract inference(data: MessageRequest): Promise<ThreadMessage>
} }

20
core/src/extensions/model.ts
View File

@ -1,5 +1,5 @@
import { BaseExtension, ExtensionTypeEnum } from "../extension"; import { BaseExtension, ExtensionTypeEnum } from '../extension'
import { Model, ModelInterface } from "../index"; import { Model, ModelInterface } from '../index'
/** /**
* Model extension for managing models. * Model extension for managing models.
@ -9,16 +9,16 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter
* Model extension type. * Model extension type.
*/ */
type(): ExtensionTypeEnum | undefined { type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Model; return ExtensionTypeEnum.Model
} }
abstract downloadModel( abstract downloadModel(
model: Model, model: Model,
network?: { proxy: string; ignoreSSL?: boolean }, network?: { proxy: string; ignoreSSL?: boolean }
): Promise<void>; ): Promise<void>
abstract cancelModelDownload(modelId: string): Promise<void>; abstract cancelModelDownload(modelId: string): Promise<void>
abstract deleteModel(modelId: string): Promise<void>; abstract deleteModel(modelId: string): Promise<void>
abstract saveModel(model: Model): Promise<void>; abstract saveModel(model: Model): Promise<void>
abstract getDownloadedModels(): Promise<Model[]>; abstract getDownloadedModels(): Promise<Model[]>
abstract getConfiguredModels(): Promise<Model[]>; abstract getConfiguredModels(): Promise<Model[]>
} }

10
core/src/extensions/monitoring.ts
View File

@ -1,5 +1,5 @@
import { BaseExtension, ExtensionTypeEnum } from "../extension"; import { BaseExtension, ExtensionTypeEnum } from '../extension'
import { MonitoringInterface } from "../index"; import { MonitoringInterface } from '../index'
/** /**
* Monitoring extension for system monitoring. * Monitoring extension for system monitoring.
@ -10,9 +10,9 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit
* Monitoring extension type. * Monitoring extension type.
*/ */
type(): ExtensionTypeEnum | undefined { type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.SystemMonitoring; return ExtensionTypeEnum.SystemMonitoring
} }
abstract getResourcesInfo(): Promise<any>; abstract getResourcesInfo(): Promise<any>
abstract getCurrentLoad(): Promise<any>; abstract getCurrentLoad(): Promise<any>
} }

8
core/src/node/api/common/adapter.ts
View File

@ -1,4 +1,10 @@
import { AppRoute, DownloadRoute, ExtensionRoute, FileManagerRoute, FileSystemRoute } from '../../../api' import {
AppRoute,
DownloadRoute,
ExtensionRoute,
FileManagerRoute,
FileSystemRoute,
} from '../../../api'
import { Downloader } from '../processors/download' import { Downloader } from '../processors/download'
import { FileSystem } from '../processors/fs' import { FileSystem } from '../processors/fs'
import { Extension } from '../processors/extension' import { Extension } from '../processors/extension'

2
core/src/node/api/restful/v1.ts
View File

@ -6,7 +6,7 @@ import { handleRequests } from './app/handlers'
export const v1Router = async (app: HttpServer) => { export const v1Router = async (app: HttpServer) => {
// MARK: Public API Routes // MARK: Public API Routes
app.register(commonRouter) app.register(commonRouter)
// MARK: Internal Application Routes // MARK: Internal Application Routes
handleRequests(app) handleRequests(app)

2
core/src/node/extension/extension.ts
View File

@ -104,7 +104,7 @@ export default class Extension {
await pacote.extract( await pacote.extract(
this.specifier, this.specifier,
join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''), join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''),
this.installOptions, this.installOptions
) )
// Set the url using the custom extensions protocol // Set the url using the custom extensions protocol

61
core/src/node/extension/store.ts
View File

@ -1,6 +1,6 @@
import { writeFileSync } from "fs"; import { writeFileSync } from 'fs'
import Extension from "./extension"; import Extension from './extension'
import { ExtensionManager } from "./manager"; import { ExtensionManager } from './manager'
/** /**
* @module store * @module store
@ -11,7 +11,7 @@ import { ExtensionManager } from "./manager";
* Register of installed extensions * Register of installed extensions
* @type {Object.<string, Extension>} extension - List of installed extensions * @type {Object.<string, Extension>} extension - List of installed extensions
*/ */
const extensions: Record<string, Extension> = {}; const extensions: Record<string, Extension> = {}
/** /**
* Get a extension from the stored extensions. * Get a extension from the stored extensions.
@ -21,10 +21,10 @@ const extensions: Record<string, Extension> = {};
*/ */
export function getExtension(name: string) { export function getExtension(name: string) {
if (!Object.prototype.hasOwnProperty.call(extensions, name)) { if (!Object.prototype.hasOwnProperty.call(extensions, name)) {
throw new Error(`Extension ${name} does not exist`); throw new Error(`Extension ${name} does not exist`)
} }
return extensions[name]; return extensions[name]
} }
/** /**
@ -33,7 +33,7 @@ export function getExtension(name: string) {
* @alias extensionManager.getAllExtensions * @alias extensionManager.getAllExtensions
*/ */
export function getAllExtensions() { export function getAllExtensions() {
return Object.values(extensions); return Object.values(extensions)
} }
/** /**
@ -42,7 +42,7 @@ export function getAllExtensions() {
* @alias extensionManager.getActiveExtensions * @alias extensionManager.getActiveExtensions
*/ */
export function getActiveExtensions() { export function getActiveExtensions() {
return Object.values(extensions).filter((extension) => extension.active); return Object.values(extensions).filter((extension) => extension.active)
} }
/** /**
@ -53,9 +53,9 @@ export function getActiveExtensions() {
* @alias extensionManager.removeExtension * @alias extensionManager.removeExtension
*/ */
export function removeExtension(name: string, persist = true) { export function removeExtension(name: string, persist = true) {
const del = delete extensions[name]; const del = delete extensions[name]
if (persist) persistExtensions(); if (persist) persistExtensions()
return del; return del
} }
/** /**
@ -65,10 +65,10 @@ export function removeExtension(name: string, persist = true) {
* @returns {void} * @returns {void}
*/ */
export function addExtension(extension: Extension, persist = true) { export function addExtension(extension: Extension, persist = true) {
if (extension.name) extensions[extension.name] = extension; if (extension.name) extensions[extension.name] = extension
if (persist) { if (persist) {
persistExtensions(); persistExtensions()
extension.subscribe("pe-persist", persistExtensions); extension.subscribe('pe-persist', persistExtensions)
} }
} }
@ -77,14 +77,11 @@ export function addExtension(extension: Extension, persist = true) {
* @returns {void} * @returns {void}
*/ */
export function persistExtensions() { export function persistExtensions() {
const persistData: Record<string, Extension> = {}; const persistData: Record<string, Extension> = {}
for (const name in extensions) { for (const name in extensions) {
persistData[name] = extensions[name]; persistData[name] = extensions[name]
} }
writeFileSync( writeFileSync(ExtensionManager.instance.getExtensionsFile(), JSON.stringify(persistData))
ExtensionManager.instance.getExtensionsFile(),
JSON.stringify(persistData),
);
} }
/** /**
@ -95,26 +92,28 @@ export function persistExtensions() {
* @alias extensionManager.installExtensions * @alias extensionManager.installExtensions
*/ */
export async function installExtensions(extensions: any) { export async function installExtensions(extensions: any) {
const installed: Extension[] = []; const installed: Extension[] = []
for (const ext of extensions) { for (const ext of extensions) {
// Set install options and activation based on input type // Set install options and activation based on input type
const isObject = typeof ext === "object"; const isObject = typeof ext === 'object'
const spec = isObject ? [ext.specifier, ext] : [ext]; const spec = isObject ? [ext.specifier, ext] : [ext]
const activate = isObject ? ext.activate !== false : true; const activate = isObject ? ext.activate !== false : true
// Install and possibly activate extension // Install and possibly activate extension
const extension = new Extension(...spec); const extension = new Extension(...spec)
if(!extension.origin) { continue } if (!extension.origin) {
await extension._install(); continue
if (activate) extension.setActive(true); }
await extension._install()
if (activate) extension.setActive(true)
// Add extension to store if needed // Add extension to store if needed
addExtension(extension); addExtension(extension)
installed.push(extension); installed.push(extension)
} }
// Return list of all installed extensions // Return list of all installed extensions
return installed; return installed
} }
/** /**

2
core/src/node/helper/path.ts
View File

@ -1,4 +1,4 @@
import { join } from "path" import { join } from 'path'
/** /**
* Normalize file path * Normalize file path

6
core/src/node/helper/resource.ts
View File

@ -1,6 +1,6 @@
import { SystemResourceInfo } from "../../types" import { SystemResourceInfo } from '../../types'
import { physicalCpuCount } from "./config" import { physicalCpuCount } from './config'
import { log, logServer } from "./log" import { log, logServer } from './log'
export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => { export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
const cpu = await physicalCpuCount() const cpu = await physicalCpuCount()

7
core/src/types/assistant/assistantEvent.ts
View File

@ -2,7 +2,6 @@
* The `EventName` enumeration contains the names of all the available events in the Jan platform. * The `EventName` enumeration contains the names of all the available events in the Jan platform.
*/ */
export enum AssistantEvent { export enum AssistantEvent {
/** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */ /** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */
OnAssistantsUpdate = 'OnAssistantsUpdate', OnAssistantsUpdate = 'OnAssistantsUpdate',
} }

8
core/src/types/message/messageRequestType.ts
View File

@ -1,5 +1,5 @@
export enum MessageRequestType { export enum MessageRequestType {
Thread = 'Thread', Thread = 'Thread',
Assistant = 'Assistant', Assistant = 'Assistant',
Summary = 'Summary', Summary = 'Summary',
} }

2
core/src/types/model/modelInterface.ts
View File

@ -10,7 +10,7 @@ export interface ModelInterface {
* @param network - Optional object to specify proxy/whether to ignore SSL certificates. * @param network - Optional object to specify proxy/whether to ignore SSL certificates.
* @returns A Promise that resolves when the model has been downloaded. * @returns A Promise that resolves when the model has been downloaded.
*/ */
downloadModel(model: Model, network?: { ignoreSSL?: boolean, proxy?: string }): Promise<void> downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise<void>
/** /**
* Cancels the download of a specific model. * Cancels the download of a specific model.

7
core/tslint.json
View File

@ -1,6 +1,3 @@
{ {
"extends": [ "extends": ["tslint-config-standard", "tslint-config-prettier"]
"tslint-config-standard", }
"tslint-config-prettier"
]
}

4
electron/main.ts
View File

@ -1,4 +1,4 @@
import { app, BrowserWindow } from 'electron' import { app, BrowserWindow, shell } from 'electron'
import { join } from 'path' import { join } from 'path'
/** /**
* Managers * Managers
@ -77,7 +77,7 @@ function createMainWindow() {
/* Open external links in the default browser */ /* Open external links in the default browser */
mainWindow.webContents.setWindowOpenHandler(({ url }) => { mainWindow.webContents.setWindowOpenHandler(({ url }) => {
require('electron').shell.openExternal(url) shell.openExternal(url)
return { action: 'deny' } return { action: 'deny' }
}) })

4
electron/merge-latest-ymls.js
View File

@ -9,7 +9,9 @@ const file3 = args[2]
// check that all arguments are present and throw error instead // check that all arguments are present and throw error instead
if (!file1 || !file2 || !file3) { if (!file1 || !file2 || !file3) {
throw new Error('Please provide 3 file paths as arguments: path to file1, to file2 and destination path') throw new Error(
'Please provide 3 file paths as arguments: path to file1, to file2 and destination path'
)
} }
const doc1 = yaml.load(fs.readFileSync(file1, 'utf8')) const doc1 = yaml.load(fs.readFileSync(file1, 'utf8'))

82
electron/sign.js
View File

@ -1,44 +1,48 @@
const { exec } = require('child_process'); const { exec } = require('child_process')
function sign({
path,
name,
certUrl,
clientId,
tenantId,
clientSecret,
certName,
timestampServer,
version,
}) {
return new Promise((resolve, reject) => {
const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`
function sign({ path, name, certUrl, clientId, tenantId, clientSecret, certName, timestampServer, version }) { exec(command, (error, stdout, stderr) => {
return new Promise((resolve, reject) => { if (error) {
console.error(`Error: ${error}`)
const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`; return reject(error)
}
console.log(`stdout: ${stdout}`)
exec(command, (error, stdout, stderr) => { console.error(`stderr: ${stderr}`)
if (error) { resolve()
console.error(`Error: ${error}`); })
return reject(error); })
}
console.log(`stdout: ${stdout}`);
console.error(`stderr: ${stderr}`);
resolve();
});
});
} }
exports.default = async function (options) {
const certUrl = process.env.AZURE_KEY_VAULT_URI
const clientId = process.env.AZURE_CLIENT_ID
const tenantId = process.env.AZURE_TENANT_ID
const clientSecret = process.env.AZURE_CLIENT_SECRET
const certName = process.env.AZURE_CERT_NAME
const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1'
exports.default = async function(options) { await sign({
path: options.path,
const certUrl = process.env.AZURE_KEY_VAULT_URI; name: 'jan-win-x64',
const clientId = process.env.AZURE_CLIENT_ID; certUrl,
const tenantId = process.env.AZURE_TENANT_ID; clientId,
const clientSecret = process.env.AZURE_CLIENT_SECRET; tenantId,
const certName = process.env.AZURE_CERT_NAME; clientSecret,
const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1'; certName,
timestampServer,
version: options.version,
await sign({ })
path: options.path, }
name: "jan-win-x64",
certUrl,
clientId,
tenantId,
clientSecret,
certName,
timestampServer,
version: options.version
});
};

6
electron/utils/disposable.ts
View File

@ -1,8 +1,8 @@
export function dispose(requiredModules: Record<string, any>) { export function dispose(requiredModules: Record<string, any>) {
for (const key in requiredModules) { for (const key in requiredModules) {
const module = requiredModules[key]; const module = requiredModules[key]
if (typeof module["dispose"] === "function") { if (typeof module['dispose'] === 'function') {
module["dispose"](); module['dispose']()
} }
} }
} }

12
electron/utils/setup.ts
View File

@ -1,9 +1,9 @@
import { app } from 'electron' import { app } from 'electron'
export const setupCore = async () => { export const setupCore = async () => {
// Setup core api for main process // Setup core api for main process
global.core = { global.core = {
// Define appPath function for app to retrieve app path globaly // Define appPath function for app to retrieve app path globaly
appPath: () => app.getPath('userData') appPath: () => app.getPath('userData'),
} }
} }

36
extensions/assistant-extension/rollup.config.ts
View File

@ -1,22 +1,22 @@
import resolve from "@rollup/plugin-node-resolve"; import resolve from '@rollup/plugin-node-resolve'
import commonjs from "@rollup/plugin-commonjs"; import commonjs from '@rollup/plugin-commonjs'
import sourceMaps from "rollup-plugin-sourcemaps"; import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from "rollup-plugin-typescript2"; import typescript from 'rollup-plugin-typescript2'
import json from "@rollup/plugin-json"; import json from '@rollup/plugin-json'
import replace from "@rollup/plugin-replace"; import replace from '@rollup/plugin-replace'
const packageJson = require("./package.json"); const packageJson = require('./package.json')
const pkg = require("./package.json"); const pkg = require('./package.json')
export default [ export default [
{ {
input: `src/index.ts`, input: `src/index.ts`,
output: [{ file: pkg.main, format: "es", sourcemap: true }], output: [{ file: pkg.main, format: 'es', sourcemap: true }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [], external: [],
watch: { watch: {
include: "src/**", include: 'src/**',
}, },
plugins: [ plugins: [
replace({ replace({
@ -35,7 +35,7 @@ export default [
// which external modules to include in the bundle // which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage // https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({ resolve({
extensions: [".js", ".ts", ".svelte"], extensions: ['.js', '.ts', '.svelte'],
}), }),
// Resolve source maps to the original source // Resolve source maps to the original source
@ -44,15 +44,11 @@ export default [
}, },
{ {
input: `src/node/index.ts`, input: `src/node/index.ts`,
output: [{ dir: "dist/node", format: "cjs", sourcemap: false }], output: [{ dir: 'dist/node', format: 'cjs', sourcemap: false }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [ external: ['@janhq/core/node', 'path', 'hnswlib-node'],
"@janhq/core/node",
"path",
"hnswlib-node",
],
watch: { watch: {
include: "src/node/**", include: 'src/node/**',
}, },
// inlineDynamicImports: true, // inlineDynamicImports: true,
plugins: [ plugins: [
@ -68,11 +64,11 @@ export default [
// which external modules to include in the bundle // which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage // https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({ resolve({
extensions: [".ts", ".js", ".json"], extensions: ['.ts', '.js', '.json'],
}), }),
// Resolve source maps to the original source // Resolve source maps to the original source
// sourceMaps(), // sourceMaps(),
], ],
}, },
]; ]

6
extensions/assistant-extension/src/@types/global.d.ts vendored
View File

@ -1,3 +1,3 @@
declare const NODE: string; declare const NODE: string
declare const EXTENSION_NAME: string; declare const EXTENSION_NAME: string
declare const VERSION: string; declare const VERSION: string

157
extensions/assistant-extension/src/index.ts
View File

@ -10,57 +10,56 @@ import {
executeOnMain, executeOnMain,
AssistantExtension, AssistantExtension,
AssistantEvent, AssistantEvent,
} from "@janhq/core"; } from '@janhq/core'
export default class JanAssistantExtension extends AssistantExtension { export default class JanAssistantExtension extends AssistantExtension {
private static readonly _homeDir = "file://assistants"; private static readonly _homeDir = 'file://assistants'
private static readonly _threadDir = "file://threads"; private static readonly _threadDir = 'file://threads'
controller = new AbortController(); controller = new AbortController()
isCancelled = false; isCancelled = false
retrievalThreadId: string | undefined = undefined; retrievalThreadId: string | undefined = undefined
async onLoad() { async onLoad() {
// making the assistant directory // making the assistant directory
const assistantDirExist = await fs.existsSync( const assistantDirExist = await fs.existsSync(
JanAssistantExtension._homeDir JanAssistantExtension._homeDir
); )
if ( if (
localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION || localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION ||
!assistantDirExist !assistantDirExist
) { ) {
if (!assistantDirExist) if (!assistantDirExist) await fs.mkdirSync(JanAssistantExtension._homeDir)
await fs.mkdirSync(JanAssistantExtension._homeDir);
// Write assistant metadata // Write assistant metadata
await this.createJanAssistant(); await this.createJanAssistant()
// Finished migration // Finished migration
localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION); localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION)
// Update the assistant list // Update the assistant list
events.emit(AssistantEvent.OnAssistantsUpdate, {}); events.emit(AssistantEvent.OnAssistantsUpdate, {})
} }
// Events subscription // Events subscription
events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
JanAssistantExtension.handleMessageRequest(data, this) JanAssistantExtension.handleMessageRequest(data, this)
); )
events.on(InferenceEvent.OnInferenceStopped, () => { events.on(InferenceEvent.OnInferenceStopped, () => {
JanAssistantExtension.handleInferenceStopped(this); JanAssistantExtension.handleInferenceStopped(this)
}); })
} }
private static async handleInferenceStopped(instance: JanAssistantExtension) { private static async handleInferenceStopped(instance: JanAssistantExtension) {
instance.isCancelled = true; instance.isCancelled = true
instance.controller?.abort(); instance.controller?.abort()
} }
private static async handleMessageRequest( private static async handleMessageRequest(
data: MessageRequest, data: MessageRequest,
instance: JanAssistantExtension instance: JanAssistantExtension
) { ) {
instance.isCancelled = false; instance.isCancelled = false
instance.controller = new AbortController(); instance.controller = new AbortController()
if ( if (
data.model?.engine !== InferenceEngine.tool_retrieval_enabled || data.model?.engine !== InferenceEngine.tool_retrieval_enabled ||
@ -69,26 +68,26 @@ export default class JanAssistantExtension extends AssistantExtension {
// That could lead to an issue where thread stuck at generating response // That could lead to an issue where thread stuck at generating response
!data.thread?.assistants[0]?.tools !data.thread?.assistants[0]?.tools
) { ) {
return; return
} }
const latestMessage = data.messages[data.messages.length - 1]; const latestMessage = data.messages[data.messages.length - 1]
// 1. Ingest the document if needed // 1. Ingest the document if needed
if ( if (
latestMessage && latestMessage &&
latestMessage.content && latestMessage.content &&
typeof latestMessage.content !== "string" && typeof latestMessage.content !== 'string' &&
latestMessage.content.length > 1 latestMessage.content.length > 1
) { ) {
const docFile = latestMessage.content[1]?.doc_url?.url; const docFile = latestMessage.content[1]?.doc_url?.url
if (docFile) { if (docFile) {
await executeOnMain( await executeOnMain(
NODE, NODE,
"toolRetrievalIngestNewDocument", 'toolRetrievalIngestNewDocument',
docFile, docFile,
data.model?.proxyEngine data.model?.proxyEngine
); )
} }
} else if ( } else if (
// Check whether we need to ingest document or not // Check whether we need to ingest document or not
@ -97,7 +96,7 @@ export default class JanAssistantExtension extends AssistantExtension {
await joinPath([ await joinPath([
JanAssistantExtension._threadDir, JanAssistantExtension._threadDir,
data.threadId, data.threadId,
"memory", 'memory',
]) ])
)) ))
) { ) {
@ -108,61 +107,61 @@ export default class JanAssistantExtension extends AssistantExtension {
...data.model, ...data.model,
engine: data.model.proxyEngine, engine: data.model.proxyEngine,
}, },
}; }
events.emit(MessageEvent.OnMessageSent, output); events.emit(MessageEvent.OnMessageSent, output)
return; return
} }
// 2. Load agent on thread changed // 2. Load agent on thread changed
if (instance.retrievalThreadId !== data.threadId) { if (instance.retrievalThreadId !== data.threadId) {
await executeOnMain(NODE, "toolRetrievalLoadThreadMemory", data.threadId); await executeOnMain(NODE, 'toolRetrievalLoadThreadMemory', data.threadId)
instance.retrievalThreadId = data.threadId; instance.retrievalThreadId = data.threadId
// Update the text splitter // Update the text splitter
await executeOnMain( await executeOnMain(
NODE, NODE,
"toolRetrievalUpdateTextSplitter", 'toolRetrievalUpdateTextSplitter',
data.thread.assistants[0].tools[0]?.settings?.chunk_size ?? 4000, data.thread.assistants[0].tools[0]?.settings?.chunk_size ?? 4000,
data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200 data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200
); )
} }
// 3. Using the retrieval template with the result and query // 3. Using the retrieval template with the result and query
if (latestMessage.content) { if (latestMessage.content) {
const prompt = const prompt =
typeof latestMessage.content === "string" typeof latestMessage.content === 'string'
? latestMessage.content ? latestMessage.content
: latestMessage.content[0].text; : latestMessage.content[0].text
// Retrieve the result // Retrieve the result
const retrievalResult = await executeOnMain( const retrievalResult = await executeOnMain(
NODE, NODE,
"toolRetrievalQueryResult", 'toolRetrievalQueryResult',
prompt prompt
); )
console.debug("toolRetrievalQueryResult", retrievalResult); console.debug('toolRetrievalQueryResult', retrievalResult)
// Update message content // Update message content
if (data.thread?.assistants[0]?.tools && retrievalResult) if (data.thread?.assistants[0]?.tools && retrievalResult)
data.messages[data.messages.length - 1].content = data.messages[data.messages.length - 1].content =
data.thread.assistants[0].tools[0].settings?.retrieval_template data.thread.assistants[0].tools[0].settings?.retrieval_template
?.replace("{CONTEXT}", retrievalResult) ?.replace('{CONTEXT}', retrievalResult)
.replace("{QUESTION}", prompt); .replace('{QUESTION}', prompt)
} }
// Filter out all the messages that are not text // Filter out all the messages that are not text
data.messages = data.messages.map((message) => { data.messages = data.messages.map((message) => {
if ( if (
message.content && message.content &&
typeof message.content !== "string" && typeof message.content !== 'string' &&
(message.content.length ?? 0) > 0 (message.content.length ?? 0) > 0
) { ) {
return { return {
...message, ...message,
content: [message.content[0]], content: [message.content[0]],
}; }
} }
return message; return message
}); })
// 4. Reroute the result to inference engine // 4. Reroute the result to inference engine
const output = { const output = {
@ -171,8 +170,8 @@ export default class JanAssistantExtension extends AssistantExtension {
...data.model, ...data.model,
engine: data.model.proxyEngine, engine: data.model.proxyEngine,
}, },
}; }
events.emit(MessageEvent.OnMessageSent, output); events.emit(MessageEvent.OnMessageSent, output)
} }
/** /**
@ -184,88 +183,88 @@ export default class JanAssistantExtension extends AssistantExtension {
const assistantDir = await joinPath([ const assistantDir = await joinPath([
JanAssistantExtension._homeDir, JanAssistantExtension._homeDir,
assistant.id, assistant.id,
]); ])
if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir); if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir)
// store the assistant metadata json // store the assistant metadata json
const assistantMetadataPath = await joinPath([ const assistantMetadataPath = await joinPath([
assistantDir, assistantDir,
"assistant.json", 'assistant.json',
]); ])
try { try {
await fs.writeFileSync( await fs.writeFileSync(
assistantMetadataPath, assistantMetadataPath,
JSON.stringify(assistant, null, 2) JSON.stringify(assistant, null, 2)
); )
} catch (err) { } catch (err) {
console.error(err); console.error(err)
} }
} }
async getAssistants(): Promise<Assistant[]> { async getAssistants(): Promise<Assistant[]> {
// get all the assistant directories // get all the assistant directories
// get all the assistant metadata json // get all the assistant metadata json
const results: Assistant[] = []; const results: Assistant[] = []
const allFileName: string[] = await fs.readdirSync( const allFileName: string[] = await fs.readdirSync(
JanAssistantExtension._homeDir JanAssistantExtension._homeDir
); )
for (const fileName of allFileName) { for (const fileName of allFileName) {
const filePath = await joinPath([ const filePath = await joinPath([
JanAssistantExtension._homeDir, JanAssistantExtension._homeDir,
fileName, fileName,
]); ])
if (filePath.includes(".DS_Store")) continue; if (filePath.includes('.DS_Store')) continue
const jsonFiles: string[] = (await fs.readdirSync(filePath)).filter( const jsonFiles: string[] = (await fs.readdirSync(filePath)).filter(
(file: string) => file === "assistant.json" (file: string) => file === 'assistant.json'
); )
if (jsonFiles.length !== 1) { if (jsonFiles.length !== 1) {
// has more than one assistant file -> ignore // has more than one assistant file -> ignore
continue; continue
} }
const content = await fs.readFileSync( const content = await fs.readFileSync(
await joinPath([filePath, jsonFiles[0]]), await joinPath([filePath, jsonFiles[0]]),
"utf-8" 'utf-8'
); )
const assistant: Assistant = const assistant: Assistant =
typeof content === "object" ? content : JSON.parse(content); typeof content === 'object' ? content : JSON.parse(content)
results.push(assistant); results.push(assistant)
} }
return results; return results
} }
async deleteAssistant(assistant: Assistant): Promise<void> { async deleteAssistant(assistant: Assistant): Promise<void> {
if (assistant.id === "jan") { if (assistant.id === 'jan') {
return Promise.reject("Cannot delete Jan Assistant"); return Promise.reject('Cannot delete Jan Assistant')
} }
// remove the directory // remove the directory
const assistantDir = await joinPath([ const assistantDir = await joinPath([
JanAssistantExtension._homeDir, JanAssistantExtension._homeDir,
assistant.id, assistant.id,
]); ])
await fs.rmdirSync(assistantDir); await fs.rmdirSync(assistantDir)
return Promise.resolve(); return Promise.resolve()
} }
private async createJanAssistant(): Promise<void> { private async createJanAssistant(): Promise<void> {
const janAssistant: Assistant = { const janAssistant: Assistant = {
avatar: "", avatar: '',
thread_location: undefined, thread_location: undefined,
id: "jan", id: 'jan',
object: "assistant", object: 'assistant',
created_at: Date.now(), created_at: Date.now(),
name: "Jan", name: 'Jan',
description: "A default assistant that can use all downloaded models", description: 'A default assistant that can use all downloaded models',
model: "*", model: '*',
instructions: "", instructions: '',
tools: [ tools: [
{ {
type: "retrieval", type: 'retrieval',
enabled: false, enabled: false,
settings: { settings: {
top_k: 2, top_k: 2,
@ -283,8 +282,8 @@ Helpful Answer:`,
], ],
file_ids: [], file_ids: [],
metadata: undefined, metadata: undefined,
}; }
await this.createAssistant(janAssistant); await this.createAssistant(janAssistant)
} }
} }

16
extensions/assistant-extension/src/node/engine.ts
View File

@ -1,13 +1,13 @@
import fs from "fs"; import fs from 'fs'
import path from "path"; import path from 'path'
import { getJanDataFolderPath } from "@janhq/core/node"; import { getJanDataFolderPath } from '@janhq/core/node'
// Sec: Do not send engine settings over requests // Sec: Do not send engine settings over requests
// Read it manually instead // Read it manually instead
export const readEmbeddingEngine = (engineName: string) => { export const readEmbeddingEngine = (engineName: string) => {
const engineSettings = fs.readFileSync( const engineSettings = fs.readFileSync(
path.join(getJanDataFolderPath(), "engines", `${engineName}.json`), path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`),
"utf-8", 'utf-8'
); )
return JSON.parse(engineSettings); return JSON.parse(engineSettings)
}; }

28
extensions/assistant-extension/src/node/index.ts
View File

@ -1,39 +1,39 @@
import { getJanDataFolderPath, normalizeFilePath } from "@janhq/core/node"; import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node'
import { retrieval } from "./tools/retrieval"; import { retrieval } from './tools/retrieval'
import path from "path"; import path from 'path'
export function toolRetrievalUpdateTextSplitter( export function toolRetrievalUpdateTextSplitter(
chunkSize: number, chunkSize: number,
chunkOverlap: number chunkOverlap: number
) { ) {
retrieval.updateTextSplitter(chunkSize, chunkOverlap); retrieval.updateTextSplitter(chunkSize, chunkOverlap)
} }
export async function toolRetrievalIngestNewDocument( export async function toolRetrievalIngestNewDocument(
file: string, file: string,
engine: string engine: string
) { ) {
const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)); const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file))
const threadPath = path.dirname(filePath.replace("files", "")); const threadPath = path.dirname(filePath.replace('files', ''))
retrieval.updateEmbeddingEngine(engine); retrieval.updateEmbeddingEngine(engine)
return retrieval return retrieval
.ingestAgentKnowledge(filePath, `${threadPath}/memory`) .ingestAgentKnowledge(filePath, `${threadPath}/memory`)
.catch((err) => { .catch((err) => {
console.error(err); console.error(err)
}); })
} }
export async function toolRetrievalLoadThreadMemory(threadId: string) { export async function toolRetrievalLoadThreadMemory(threadId: string) {
return retrieval return retrieval
.loadRetrievalAgent( .loadRetrievalAgent(
path.join(getJanDataFolderPath(), "threads", threadId, "memory") path.join(getJanDataFolderPath(), 'threads', threadId, 'memory')
) )
.catch((err) => { .catch((err) => {
console.error(err); console.error(err)
}); })
} }
export async function toolRetrievalQueryResult(query: string) { export async function toolRetrievalQueryResult(query: string) {
return retrieval.generateResult(query).catch((err) => { return retrieval.generateResult(query).catch((err) => {
console.error(err); console.error(err)
}); })
} }

80
extensions/assistant-extension/src/node/tools/retrieval/index.ts
View File

@ -1,80 +1,80 @@
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
import { formatDocumentsAsString } from "langchain/util/document"; import { formatDocumentsAsString } from 'langchain/util/document'
import { PDFLoader } from "langchain/document_loaders/fs/pdf"; import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
import { HNSWLib } from "langchain/vectorstores/hnswlib"; import { HNSWLib } from 'langchain/vectorstores/hnswlib'
import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { OpenAIEmbeddings } from 'langchain/embeddings/openai'
import { readEmbeddingEngine } from "../../engine"; import { readEmbeddingEngine } from '../../engine'
export class Retrieval { export class Retrieval {
public chunkSize: number = 100; public chunkSize: number = 100
public chunkOverlap?: number = 0; public chunkOverlap?: number = 0
private retriever: any; private retriever: any
private embeddingModel?: OpenAIEmbeddings = undefined; private embeddingModel?: OpenAIEmbeddings = undefined
private textSplitter?: RecursiveCharacterTextSplitter; private textSplitter?: RecursiveCharacterTextSplitter
constructor(chunkSize: number = 4000, chunkOverlap: number = 200) { constructor(chunkSize: number = 4000, chunkOverlap: number = 200) {
this.updateTextSplitter(chunkSize, chunkOverlap); this.updateTextSplitter(chunkSize, chunkOverlap)
} }
public updateTextSplitter(chunkSize: number, chunkOverlap: number): void { public updateTextSplitter(chunkSize: number, chunkOverlap: number): void {
this.chunkSize = chunkSize; this.chunkSize = chunkSize
this.chunkOverlap = chunkOverlap; this.chunkOverlap = chunkOverlap
this.textSplitter = new RecursiveCharacterTextSplitter({ this.textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: chunkSize, chunkSize: chunkSize,
chunkOverlap: chunkOverlap, chunkOverlap: chunkOverlap,
}); })
} }
public updateEmbeddingEngine(engine: string): void { public updateEmbeddingEngine(engine: string): void {
// Engine settings are not compatible with the current embedding model params // Engine settings are not compatible with the current embedding model params
// Switch case manually for now // Switch case manually for now
const settings = readEmbeddingEngine(engine); const settings = readEmbeddingEngine(engine)
if (engine === "nitro") { if (engine === 'nitro') {
this.embeddingModel = new OpenAIEmbeddings( this.embeddingModel = new OpenAIEmbeddings(
{ openAIApiKey: "nitro-embedding" }, { openAIApiKey: 'nitro-embedding' },
// TODO: Raw settings // TODO: Raw settings
{ basePath: "http://127.0.0.1:3928/v1" }, { basePath: 'http://127.0.0.1:3928/v1' }
); )
} else { } else {
// Fallback to OpenAI Settings // Fallback to OpenAI Settings
this.embeddingModel = new OpenAIEmbeddings({ this.embeddingModel = new OpenAIEmbeddings({
openAIApiKey: settings.api_key, openAIApiKey: settings.api_key,
}); })
} }
} }
public ingestAgentKnowledge = async ( public ingestAgentKnowledge = async (
filePath: string, filePath: string,
memoryPath: string, memoryPath: string
): Promise<any> => { ): Promise<any> => {
const loader = new PDFLoader(filePath, { const loader = new PDFLoader(filePath, {
splitPages: true, splitPages: true,
}); })
if (!this.embeddingModel) return Promise.reject(); if (!this.embeddingModel) return Promise.reject()
const doc = await loader.load(); const doc = await loader.load()
const docs = await this.textSplitter!.splitDocuments(doc); const docs = await this.textSplitter!.splitDocuments(doc)
const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel); const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel)
return vectorStore.save(memoryPath); return vectorStore.save(memoryPath)
}; }
public loadRetrievalAgent = async (memoryPath: string): Promise<void> => { public loadRetrievalAgent = async (memoryPath: string): Promise<void> => {
if (!this.embeddingModel) return Promise.reject(); if (!this.embeddingModel) return Promise.reject()
const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel); const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel)
this.retriever = vectorStore.asRetriever(2); this.retriever = vectorStore.asRetriever(2)
return Promise.resolve(); return Promise.resolve()
}; }
public generateResult = async (query: string): Promise<string> => { public generateResult = async (query: string): Promise<string> => {
if (!this.retriever) { if (!this.retriever) {
return Promise.resolve(" "); return Promise.resolve(' ')
} }
const relevantDocs = await this.retriever.getRelevantDocuments(query); const relevantDocs = await this.retriever.getRelevantDocuments(query)
const serializedDoc = formatDocumentsAsString(relevantDocs); const serializedDoc = formatDocumentsAsString(relevantDocs)
return Promise.resolve(serializedDoc); return Promise.resolve(serializedDoc)
}; }
} }
export const retrieval = new Retrieval(); export const retrieval = new Retrieval()

4
extensions/assistant-extension/tsconfig.json
View File

@ -14,7 +14,7 @@
"outDir": "dist", "outDir": "dist",
"importHelpers": true, "importHelpers": true,
"typeRoots": ["node_modules/@types"], "typeRoots": ["node_modules/@types"],
"skipLibCheck": true, "skipLibCheck": true
}, },
"include": ["src"], "include": ["src"]
} }

20
extensions/conversational-extension/webpack.config.js
View File

@ -1,27 +1,27 @@
const path = require("path"); const path = require('path')
const webpack = require("webpack"); const webpack = require('webpack')
module.exports = { module.exports = {
experiments: { outputModule: true }, experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: "production", mode: 'production',
module: { module: {
rules: [ rules: [
{ {
test: /\.tsx?$/, test: /\.tsx?$/,
use: "ts-loader", use: 'ts-loader',
exclude: /node_modules/, exclude: /node_modules/,
}, },
], ],
}, },
output: { output: {
filename: "index.js", // Adjust the output file name as needed filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"), path: path.resolve(__dirname, 'dist'),
library: { type: "module" }, // Specify ESM output format library: { type: 'module' }, // Specify ESM output format
}, },
plugins: [new webpack.DefinePlugin({})], plugins: [new webpack.DefinePlugin({})],
resolve: { resolve: {
extensions: [".ts", ".js"], extensions: ['.ts', '.js'],
fallback: { fallback: {
path: require.resolve('path-browserify'), path: require.resolve('path-browserify'),
}, },
@ -31,4 +31,4 @@ module.exports = {
minimize: false, minimize: false,
}, },
// Add loaders and other configuration as needed for your project // Add loaders and other configuration as needed for your project
}; }

5
extensions/inference-nitro-extension/README.md
View File

@ -64,10 +64,10 @@ There are a few things to keep in mind when writing your plugin code:
In `index.ts`, you will see that the extension function will return a `Promise<any>`. In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript ```typescript
import { core } from "@janhq/core"; import { core } from '@janhq/core'
function onStart(): Promise<any> { function onStart(): Promise<any> {
return core.invokePluginFunc(MODULE_PATH, "run", 0); return core.invokePluginFunc(MODULE_PATH, 'run', 0)
} }
``` ```
@ -75,4 +75,3 @@ There are a few things to keep in mind when writing your plugin code:
[documentation](https://github.com/janhq/jan/blob/main/core/README.md). [documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your plugin! So, what are you waiting for? Go ahead and start customizing your plugin!

38
extensions/inference-nitro-extension/rollup.config.ts
View File

@ -1,34 +1,34 @@
import resolve from "@rollup/plugin-node-resolve"; import resolve from '@rollup/plugin-node-resolve'
import commonjs from "@rollup/plugin-commonjs"; import commonjs from '@rollup/plugin-commonjs'
import sourceMaps from "rollup-plugin-sourcemaps"; import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from "rollup-plugin-typescript2"; import typescript from 'rollup-plugin-typescript2'
import json from "@rollup/plugin-json"; import json from '@rollup/plugin-json'
import replace from "@rollup/plugin-replace"; import replace from '@rollup/plugin-replace'
const packageJson = require("./package.json"); const packageJson = require('./package.json')
const pkg = require("./package.json"); const pkg = require('./package.json')
export default [ export default [
{ {
input: `src/index.ts`, input: `src/index.ts`,
output: [{ file: pkg.main, format: "es", sourcemap: true }], output: [{ file: pkg.main, format: 'es', sourcemap: true }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [], external: [],
watch: { watch: {
include: "src/**", include: 'src/**',
}, },
plugins: [ plugins: [
replace({ replace({
NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
INFERENCE_URL: JSON.stringify( INFERENCE_URL: JSON.stringify(
process.env.INFERENCE_URL || process.env.INFERENCE_URL ||
"http://127.0.0.1:3928/inferences/llamacpp/chat_completion" 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion'
), ),
TROUBLESHOOTING_URL: JSON.stringify( TROUBLESHOOTING_URL: JSON.stringify(
"https://jan.ai/guides/troubleshooting" 'https://jan.ai/guides/troubleshooting'
), ),
JAN_SERVER_INFERENCE_URL: JSON.stringify( JAN_SERVER_INFERENCE_URL: JSON.stringify(
"http://localhost:1337/v1/chat/completions" 'http://localhost:1337/v1/chat/completions'
), ),
}), }),
// Allow json resolution // Allow json resolution
@ -42,7 +42,7 @@ export default [
// which external modules to include in the bundle // which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage // https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({ resolve({
extensions: [".js", ".ts", ".svelte"], extensions: ['.js', '.ts', '.svelte'],
}), }),
// Resolve source maps to the original source // Resolve source maps to the original source
@ -52,12 +52,12 @@ export default [
{ {
input: `src/node/index.ts`, input: `src/node/index.ts`,
output: [ output: [
{ file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true }, { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
], ],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: ["@janhq/core/node"], external: ['@janhq/core/node'],
watch: { watch: {
include: "src/node/**", include: 'src/node/**',
}, },
plugins: [ plugins: [
// Allow json resolution // Allow json resolution
@ -70,11 +70,11 @@ export default [
// which external modules to include in the bundle // which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage // https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({ resolve({
extensions: [".ts", ".js", ".json"], extensions: ['.ts', '.js', '.json'],
}), }),
// Resolve source maps to the original source // Resolve source maps to the original source
sourceMaps(), sourceMaps(),
], ],
}, },
]; ]

12
extensions/inference-nitro-extension/src/@types/global.d.ts vendored
View File

@ -1,13 +1,13 @@
declare const NODE: string; declare const NODE: string
declare const INFERENCE_URL: string; declare const INFERENCE_URL: string
declare const TROUBLESHOOTING_URL: string; declare const TROUBLESHOOTING_URL: string
declare const JAN_SERVER_INFERENCE_URL: string; declare const JAN_SERVER_INFERENCE_URL: string
/** /**
* The response from the initModel function. * The response from the initModel function.
* @property error - An error message if the model fails to load. * @property error - An error message if the model fails to load.
*/ */
interface ModelOperationResponse { interface ModelOperationResponse {
error?: any; error?: any
modelFile?: string; modelFile?: string
} }

56
extensions/inference-nitro-extension/src/helpers/sse.ts
View File

@ -1,5 +1,5 @@
import { Model } from "@janhq/core"; import { Model } from '@janhq/core'
import { Observable } from "rxjs"; import { Observable } from 'rxjs'
/** /**
* Sends a request to the inference server to generate a response based on the recent messages. * Sends a request to the inference server to generate a response based on the recent messages.
* @param recentMessages - An array of recent messages to use as context for the inference. * @param recentMessages - An array of recent messages to use as context for the inference.
@ -17,50 +17,50 @@ export function requestInference(
model: model.id, model: model.id,
stream: true, stream: true,
...model.parameters, ...model.parameters,
}); })
fetch(inferenceUrl, { fetch(inferenceUrl, {
method: "POST", method: 'POST',
headers: { headers: {
"Content-Type": "application/json", 'Content-Type': 'application/json',
"Access-Control-Allow-Origin": "*", 'Access-Control-Allow-Origin': '*',
Accept: model.parameters.stream 'Accept': model.parameters.stream
? "text/event-stream" ? 'text/event-stream'
: "application/json", : 'application/json',
}, },
body: requestBody, body: requestBody,
signal: controller?.signal, signal: controller?.signal,
}) })
.then(async (response) => { .then(async (response) => {
if (model.parameters.stream === false) { if (model.parameters.stream === false) {
const data = await response.json(); const data = await response.json()
subscriber.next(data.choices[0]?.message?.content ?? ""); subscriber.next(data.choices[0]?.message?.content ?? '')
} else { } else {
const stream = response.body; const stream = response.body
const decoder = new TextDecoder("utf-8"); const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader(); const reader = stream?.getReader()
let content = ""; let content = ''
while (true && reader) { while (true && reader) {
const { done, value } = await reader.read(); const { done, value } = await reader.read()
if (done) { if (done) {
break; break
} }
const text = decoder.decode(value); const text = decoder.decode(value)
const lines = text.trim().split("\n"); const lines = text.trim().split('\n')
for (const line of lines) { for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { if (line.startsWith('data: ') && !line.includes('data: [DONE]')) {
const data = JSON.parse(line.replace("data: ", "")); const data = JSON.parse(line.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ""; content += data.choices[0]?.delta?.content ?? ''
if (content.startsWith("assistant: ")) { if (content.startsWith('assistant: ')) {
content = content.replace("assistant: ", ""); content = content.replace('assistant: ', '')
} }
subscriber.next(content); subscriber.next(content)
} }
} }
} }
} }
subscriber.complete(); subscriber.complete()
}) })
.catch((err) => subscriber.error(err)); .catch((err) => subscriber.error(err))
}); })
} }

175
extensions/inference-nitro-extension/src/index.ts
View File

@ -27,9 +27,9 @@ import {
InferenceEvent, InferenceEvent,
ModelSettingParams, ModelSettingParams,
getJanDataFolderPath, getJanDataFolderPath,
} from "@janhq/core"; } from '@janhq/core'
import { requestInference } from "./helpers/sse"; import { requestInference } from './helpers/sse'
import { ulid } from "ulid"; import { ulid } from 'ulid'
/** /**
* A class that implements the InferenceExtension interface from the @janhq/core package. * A class that implements the InferenceExtension interface from the @janhq/core package.
@ -37,16 +37,16 @@ import { ulid } from "ulid";
* It also subscribes to events emitted by the @janhq/core package and handles new message requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/ */
export default class JanInferenceNitroExtension extends InferenceExtension { export default class JanInferenceNitroExtension extends InferenceExtension {
private static readonly _homeDir = "file://engines"; private static readonly _homeDir = 'file://engines'
private static readonly _settingsDir = "file://settings"; private static readonly _settingsDir = 'file://settings'
private static readonly _engineMetadataFileName = "nitro.json"; private static readonly _engineMetadataFileName = 'nitro.json'
/** /**
* Checking the health for Nitro's process each 5 secs. * Checking the health for Nitro's process each 5 secs.
*/ */
private static readonly _intervalHealthCheck = 5 * 1000; private static readonly _intervalHealthCheck = 5 * 1000
private _currentModel: Model | undefined; private _currentModel: Model | undefined
private _engineSettings: ModelSettingParams = { private _engineSettings: ModelSettingParams = {
ctx_len: 2048, ctx_len: 2048,
@ -54,23 +54,22 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
cpu_threads: 1, cpu_threads: 1,
cont_batching: false, cont_batching: false,
embedding: true, embedding: true,
}; }
controller = new AbortController(); controller = new AbortController()
isCancelled = false; isCancelled = false
/** /**
* The interval id for the health check. Used to stop the health check. * The interval id for the health check. Used to stop the health check.
*/ */
private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = undefined
undefined;
/** /**
* Tracking the current state of nitro process. * Tracking the current state of nitro process.
*/ */
private nitroProcessInfo: any = undefined; private nitroProcessInfo: any = undefined
private inferenceUrl = ""; private inferenceUrl = ''
/** /**
* Subscribes to events emitted by the @janhq/core package. * Subscribes to events emitted by the @janhq/core package.
@ -78,44 +77,40 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
async onLoad() { async onLoad() {
if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) {
try { try {
await fs.mkdirSync(JanInferenceNitroExtension._homeDir); await fs.mkdirSync(JanInferenceNitroExtension._homeDir)
} catch (e) { } catch (e) {
console.debug(e); console.debug(e)
} }
} }
// init inference url // init inference url
// @ts-ignore // @ts-ignore
const electronApi = window?.electronAPI; const electronApi = window?.electronAPI
this.inferenceUrl = INFERENCE_URL; this.inferenceUrl = INFERENCE_URL
if (!electronApi) { if (!electronApi) {
this.inferenceUrl = JAN_SERVER_INFERENCE_URL; this.inferenceUrl = JAN_SERVER_INFERENCE_URL
} }
console.debug("Inference url: ", this.inferenceUrl); console.debug('Inference url: ', this.inferenceUrl)
if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir)))
await fs.mkdirSync(JanInferenceNitroExtension._settingsDir); await fs.mkdirSync(JanInferenceNitroExtension._settingsDir)
this.writeDefaultEngineSettings(); this.writeDefaultEngineSettings()
// Events subscription // Events subscription
events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.onMessageRequest(data) this.onMessageRequest(data)
); )
events.on(ModelEvent.OnModelInit, (model: Model) => events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model))
this.onModelInit(model)
);
events.on(ModelEvent.OnModelStop, (model: Model) => events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model))
this.onModelStop(model)
);
events.on(InferenceEvent.OnInferenceStopped, () => events.on(InferenceEvent.OnInferenceStopped, () =>
this.onInferenceStopped() this.onInferenceStopped()
); )
// Attempt to fetch nvidia info // Attempt to fetch nvidia info
await executeOnMain(NODE, "updateNvidiaInfo", {}); await executeOnMain(NODE, 'updateNvidiaInfo', {})
} }
/** /**
@ -128,59 +123,59 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
const engineFile = await joinPath([ const engineFile = await joinPath([
JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._homeDir,
JanInferenceNitroExtension._engineMetadataFileName, JanInferenceNitroExtension._engineMetadataFileName,
]); ])
if (await fs.existsSync(engineFile)) { if (await fs.existsSync(engineFile)) {
const engine = await fs.readFileSync(engineFile, "utf-8"); const engine = await fs.readFileSync(engineFile, 'utf-8')
this._engineSettings = this._engineSettings =
typeof engine === "object" ? engine : JSON.parse(engine); typeof engine === 'object' ? engine : JSON.parse(engine)
} else { } else {
await fs.writeFileSync( await fs.writeFileSync(
engineFile, engineFile,
JSON.stringify(this._engineSettings, null, 2) JSON.stringify(this._engineSettings, null, 2)
); )
} }
} catch (err) { } catch (err) {
console.error(err); console.error(err)
} }
} }
private async onModelInit(model: Model) { private async onModelInit(model: Model) {
if (model.engine !== InferenceEngine.nitro) return; if (model.engine !== InferenceEngine.nitro) return
const modelFolder = await joinPath([ const modelFolder = await joinPath([
await getJanDataFolderPath(), await getJanDataFolderPath(),
"models", 'models',
model.id, model.id,
]); ])
this._currentModel = model; this._currentModel = model
const nitroInitResult = await executeOnMain(NODE, "runModel", { const nitroInitResult = await executeOnMain(NODE, 'runModel', {
modelFolder, modelFolder,
model, model,
}); })
if (nitroInitResult?.error) { if (nitroInitResult?.error) {
events.emit(ModelEvent.OnModelFail, model); events.emit(ModelEvent.OnModelFail, model)
return; return
} }
events.emit(ModelEvent.OnModelReady, model); events.emit(ModelEvent.OnModelReady, model)
this.getNitroProcesHealthIntervalId = setInterval( this.getNitroProcesHealthIntervalId = setInterval(
() => this.periodicallyGetNitroHealth(), () => this.periodicallyGetNitroHealth(),
JanInferenceNitroExtension._intervalHealthCheck JanInferenceNitroExtension._intervalHealthCheck
); )
} }
private async onModelStop(model: Model) { private async onModelStop(model: Model) {
if (model.engine !== "nitro") return; if (model.engine !== 'nitro') return
await executeOnMain(NODE, "stopModel"); await executeOnMain(NODE, 'stopModel')
events.emit(ModelEvent.OnModelStopped, {}); events.emit(ModelEvent.OnModelStopped, {})
// stop the periocally health check // stop the periocally health check
if (this.getNitroProcesHealthIntervalId) { if (this.getNitroProcesHealthIntervalId) {
clearInterval(this.getNitroProcesHealthIntervalId); clearInterval(this.getNitroProcesHealthIntervalId)
this.getNitroProcesHealthIntervalId = undefined; this.getNitroProcesHealthIntervalId = undefined
} }
} }
@ -188,19 +183,19 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
* Periodically check for nitro process's health. * Periodically check for nitro process's health.
*/ */
private async periodicallyGetNitroHealth(): Promise<void> { private async periodicallyGetNitroHealth(): Promise<void> {
const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo"); const health = await executeOnMain(NODE, 'getCurrentNitroProcessInfo')
const isRunning = this.nitroProcessInfo?.isRunning ?? false; const isRunning = this.nitroProcessInfo?.isRunning ?? false
if (isRunning && health.isRunning === false) { if (isRunning && health.isRunning === false) {
console.debug("Nitro process is stopped"); console.debug('Nitro process is stopped')
events.emit(ModelEvent.OnModelStopped, {}); events.emit(ModelEvent.OnModelStopped, {})
} }
this.nitroProcessInfo = health; this.nitroProcessInfo = health
} }
private async onInferenceStopped() { private async onInferenceStopped() {
this.isCancelled = true; this.isCancelled = true
this.controller?.abort(); this.controller?.abort()
} }
/** /**
@ -209,20 +204,20 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
* @returns {Promise<any>} A promise that resolves with the inference response. * @returns {Promise<any>} A promise that resolves with the inference response.
*/ */
async inference(data: MessageRequest): Promise<ThreadMessage> { async inference(data: MessageRequest): Promise<ThreadMessage> {
const timestamp = Date.now(); const timestamp = Date.now()
const message: ThreadMessage = { const message: ThreadMessage = {
thread_id: data.threadId, thread_id: data.threadId,
created: timestamp, created: timestamp,
updated: timestamp, updated: timestamp,
status: MessageStatus.Ready, status: MessageStatus.Ready,
id: "", id: '',
role: ChatCompletionRole.Assistant, role: ChatCompletionRole.Assistant,
object: "thread.message", object: 'thread.message',
content: [], content: [],
}; }
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
if (!this._currentModel) return Promise.reject("No model loaded"); if (!this._currentModel) return Promise.reject('No model loaded')
requestInference( requestInference(
this.inferenceUrl, this.inferenceUrl,
@ -231,13 +226,13 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
).subscribe({ ).subscribe({
next: (_content: any) => {}, next: (_content: any) => {},
complete: async () => { complete: async () => {
resolve(message); resolve(message)
}, },
error: async (err: any) => { error: async (err: any) => {
reject(err); reject(err)
}, },
}); })
}); })
} }
/** /**
@ -248,10 +243,10 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
*/ */
private async onMessageRequest(data: MessageRequest) { private async onMessageRequest(data: MessageRequest) {
if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) { if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) {
return; return
} }
const timestamp = Date.now(); const timestamp = Date.now()
const message: ThreadMessage = { const message: ThreadMessage = {
id: ulid(), id: ulid(),
thread_id: data.threadId, thread_id: data.threadId,
@ -262,21 +257,21 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
status: MessageStatus.Pending, status: MessageStatus.Pending,
created: timestamp, created: timestamp,
updated: timestamp, updated: timestamp,
object: "thread.message", object: 'thread.message',
};
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message);
} }
this.isCancelled = false; if (data.type !== MessageRequestType.Summary) {
this.controller = new AbortController(); events.emit(MessageEvent.OnMessageResponse, message)
}
this.isCancelled = false
this.controller = new AbortController()
// @ts-ignore // @ts-ignore
const model: Model = { const model: Model = {
...(this._currentModel || {}), ...(this._currentModel || {}),
...(data.model || {}), ...(data.model || {}),
}; }
requestInference( requestInference(
this.inferenceUrl, this.inferenceUrl,
data.messages ?? [], data.messages ?? [],
@ -290,26 +285,26 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
value: content.trim(), value: content.trim(),
annotations: [], annotations: [],
}, },
}; }
message.content = [messageContent]; message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
complete: async () => { complete: async () => {
message.status = message.content.length message.status = message.content.length
? MessageStatus.Ready ? MessageStatus.Ready
: MessageStatus.Error; : MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
error: async (err: any) => { error: async (err: any) => {
if (this.isCancelled || message.content.length) { if (this.isCancelled || message.content.length) {
message.status = MessageStatus.Stopped; message.status = MessageStatus.Stopped
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
return; return
} }
message.status = MessageStatus.Error; message.status = MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
log(`[APP]::Error: ${err.message}`); log(`[APP]::Error: ${err.message}`)
}, },
}); })
} }
} }

60
extensions/inference-nitro-extension/src/node/execute.ts
View File

@ -1,65 +1,65 @@
import { readFileSync } from "fs"; import { readFileSync } from 'fs'
import * as path from "path"; import * as path from 'path'
import { NVIDIA_INFO_FILE } from "./nvidia"; import { NVIDIA_INFO_FILE } from './nvidia'
export interface NitroExecutableOptions { export interface NitroExecutableOptions {
executablePath: string; executablePath: string
cudaVisibleDevices: string; cudaVisibleDevices: string
} }
/** /**
* Find which executable file to run based on the current platform. * Find which executable file to run based on the current platform.
* @returns The name of the executable file to run. * @returns The name of the executable file to run.
*/ */
export const executableNitroFile = (): NitroExecutableOptions => { export const executableNitroFile = (): NitroExecutableOptions => {
let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default
let cudaVisibleDevices = ""; let cudaVisibleDevices = ''
let binaryName = "nitro"; let binaryName = 'nitro'
/** /**
* The binary folder is different for each platform. * The binary folder is different for each platform.
*/ */
if (process.platform === "win32") { if (process.platform === 'win32') {
/** /**
* For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0
*/ */
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (nvidiaInfo["run_mode"] === "cpu") { if (nvidiaInfo['run_mode'] === 'cpu') {
binaryFolder = path.join(binaryFolder, "win-cpu"); binaryFolder = path.join(binaryFolder, 'win-cpu')
} else { } else {
if (nvidiaInfo["cuda"].version === "11") { if (nvidiaInfo['cuda'].version === '11') {
binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); binaryFolder = path.join(binaryFolder, 'win-cuda-11-7')
} else { } else {
binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); binaryFolder = path.join(binaryFolder, 'win-cuda-12-0')
} }
cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(","); cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
} }
binaryName = "nitro.exe"; binaryName = 'nitro.exe'
} else if (process.platform === "darwin") { } else if (process.platform === 'darwin') {
/** /**
* For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL)
*/ */
if (process.arch === "arm64") { if (process.arch === 'arm64') {
binaryFolder = path.join(binaryFolder, "mac-arm64"); binaryFolder = path.join(binaryFolder, 'mac-arm64')
} else { } else {
binaryFolder = path.join(binaryFolder, "mac-x64"); binaryFolder = path.join(binaryFolder, 'mac-x64')
} }
} else { } else {
/** /**
* For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0
*/ */
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (nvidiaInfo["run_mode"] === "cpu") { if (nvidiaInfo['run_mode'] === 'cpu') {
binaryFolder = path.join(binaryFolder, "linux-cpu"); binaryFolder = path.join(binaryFolder, 'linux-cpu')
} else { } else {
if (nvidiaInfo["cuda"].version === "11") { if (nvidiaInfo['cuda'].version === '11') {
binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7')
} else { } else {
binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0')
} }
cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(","); cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
} }
} }
return { return {
executablePath: path.join(binaryFolder, binaryName), executablePath: path.join(binaryFolder, binaryName),
cudaVisibleDevices, cudaVisibleDevices,
}; }
}; }

160
extensions/inference-nitro-extension/src/node/nvidia.ts
View File

@ -1,47 +1,47 @@
import { writeFileSync, existsSync, readFileSync } from "fs"; import { writeFileSync, existsSync, readFileSync } from 'fs'
import { exec } from "child_process"; import { exec } from 'child_process'
import path from "path"; import path from 'path'
import { getJanDataFolderPath } from "@janhq/core/node"; import { getJanDataFolderPath } from '@janhq/core/node'
/** /**
* Default GPU settings * Default GPU settings
**/ **/
const DEFALT_SETTINGS = { const DEFALT_SETTINGS = {
notify: true, notify: true,
run_mode: "cpu", run_mode: 'cpu',
nvidia_driver: { nvidia_driver: {
exist: false, exist: false,
version: "", version: '',
}, },
cuda: { cuda: {
exist: false, exist: false,
version: "", version: '',
}, },
gpus: [], gpus: [],
gpu_highest_vram: "", gpu_highest_vram: '',
gpus_in_use: [], gpus_in_use: [],
is_initial: true, is_initial: true,
}; }
/** /**
* Path to the settings file * Path to the settings file
**/ **/
export const NVIDIA_INFO_FILE = path.join( export const NVIDIA_INFO_FILE = path.join(
getJanDataFolderPath(), getJanDataFolderPath(),
"settings", 'settings',
"settings.json" 'settings.json'
); )
/** /**
* Current nitro process * Current nitro process
*/ */
let nitroProcessInfo: NitroProcessInfo | undefined = undefined; let nitroProcessInfo: NitroProcessInfo | undefined = undefined
/** /**
* Nitro process info * Nitro process info
*/ */
export interface NitroProcessInfo { export interface NitroProcessInfo {
isRunning: boolean; isRunning: boolean
} }
/** /**
@ -49,16 +49,16 @@ export interface NitroProcessInfo {
* Will be called when the extension is loaded to turn on GPU acceleration if supported * Will be called when the extension is loaded to turn on GPU acceleration if supported
*/ */
export async function updateNvidiaInfo() { export async function updateNvidiaInfo() {
if (process.platform !== "darwin") { if (process.platform !== 'darwin') {
let data; let data
try { try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
} catch (error) { } catch (error) {
data = DEFALT_SETTINGS; data = DEFALT_SETTINGS
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
} }
updateNvidiaDriverInfo(); updateNvidiaDriverInfo()
updateGpuInfo(); updateGpuInfo()
} }
} }
@ -68,31 +68,31 @@ export async function updateNvidiaInfo() {
export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => {
nitroProcessInfo = { nitroProcessInfo = {
isRunning: subprocess != null, isRunning: subprocess != null,
}; }
return nitroProcessInfo; return nitroProcessInfo
}; }
/** /**
* Validate nvidia and cuda for linux and windows * Validate nvidia and cuda for linux and windows
*/ */
export async function updateNvidiaDriverInfo(): Promise<void> { export async function updateNvidiaDriverInfo(): Promise<void> {
exec( exec(
"nvidia-smi --query-gpu=driver_version --format=csv,noheader", 'nvidia-smi --query-gpu=driver_version --format=csv,noheader',
(error, stdout) => { (error, stdout) => {
let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (!error) { if (!error) {
const firstLine = stdout.split("\n")[0].trim(); const firstLine = stdout.split('\n')[0].trim()
data["nvidia_driver"].exist = true; data['nvidia_driver'].exist = true
data["nvidia_driver"].version = firstLine; data['nvidia_driver'].version = firstLine
} else { } else {
data["nvidia_driver"].exist = false; data['nvidia_driver'].exist = false
} }
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
Promise.resolve(); Promise.resolve()
} }
); )
} }
/** /**
@ -102,54 +102,56 @@ export function checkFileExistenceInPaths(
file: string, file: string,
paths: string[] paths: string[]
): boolean { ): boolean {
return paths.some((p) => existsSync(path.join(p, file))); return paths.some((p) => existsSync(path.join(p, file)))
} }
/** /**
* Validate cuda for linux and windows * Validate cuda for linux and windows
*/ */
export function updateCudaExistence(data: Record<string, any> = DEFALT_SETTINGS): Record<string, any> { export function updateCudaExistence(
let filesCuda12: string[]; data: Record<string, any> = DEFALT_SETTINGS
let filesCuda11: string[]; ): Record<string, any> {
let paths: string[]; let filesCuda12: string[]
let cudaVersion: string = ""; let filesCuda11: string[]
let paths: string[]
let cudaVersion: string = ''
if (process.platform === "win32") { if (process.platform === 'win32') {
filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll']
filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll']
paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []
} else { } else {
filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12']
filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11']
paths = process.env.LD_LIBRARY_PATH paths = process.env.LD_LIBRARY_PATH
? process.env.LD_LIBRARY_PATH.split(path.delimiter) ? process.env.LD_LIBRARY_PATH.split(path.delimiter)
: []; : []
paths.push("/usr/lib/x86_64-linux-gnu/"); paths.push('/usr/lib/x86_64-linux-gnu/')
} }
let cudaExists = filesCuda12.every( let cudaExists = filesCuda12.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths) (file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
); )
if (!cudaExists) { if (!cudaExists) {
cudaExists = filesCuda11.every( cudaExists = filesCuda11.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths) (file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
); )
if (cudaExists) { if (cudaExists) {
cudaVersion = "11"; cudaVersion = '11'
} }
} else { } else {
cudaVersion = "12"; cudaVersion = '12'
} }
data["cuda"].exist = cudaExists; data['cuda'].exist = cudaExists
data["cuda"].version = cudaVersion; data['cuda'].version = cudaVersion
console.log(data["is_initial"], data["gpus_in_use"]); console.log(data['is_initial'], data['gpus_in_use'])
if (cudaExists && data["is_initial"] && data["gpus_in_use"].length > 0) { if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) {
data.run_mode = "gpu"; data.run_mode = 'gpu'
} }
data.is_initial = false; data.is_initial = false
return data; return data
} }
/** /**
@ -157,41 +159,41 @@ export function updateCudaExistence(data: Record<string, any> = DEFALT_SETTINGS)
*/ */
export async function updateGpuInfo(): Promise<void> { export async function updateGpuInfo(): Promise<void> {
exec( exec(
"nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits", 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits',
(error, stdout) => { (error, stdout) => {
let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (!error) { if (!error) {
// Get GPU info and gpu has higher memory first // Get GPU info and gpu has higher memory first
let highestVram = 0; let highestVram = 0
let highestVramId = "0"; let highestVramId = '0'
let gpus = stdout let gpus = stdout
.trim() .trim()
.split("\n") .split('\n')
.map((line) => { .map((line) => {
let [id, vram, name] = line.split(", "); let [id, vram, name] = line.split(', ')
vram = vram.replace(/\r/g, ""); vram = vram.replace(/\r/g, '')
if (parseFloat(vram) > highestVram) { if (parseFloat(vram) > highestVram) {
highestVram = parseFloat(vram); highestVram = parseFloat(vram)
highestVramId = id; highestVramId = id
} }
return { id, vram, name }; return { id, vram, name }
}); })
data.gpus = gpus; data.gpus = gpus
data.gpu_highest_vram = highestVramId; data.gpu_highest_vram = highestVramId
} else { } else {
data.gpus = []; data.gpus = []
data.gpu_highest_vram = ""; data.gpu_highest_vram = ''
} }
if (!data["gpus_in_use"] || data["gpus_in_use"].length === 0) { if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) {
data.gpus_in_use = [data["gpu_highest_vram"]]; data.gpus_in_use = [data['gpu_highest_vram']]
} }
data = updateCudaExistence(data); data = updateCudaExistence(data)
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
Promise.resolve(); Promise.resolve()
} }
); )
} }

38
extensions/inference-openai-extension/src/@types/global.d.ts vendored
View File

@ -1,26 +1,26 @@
declare const MODULE: string; declare const MODULE: string
declare const OPENAI_DOMAIN: string; declare const OPENAI_DOMAIN: string
declare interface EngineSettings { declare interface EngineSettings {
full_url?: string; full_url?: string
api_key?: string; api_key?: string
} }
enum OpenAIChatCompletionModelName { enum OpenAIChatCompletionModelName {
"gpt-3.5-turbo-instruct" = "gpt-3.5-turbo-instruct", 'gpt-3.5-turbo-instruct' = 'gpt-3.5-turbo-instruct',
"gpt-3.5-turbo-instruct-0914" = "gpt-3.5-turbo-instruct-0914", 'gpt-3.5-turbo-instruct-0914' = 'gpt-3.5-turbo-instruct-0914',
"gpt-4-1106-preview" = "gpt-4-1106-preview", 'gpt-4-1106-preview' = 'gpt-4-1106-preview',
"gpt-3.5-turbo-0613" = "gpt-3.5-turbo-0613", 'gpt-3.5-turbo-0613' = 'gpt-3.5-turbo-0613',
"gpt-3.5-turbo-0301" = "gpt-3.5-turbo-0301", 'gpt-3.5-turbo-0301' = 'gpt-3.5-turbo-0301',
"gpt-3.5-turbo" = "gpt-3.5-turbo", 'gpt-3.5-turbo' = 'gpt-3.5-turbo',
"gpt-3.5-turbo-16k-0613" = "gpt-3.5-turbo-16k-0613", 'gpt-3.5-turbo-16k-0613' = 'gpt-3.5-turbo-16k-0613',
"gpt-3.5-turbo-1106" = "gpt-3.5-turbo-1106", 'gpt-3.5-turbo-1106' = 'gpt-3.5-turbo-1106',
"gpt-4-vision-preview" = "gpt-4-vision-preview", 'gpt-4-vision-preview' = 'gpt-4-vision-preview',
"gpt-4" = "gpt-4", 'gpt-4' = 'gpt-4',
"gpt-4-0314" = "gpt-4-0314", 'gpt-4-0314' = 'gpt-4-0314',
"gpt-4-0613" = "gpt-4-0613", 'gpt-4-0613' = 'gpt-4-0613',
} }
declare type OpenAIModel = Omit<Model, "id"> & { declare type OpenAIModel = Omit<Model, 'id'> & {
id: OpenAIChatCompletionModelName; id: OpenAIChatCompletionModelName
}; }

70
extensions/inference-openai-extension/src/helpers/sse.ts
View File

@ -1,4 +1,4 @@
import { Observable } from "rxjs"; import { Observable } from 'rxjs'
/** /**
* Sends a request to the inference server to generate a response based on the recent messages. * Sends a request to the inference server to generate a response based on the recent messages.
@ -14,26 +14,26 @@ export function requestInference(
controller?: AbortController controller?: AbortController
): Observable<string> { ): Observable<string> {
return new Observable((subscriber) => { return new Observable((subscriber) => {
let model_id: string = model.id; let model_id: string = model.id
if (engine.full_url.includes(OPENAI_DOMAIN)) { if (engine.full_url.includes(OPENAI_DOMAIN)) {
model_id = engine.full_url.split("/")[5]; model_id = engine.full_url.split('/')[5]
} }
const requestBody = JSON.stringify({ const requestBody = JSON.stringify({
messages: recentMessages, messages: recentMessages,
stream: true, stream: true,
model: model_id, model: model_id,
...model.parameters, ...model.parameters,
}); })
fetch(`${engine.full_url}`, { fetch(`${engine.full_url}`, {
method: "POST", method: 'POST',
headers: { headers: {
"Content-Type": "application/json", 'Content-Type': 'application/json',
Accept: model.parameters.stream 'Accept': model.parameters.stream
? "text/event-stream" ? 'text/event-stream'
: "application/json", : 'application/json',
"Access-Control-Allow-Origin": "*", 'Access-Control-Allow-Origin': '*',
Authorization: `Bearer ${engine.api_key}`, 'Authorization': `Bearer ${engine.api_key}`,
"api-key": `${engine.api_key}`, 'api-key': `${engine.api_key}`,
}, },
body: requestBody, body: requestBody,
signal: controller?.signal, signal: controller?.signal,
@ -41,41 +41,41 @@ export function requestInference(
.then(async (response) => { .then(async (response) => {
if (!response.ok) { if (!response.ok) {
subscriber.next( subscriber.next(
(await response.json()).error?.message ?? "Error occurred." (await response.json()).error?.message ?? 'Error occurred.'
); )
subscriber.complete(); subscriber.complete()
return; return
} }
if (model.parameters.stream === false) { if (model.parameters.stream === false) {
const data = await response.json(); const data = await response.json()
subscriber.next(data.choices[0]?.message?.content ?? ""); subscriber.next(data.choices[0]?.message?.content ?? '')
} else { } else {
const stream = response.body; const stream = response.body
const decoder = new TextDecoder("utf-8"); const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader(); const reader = stream?.getReader()
let content = ""; let content = ''
while (true && reader) { while (true && reader) {
const { done, value } = await reader.read(); const { done, value } = await reader.read()
if (done) { if (done) {
break; break
} }
const text = decoder.decode(value); const text = decoder.decode(value)
const lines = text.trim().split("\n"); const lines = text.trim().split('\n')
for (const line of lines) { for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { if (line.startsWith('data: ') && !line.includes('data: [DONE]')) {
const data = JSON.parse(line.replace("data: ", "")); const data = JSON.parse(line.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ""; content += data.choices[0]?.delta?.content ?? ''
if (content.startsWith("assistant: ")) { if (content.startsWith('assistant: ')) {
content = content.replace("assistant: ", ""); content = content.replace('assistant: ', '')
} }
subscriber.next(content); subscriber.next(content)
} }
} }
} }
} }
subscriber.complete(); subscriber.complete()
}) })
.catch((err) => subscriber.error(err)); .catch((err) => subscriber.error(err))
}); })
} }

138
extensions/inference-openai-extension/src/index.ts
View File

@ -23,10 +23,10 @@ import {
InferenceEvent, InferenceEvent,
AppConfigurationEventName, AppConfigurationEventName,
joinPath, joinPath,
} from "@janhq/core"; } from '@janhq/core'
import { requestInference } from "./helpers/sse"; import { requestInference } from './helpers/sse'
import { ulid } from "ulid"; import { ulid } from 'ulid'
import { join } from "path"; import { join } from 'path'
/** /**
* A class that implements the InferenceExtension interface from the @janhq/core package. * A class that implements the InferenceExtension interface from the @janhq/core package.
@ -34,18 +34,18 @@ import { join } from "path";
* It also subscribes to events emitted by the @janhq/core package and handles new message requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/ */
export default class JanInferenceOpenAIExtension extends BaseExtension { export default class JanInferenceOpenAIExtension extends BaseExtension {
private static readonly _engineDir = "file://engines"; private static readonly _engineDir = 'file://engines'
private static readonly _engineMetadataFileName = "openai.json"; private static readonly _engineMetadataFileName = 'openai.json'
private static _currentModel: OpenAIModel; private static _currentModel: OpenAIModel
private static _engineSettings: EngineSettings = { private static _engineSettings: EngineSettings = {
full_url: "https://api.openai.com/v1/chat/completions", full_url: 'https://api.openai.com/v1/chat/completions',
api_key: "sk-<your key here>", api_key: 'sk-<your key here>',
}; }
controller = new AbortController(); controller = new AbortController()
isCancelled = false; isCancelled = false
/** /**
* Subscribes to events emitted by the @janhq/core package. * Subscribes to events emitted by the @janhq/core package.
@ -54,40 +54,40 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) {
await fs await fs
.mkdirSync(JanInferenceOpenAIExtension._engineDir) .mkdirSync(JanInferenceOpenAIExtension._engineDir)
.catch((err) => console.debug(err)); .catch((err) => console.debug(err))
} }
JanInferenceOpenAIExtension.writeDefaultEngineSettings(); JanInferenceOpenAIExtension.writeDefaultEngineSettings()
// Events subscription // Events subscription
events.on(MessageEvent.OnMessageSent, (data) => events.on(MessageEvent.OnMessageSent, (data) =>
JanInferenceOpenAIExtension.handleMessageRequest(data, this), JanInferenceOpenAIExtension.handleMessageRequest(data, this)
); )
events.on(ModelEvent.OnModelInit, (model: OpenAIModel) => { events.on(ModelEvent.OnModelInit, (model: OpenAIModel) => {
JanInferenceOpenAIExtension.handleModelInit(model); JanInferenceOpenAIExtension.handleModelInit(model)
}); })
events.on(ModelEvent.OnModelStop, (model: OpenAIModel) => { events.on(ModelEvent.OnModelStop, (model: OpenAIModel) => {
JanInferenceOpenAIExtension.handleModelStop(model); JanInferenceOpenAIExtension.handleModelStop(model)
}); })
events.on(InferenceEvent.OnInferenceStopped, () => { events.on(InferenceEvent.OnInferenceStopped, () => {
JanInferenceOpenAIExtension.handleInferenceStopped(this); JanInferenceOpenAIExtension.handleInferenceStopped(this)
}); })
const settingsFilePath = await joinPath([ const settingsFilePath = await joinPath([
JanInferenceOpenAIExtension._engineDir, JanInferenceOpenAIExtension._engineDir,
JanInferenceOpenAIExtension._engineMetadataFileName, JanInferenceOpenAIExtension._engineMetadataFileName,
]); ])
events.on( events.on(
AppConfigurationEventName.OnConfigurationUpdate, AppConfigurationEventName.OnConfigurationUpdate,
(settingsKey: string) => { (settingsKey: string) => {
// Update settings on changes // Update settings on changes
if (settingsKey === settingsFilePath) if (settingsKey === settingsFilePath)
JanInferenceOpenAIExtension.writeDefaultEngineSettings(); JanInferenceOpenAIExtension.writeDefaultEngineSettings()
}, }
); )
} }
/** /**
@ -99,45 +99,45 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
try { try {
const engineFile = join( const engineFile = join(
JanInferenceOpenAIExtension._engineDir, JanInferenceOpenAIExtension._engineDir,
JanInferenceOpenAIExtension._engineMetadataFileName, JanInferenceOpenAIExtension._engineMetadataFileName
); )
if (await fs.existsSync(engineFile)) { if (await fs.existsSync(engineFile)) {
const engine = await fs.readFileSync(engineFile, "utf-8"); const engine = await fs.readFileSync(engineFile, 'utf-8')
JanInferenceOpenAIExtension._engineSettings = JanInferenceOpenAIExtension._engineSettings =
typeof engine === "object" ? engine : JSON.parse(engine); typeof engine === 'object' ? engine : JSON.parse(engine)
} else { } else {
await fs.writeFileSync( await fs.writeFileSync(
engineFile, engineFile,
JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2), JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2)
); )
} }
} catch (err) { } catch (err) {
console.error(err); console.error(err)
} }
} }
private static async handleModelInit(model: OpenAIModel) { private static async handleModelInit(model: OpenAIModel) {
if (model.engine !== InferenceEngine.openai) { if (model.engine !== InferenceEngine.openai) {
return; return
} else { } else {
JanInferenceOpenAIExtension._currentModel = model; JanInferenceOpenAIExtension._currentModel = model
JanInferenceOpenAIExtension.writeDefaultEngineSettings(); JanInferenceOpenAIExtension.writeDefaultEngineSettings()
// Todo: Check model list with API key // Todo: Check model list with API key
events.emit(ModelEvent.OnModelReady, model); events.emit(ModelEvent.OnModelReady, model)
} }
} }
private static async handleModelStop(model: OpenAIModel) { private static async handleModelStop(model: OpenAIModel) {
if (model.engine !== "openai") { if (model.engine !== 'openai') {
return; return
} }
events.emit(ModelEvent.OnModelStopped, model); events.emit(ModelEvent.OnModelStopped, model)
} }
private static async handleInferenceStopped( private static async handleInferenceStopped(
instance: JanInferenceOpenAIExtension, instance: JanInferenceOpenAIExtension
) { ) {
instance.isCancelled = true; instance.isCancelled = true
instance.controller?.abort(); instance.controller?.abort()
} }
/** /**
@ -148,13 +148,13 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
*/ */
private static async handleMessageRequest( private static async handleMessageRequest(
data: MessageRequest, data: MessageRequest,
instance: JanInferenceOpenAIExtension, instance: JanInferenceOpenAIExtension
) { ) {
if (data.model.engine !== "openai") { if (data.model.engine !== 'openai') {
return; return
} }
const timestamp = Date.now(); const timestamp = Date.now()
const message: ThreadMessage = { const message: ThreadMessage = {
id: ulid(), id: ulid(),
thread_id: data.threadId, thread_id: data.threadId,
@ -165,15 +165,15 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
status: MessageStatus.Pending, status: MessageStatus.Pending,
created: timestamp, created: timestamp,
updated: timestamp, updated: timestamp,
object: "thread.message", object: 'thread.message',
};
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message);
} }
instance.isCancelled = false; if (data.type !== MessageRequestType.Summary) {
instance.controller = new AbortController(); events.emit(MessageEvent.OnMessageResponse, message)
}
instance.isCancelled = false
instance.controller = new AbortController()
requestInference( requestInference(
data?.messages ?? [], data?.messages ?? [],
@ -182,7 +182,7 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
...JanInferenceOpenAIExtension._currentModel, ...JanInferenceOpenAIExtension._currentModel,
parameters: data.model.parameters, parameters: data.model.parameters,
}, },
instance.controller, instance.controller
).subscribe({ ).subscribe({
next: (content) => { next: (content) => {
const messageContent: ThreadContent = { const messageContent: ThreadContent = {
@ -191,33 +191,33 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
value: content.trim(), value: content.trim(),
annotations: [], annotations: [],
}, },
}; }
message.content = [messageContent]; message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
complete: async () => { complete: async () => {
message.status = message.content.length message.status = message.content.length
? MessageStatus.Ready ? MessageStatus.Ready
: MessageStatus.Error; : MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
error: async (err) => { error: async (err) => {
if (instance.isCancelled || message.content.length > 0) { if (instance.isCancelled || message.content.length > 0) {
message.status = MessageStatus.Stopped; message.status = MessageStatus.Stopped
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
return; return
} }
const messageContent: ThreadContent = { const messageContent: ThreadContent = {
type: ContentType.Text, type: ContentType.Text,
text: { text: {
value: "Error occurred: " + err.message, value: 'Error occurred: ' + err.message,
annotations: [], annotations: [],
}, },
}; }
message.content = [messageContent]; message.content = [messageContent]
message.status = MessageStatus.Error; message.status = MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
}); })
} }
} }

4
extensions/inference-openai-extension/tsconfig.json
View File

@ -8,7 +8,7 @@
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"strict": false, "strict": false,
"skipLibCheck": true, "skipLibCheck": true,
"rootDir": "./src", "rootDir": "./src"
}, },
"include": ["./src"], "include": ["./src"]
} }

26
extensions/inference-openai-extension/webpack.config.js
View File

@ -1,16 +1,16 @@
const path = require("path"); const path = require('path')
const webpack = require("webpack"); const webpack = require('webpack')
const packageJson = require("./package.json"); const packageJson = require('./package.json')
module.exports = { module.exports = {
experiments: { outputModule: true }, experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: "production", mode: 'production',
module: { module: {
rules: [ rules: [
{ {
test: /\.tsx?$/, test: /\.tsx?$/,
use: "ts-loader", use: 'ts-loader',
exclude: /node_modules/, exclude: /node_modules/,
}, },
], ],
@ -18,22 +18,22 @@ module.exports = {
plugins: [ plugins: [
new webpack.DefinePlugin({ new webpack.DefinePlugin({
MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
OPENAI_DOMAIN: JSON.stringify("openai.azure.com"), OPENAI_DOMAIN: JSON.stringify('openai.azure.com'),
}), }),
], ],
output: { output: {
filename: "index.js", // Adjust the output file name as needed filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"), path: path.resolve(__dirname, 'dist'),
library: { type: "module" }, // Specify ESM output format library: { type: 'module' }, // Specify ESM output format
}, },
resolve: { resolve: {
extensions: [".ts", ".js"], extensions: ['.ts', '.js'],
fallback: { fallback: {
path: require.resolve("path-browserify"), path: require.resolve('path-browserify'),
}, },
}, },
optimization: { optimization: {
minimize: false, minimize: false,
}, },
// Add loaders and other configuration as needed for your project // Add loaders and other configuration as needed for your project
}; }

4
extensions/inference-triton-trtllm-extension/src/@types/global.d.ts vendored
View File

@ -1,5 +1,5 @@
import { Model } from "@janhq/core"; import { Model } from '@janhq/core'
declare interface EngineSettings { declare interface EngineSettings {
base_url?: string; base_url?: string
} }

54
extensions/inference-triton-trtllm-extension/src/helpers/sse.ts
View File

@ -1,6 +1,6 @@
import { Observable } from "rxjs"; import { Observable } from 'rxjs'
import { EngineSettings } from "../@types/global"; import { EngineSettings } from '../@types/global'
import { Model } from "@janhq/core"; import { Model } from '@janhq/core'
/** /**
* Sends a request to the inference server to generate a response based on the recent messages. * Sends a request to the inference server to generate a response based on the recent messages.
@ -16,48 +16,48 @@ export function requestInference(
controller?: AbortController controller?: AbortController
): Observable<string> { ): Observable<string> {
return new Observable((subscriber) => { return new Observable((subscriber) => {
const text_input = recentMessages.map((message) => message.text).join("\n"); const text_input = recentMessages.map((message) => message.text).join('\n')
const requestBody = JSON.stringify({ const requestBody = JSON.stringify({
text_input: text_input, text_input: text_input,
max_tokens: 4096, max_tokens: 4096,
temperature: 0, temperature: 0,
bad_words: "", bad_words: '',
stop_words: "[DONE]", stop_words: '[DONE]',
stream: true stream: true,
}); })
fetch(`${engine.base_url}/v2/models/ensemble/generate_stream`, { fetch(`${engine.base_url}/v2/models/ensemble/generate_stream`, {
method: "POST", method: 'POST',
headers: { headers: {
"Content-Type": "application/json", 'Content-Type': 'application/json',
Accept: "text/event-stream", 'Accept': 'text/event-stream',
"Access-Control-Allow-Origin": "*", 'Access-Control-Allow-Origin': '*',
}, },
body: requestBody, body: requestBody,
signal: controller?.signal, signal: controller?.signal,
}) })
.then(async (response) => { .then(async (response) => {
const stream = response.body; const stream = response.body
const decoder = new TextDecoder("utf-8"); const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader(); const reader = stream?.getReader()
let content = ""; let content = ''
while (true && reader) { while (true && reader) {
const { done, value } = await reader.read(); const { done, value } = await reader.read()
if (done) { if (done) {
break; break
} }
const text = decoder.decode(value); const text = decoder.decode(value)
const lines = text.trim().split("\n"); const lines = text.trim().split('\n')
for (const line of lines) { for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { if (line.startsWith('data: ') && !line.includes('data: [DONE]')) {
const data = JSON.parse(line.replace("data: ", "")); const data = JSON.parse(line.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ""; content += data.choices[0]?.delta?.content ?? ''
subscriber.next(content); subscriber.next(content)
} }
} }
} }
subscriber.complete(); subscriber.complete()
}) })
.catch((err) => subscriber.error(err)); .catch((err) => subscriber.error(err))
}); })
} }

116
extensions/inference-triton-trtllm-extension/src/index.ts
View File

@ -20,51 +20,49 @@ import {
BaseExtension, BaseExtension,
MessageEvent, MessageEvent,
ModelEvent, ModelEvent,
} from "@janhq/core"; } from '@janhq/core'
import { requestInference } from "./helpers/sse"; import { requestInference } from './helpers/sse'
import { ulid } from "ulid"; import { ulid } from 'ulid'
import { join } from "path"; import { join } from 'path'
import { EngineSettings } from "./@types/global"; import { EngineSettings } from './@types/global'
/** /**
* A class that implements the InferenceExtension interface from the @janhq/core package. * A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests. * The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/ */
export default class JanInferenceTritonTrtLLMExtension export default class JanInferenceTritonTrtLLMExtension extends BaseExtension {
extends BaseExtension private static readonly _homeDir = 'file://engines'
{ private static readonly _engineMetadataFileName = 'triton_trtllm.json'
private static readonly _homeDir = "file://engines";
private static readonly _engineMetadataFileName = "triton_trtllm.json";
static _currentModel: Model; static _currentModel: Model
static _engineSettings: EngineSettings = { static _engineSettings: EngineSettings = {
base_url: "", base_url: '',
}; }
controller = new AbortController(); controller = new AbortController()
isCancelled = false; isCancelled = false
/** /**
* Subscribes to events emitted by the @janhq/core package. * Subscribes to events emitted by the @janhq/core package.
*/ */
async onLoad() { async onLoad() {
if (!(await fs.existsSync(JanInferenceTritonTrtLLMExtension._homeDir))) if (!(await fs.existsSync(JanInferenceTritonTrtLLMExtension._homeDir)))
JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings()
// Events subscription // Events subscription
events.on(MessageEvent.OnMessageSent, (data) => events.on(MessageEvent.OnMessageSent, (data) =>
JanInferenceTritonTrtLLMExtension.handleMessageRequest(data, this) JanInferenceTritonTrtLLMExtension.handleMessageRequest(data, this)
); )
events.on(ModelEvent.OnModelInit, (model: Model) => { events.on(ModelEvent.OnModelInit, (model: Model) => {
JanInferenceTritonTrtLLMExtension.handleModelInit(model); JanInferenceTritonTrtLLMExtension.handleModelInit(model)
}); })
events.on(ModelEvent.OnModelStop, (model: Model) => { events.on(ModelEvent.OnModelStop, (model: Model) => {
JanInferenceTritonTrtLLMExtension.handleModelStop(model); JanInferenceTritonTrtLLMExtension.handleModelStop(model)
}); })
} }
/** /**
@ -81,7 +79,7 @@ export default class JanInferenceTritonTrtLLMExtension
modelId: string, modelId: string,
settings?: ModelSettingParams settings?: ModelSettingParams
): Promise<void> { ): Promise<void> {
return; return
} }
static async writeDefaultEngineSettings() { static async writeDefaultEngineSettings() {
@ -89,11 +87,11 @@ export default class JanInferenceTritonTrtLLMExtension
const engine_json = join( const engine_json = join(
JanInferenceTritonTrtLLMExtension._homeDir, JanInferenceTritonTrtLLMExtension._homeDir,
JanInferenceTritonTrtLLMExtension._engineMetadataFileName JanInferenceTritonTrtLLMExtension._engineMetadataFileName
); )
if (await fs.existsSync(engine_json)) { if (await fs.existsSync(engine_json)) {
const engine = await fs.readFileSync(engine_json, "utf-8"); const engine = await fs.readFileSync(engine_json, 'utf-8')
JanInferenceTritonTrtLLMExtension._engineSettings = JanInferenceTritonTrtLLMExtension._engineSettings =
typeof engine === "object" ? engine : JSON.parse(engine); typeof engine === 'object' ? engine : JSON.parse(engine)
} else { } else {
await fs.writeFileSync( await fs.writeFileSync(
engine_json, engine_json,
@ -102,10 +100,10 @@ export default class JanInferenceTritonTrtLLMExtension
null, null,
2 2
) )
); )
} }
} catch (err) { } catch (err) {
console.error(err); console.error(err)
} }
} }
/** /**
@ -119,26 +117,26 @@ export default class JanInferenceTritonTrtLLMExtension
* @returns {Promise<void>} A promise that resolves when the streaming is stopped. * @returns {Promise<void>} A promise that resolves when the streaming is stopped.
*/ */
async stopInference(): Promise<void> { async stopInference(): Promise<void> {
this.isCancelled = true; this.isCancelled = true
this.controller?.abort(); this.controller?.abort()
} }
private static async handleModelInit(model: Model) { private static async handleModelInit(model: Model) {
if (model.engine !== "triton_trtllm") { if (model.engine !== 'triton_trtllm') {
return; return
} else { } else {
JanInferenceTritonTrtLLMExtension._currentModel = model; JanInferenceTritonTrtLLMExtension._currentModel = model
JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings()
// Todo: Check model list with API key // Todo: Check model list with API key
events.emit(ModelEvent.OnModelReady, model); events.emit(ModelEvent.OnModelReady, model)
} }
} }
private static async handleModelStop(model: Model) { private static async handleModelStop(model: Model) {
if (model.engine !== "triton_trtllm") { if (model.engine !== 'triton_trtllm') {
return; return
} }
events.emit(ModelEvent.OnModelStopped, model); events.emit(ModelEvent.OnModelStopped, model)
} }
/** /**
@ -151,11 +149,11 @@ export default class JanInferenceTritonTrtLLMExtension
data: MessageRequest, data: MessageRequest,
instance: JanInferenceTritonTrtLLMExtension instance: JanInferenceTritonTrtLLMExtension
) { ) {
if (data.model.engine !== "triton_trtllm") { if (data.model.engine !== 'triton_trtllm') {
return; return
} }
const timestamp = Date.now(); const timestamp = Date.now()
const message: ThreadMessage = { const message: ThreadMessage = {
id: ulid(), id: ulid(),
thread_id: data.threadId, thread_id: data.threadId,
@ -165,12 +163,12 @@ export default class JanInferenceTritonTrtLLMExtension
status: MessageStatus.Pending, status: MessageStatus.Pending,
created: timestamp, created: timestamp,
updated: timestamp, updated: timestamp,
object: "thread.message", object: 'thread.message',
}; }
events.emit(MessageEvent.OnMessageResponse, message); events.emit(MessageEvent.OnMessageResponse, message)
instance.isCancelled = false; instance.isCancelled = false
instance.controller = new AbortController(); instance.controller = new AbortController()
requestInference( requestInference(
data?.messages ?? [], data?.messages ?? [],
@ -188,33 +186,33 @@ export default class JanInferenceTritonTrtLLMExtension
value: content.trim(), value: content.trim(),
annotations: [], annotations: [],
}, },
}; }
message.content = [messageContent]; message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
complete: async () => { complete: async () => {
message.status = message.content.length message.status = message.content.length
? MessageStatus.Ready ? MessageStatus.Ready
: MessageStatus.Error; : MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
error: async (err) => { error: async (err) => {
if (instance.isCancelled || message.content.length) { if (instance.isCancelled || message.content.length) {
message.status = MessageStatus.Error; message.status = MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
return; return
} }
const messageContent: ThreadContent = { const messageContent: ThreadContent = {
type: ContentType.Text, type: ContentType.Text,
text: { text: {
value: "Error occurred: " + err.message, value: 'Error occurred: ' + err.message,
annotations: [], annotations: [],
}, },
}; }
message.content = [messageContent]; message.content = [messageContent]
message.status = MessageStatus.Ready; message.status = MessageStatus.Ready
events.emit(MessageEvent.OnMessageUpdate, message); events.emit(MessageEvent.OnMessageUpdate, message)
}, },
}); })
} }
} }

4
extensions/inference-triton-trtllm-extension/tsconfig.json
View File

@ -8,7 +8,7 @@
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"strict": false, "strict": false,
"skipLibCheck": true, "skipLibCheck": true,
"rootDir": "./src", "rootDir": "./src"
}, },
"include": ["./src"], "include": ["./src"]
} }

24
extensions/inference-triton-trtllm-extension/webpack.config.js
View File

@ -1,16 +1,16 @@
const path = require("path"); const path = require('path')
const webpack = require("webpack"); const webpack = require('webpack')
const packageJson = require("./package.json"); const packageJson = require('./package.json')
module.exports = { module.exports = {
experiments: { outputModule: true }, experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: "production", mode: 'production',
module: { module: {
rules: [ rules: [
{ {
test: /\.tsx?$/, test: /\.tsx?$/,
use: "ts-loader", use: 'ts-loader',
exclude: /node_modules/, exclude: /node_modules/,
}, },
], ],
@ -21,18 +21,18 @@ module.exports = {
}), }),
], ],
output: { output: {
filename: "index.js", // Adjust the output file name as needed filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"), path: path.resolve(__dirname, 'dist'),
library: { type: "module" }, // Specify ESM output format library: { type: 'module' }, // Specify ESM output format
}, },
resolve: { resolve: {
extensions: [".ts", ".js"], extensions: ['.ts', '.js'],
fallback: { fallback: {
path: require.resolve("path-browserify"), path: require.resolve('path-browserify'),
}, },
}, },
optimization: { optimization: {
minimize: false, minimize: false,
}, },
// Add loaders and other configuration as needed for your project // Add loaders and other configuration as needed for your project
}; }

4
extensions/model-extension/tsconfig.json
View File

@ -8,7 +8,7 @@
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"strict": false, "strict": false,
"skipLibCheck": true, "skipLibCheck": true,
"rootDir": "./src", "rootDir": "./src"
}, },
"include": ["./src"], "include": ["./src"]
} }

2
extensions/model-extension/webpack.config.js
View File

@ -19,7 +19,7 @@ module.exports = {
new webpack.DefinePlugin({ new webpack.DefinePlugin({
EXTENSION_NAME: JSON.stringify(packageJson.name), EXTENSION_NAME: JSON.stringify(packageJson.name),
MODULE_PATH: JSON.stringify(`${packageJson.name}/${packageJson.module}`), MODULE_PATH: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
VERSION: JSON.stringify(packageJson.version) VERSION: JSON.stringify(packageJson.version),
}), }),
], ],
output: { output: {

2
extensions/monitoring-extension/src/@types/global.d.ts vendored
View File

@ -1 +1 @@
declare const MODULE: string; declare const MODULE: string

6
extensions/monitoring-extension/src/index.ts
View File

@ -1,4 +1,4 @@
import { MonitoringExtension, executeOnMain } from "@janhq/core"; import { MonitoringExtension, executeOnMain } from '@janhq/core'
/** /**
* JanMonitoringExtension is a extension that provides system monitoring functionality. * JanMonitoringExtension is a extension that provides system monitoring functionality.
@ -20,7 +20,7 @@ export default class JanMonitoringExtension extends MonitoringExtension {
* @returns A Promise that resolves to an object containing information about the system resources. * @returns A Promise that resolves to an object containing information about the system resources.
*/ */
getResourcesInfo(): Promise<any> { getResourcesInfo(): Promise<any> {
return executeOnMain(MODULE, "getResourcesInfo"); return executeOnMain(MODULE, 'getResourcesInfo')
} }
/** /**
@ -28,6 +28,6 @@ export default class JanMonitoringExtension extends MonitoringExtension {
* @returns A Promise that resolves to an object containing information about the current system load. * @returns A Promise that resolves to an object containing information about the current system load.
*/ */
getCurrentLoad(): Promise<any> { getCurrentLoad(): Promise<any> {
return executeOnMain(MODULE, "getCurrentLoad"); return executeOnMain(MODULE, 'getCurrentLoad')
} }
} }

91
extensions/monitoring-extension/src/module.ts
View File

@ -1,73 +1,92 @@
const nodeOsUtils = require("node-os-utils"); const nodeOsUtils = require('node-os-utils')
const getJanDataFolderPath = require("@janhq/core/node").getJanDataFolderPath; const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath
const path = require("path"); const path = require('path')
const { readFileSync } = require("fs"); const { readFileSync } = require('fs')
const exec = require("child_process").exec; const exec = require('child_process').exec
const NVIDIA_INFO_FILE = path.join( const NVIDIA_INFO_FILE = path.join(
getJanDataFolderPath(), getJanDataFolderPath(),
"settings", 'settings',
"settings.json" 'settings.json'
); )
const getResourcesInfo = () => const getResourcesInfo = () =>
new Promise((resolve) => { new Promise((resolve) => {
nodeOsUtils.mem.used().then((ramUsedInfo) => { nodeOsUtils.mem.used().then((ramUsedInfo) => {
const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024; const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024
const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024; const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024
const response = { const response = {
mem: { mem: {
totalMemory, totalMemory,
usedMemory, usedMemory,
}, },
}; }
resolve(response); resolve(response)
}); })
}); })
const getCurrentLoad = () => const getCurrentLoad = () =>
new Promise((resolve, reject) => { new Promise((resolve, reject) => {
nodeOsUtils.cpu.usage().then((cpuPercentage) => { nodeOsUtils.cpu.usage().then((cpuPercentage) => {
let data = { let data = {
run_mode: "cpu", run_mode: 'cpu',
gpus_in_use: [], gpus_in_use: [],
};
if (process.platform !== "darwin") {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} }
if (data.run_mode === "gpu" && data.gpus_in_use.length > 0) { if (process.platform !== 'darwin') {
const gpuIds = data["gpus_in_use"].join(","); data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (gpuIds !== "") { }
if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) {
const gpuIds = data['gpus_in_use'].join(',')
if (gpuIds !== '') {
exec( exec(
`nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`,
(error, stdout, stderr) => { (error, stdout, stderr) => {
if (error) { if (error) {
console.error(`exec error: ${error}`); console.error(`exec error: ${error}`)
reject(error); reject(error)
return; return
} }
const gpuInfo = stdout.trim().split("\n").map((line) => { const gpuInfo = stdout
const [id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization] = line.split(", ").map(item => item.replace(/\r/g, "")); .trim()
return { id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization }; .split('\n')
}); .map((line) => {
const [
id,
name,
temperature,
utilization,
memoryTotal,
memoryFree,
memoryUtilization,
] = line.split(', ').map((item) => item.replace(/\r/g, ''))
return {
id,
name,
temperature,
utilization,
memoryTotal,
memoryFree,
memoryUtilization,
}
})
resolve({ resolve({
cpu: { usage: cpuPercentage }, cpu: { usage: cpuPercentage },
gpu: gpuInfo gpu: gpuInfo,
}); })
} }
); )
} else { } else {
// Handle the case where gpuIds is empty // Handle the case where gpuIds is empty
resolve({ cpu: { usage: cpuPercentage }, gpu: [] }); resolve({ cpu: { usage: cpuPercentage }, gpu: [] })
} }
} else { } else {
// Handle the case where run_mode is not 'gpu' or no GPUs are in use // Handle the case where run_mode is not 'gpu' or no GPUs are in use
resolve({ cpu: { usage: cpuPercentage }, gpu: [] }); resolve({ cpu: { usage: cpuPercentage }, gpu: [] })
} }
}); })
}); })
module.exports = { module.exports = {
getResourcesInfo, getResourcesInfo,
getCurrentLoad, getCurrentLoad,
}; }

22
extensions/monitoring-extension/webpack.config.js
View File

@ -1,24 +1,24 @@
const path = require("path"); const path = require('path')
const webpack = require("webpack"); const webpack = require('webpack')
const packageJson = require("./package.json"); const packageJson = require('./package.json')
module.exports = { module.exports = {
experiments: { outputModule: true }, experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: "production", mode: 'production',
module: { module: {
rules: [ rules: [
{ {
test: /\.tsx?$/, test: /\.tsx?$/,
use: "ts-loader", use: 'ts-loader',
exclude: /node_modules/, exclude: /node_modules/,
}, },
], ],
}, },
output: { output: {
filename: "index.js", // Adjust the output file name as needed filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"), path: path.resolve(__dirname, 'dist'),
library: { type: "module" }, // Specify ESM output format library: { type: 'module' }, // Specify ESM output format
}, },
plugins: [ plugins: [
new webpack.DefinePlugin({ new webpack.DefinePlugin({
@ -26,10 +26,10 @@ module.exports = {
}), }),
], ],
resolve: { resolve: {
extensions: [".ts", ".js"], extensions: ['.ts', '.js'],
}, },
optimization: { optimization: {
minimize: false, minimize: false,
}, },
// Add loaders and other configuration as needed for your project // Add loaders and other configuration as needed for your project
}; }

38
server/helpers/setup.ts
View File

@ -1,47 +1,47 @@
import { join, extname } from "path"; import { join, extname } from 'path'
import { existsSync, readdirSync, writeFileSync, mkdirSync } from "fs"; import { existsSync, readdirSync, writeFileSync, mkdirSync } from 'fs'
import { init, installExtensions } from "@janhq/core/node"; import { init, installExtensions } from '@janhq/core/node'
export async function setup() { export async function setup() {
/** /**
* Setup Jan Data Directory * Setup Jan Data Directory
*/ */
const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, "..", "jan"); const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, '..', 'jan')
console.debug(`Create app data directory at ${appDir}...`); console.debug(`Create app data directory at ${appDir}...`)
if (!existsSync(appDir)) mkdirSync(appDir); if (!existsSync(appDir)) mkdirSync(appDir)
//@ts-ignore //@ts-ignore
global.core = { global.core = {
// Define appPath function for app to retrieve app path globaly // Define appPath function for app to retrieve app path globaly
appPath: () => appDir, appPath: () => appDir,
}; }
init({ init({
extensionsPath: join(appDir, "extensions"), extensionsPath: join(appDir, 'extensions'),
}); })
/** /**
* Write app configurations. See #1619 * Write app configurations. See #1619
*/ */
console.debug("Writing config file..."); console.debug('Writing config file...')
writeFileSync( writeFileSync(
join(appDir, "settings.json"), join(appDir, 'settings.json'),
JSON.stringify({ JSON.stringify({
data_folder: appDir, data_folder: appDir,
}), }),
"utf-8" 'utf-8'
); )
/** /**
* Install extensions * Install extensions
*/ */
console.debug("Installing extensions..."); console.debug('Installing extensions...')
const baseExtensionPath = join(__dirname, "../../..", "pre-install"); const baseExtensionPath = join(__dirname, '../../..', 'pre-install')
const extensions = readdirSync(baseExtensionPath) const extensions = readdirSync(baseExtensionPath)
.filter((file) => extname(file) === ".tgz") .filter((file) => extname(file) === '.tgz')
.map((file) => join(baseExtensionPath, file)); .map((file) => join(baseExtensionPath, file))
await installExtensions(extensions); await installExtensions(extensions)
console.debug("Extensions installed"); console.debug('Extensions installed')
} }

106
server/index.ts
View File

@ -1,26 +1,26 @@
import fastify from "fastify"; import fastify from 'fastify'
import dotenv from "dotenv"; import dotenv from 'dotenv'
import { import {
getServerLogPath, getServerLogPath,
v1Router, v1Router,
logServer, logServer,
getJanExtensionsPath, getJanExtensionsPath,
} from "@janhq/core/node"; } from '@janhq/core/node'
import { join } from "path"; import { join } from 'path'
// Load environment variables // Load environment variables
dotenv.config(); dotenv.config()
// Define default settings // Define default settings
const JAN_API_HOST = process.env.JAN_API_HOST || "127.0.0.1"; const JAN_API_HOST = process.env.JAN_API_HOST || '127.0.0.1'
const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || "1337"); const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || '1337')
// Initialize server settings // Initialize server settings
let server: any | undefined = undefined; let server: any | undefined = undefined
let hostSetting: string = JAN_API_HOST; let hostSetting: string = JAN_API_HOST
let portSetting: number = JAN_API_PORT; let portSetting: number = JAN_API_PORT
let corsEnabled: boolean = true; let corsEnabled: boolean = true
let isVerbose: boolean = true; let isVerbose: boolean = true
/** /**
* Server configurations * Server configurations
@ -32,13 +32,13 @@ let isVerbose: boolean = true;
* @param baseDir - Base directory for the OpenAPI schema file * @param baseDir - Base directory for the OpenAPI schema file
*/ */
export interface ServerConfig { export interface ServerConfig {
host?: string; host?: string
port?: number; port?: number
isCorsEnabled?: boolean; isCorsEnabled?: boolean
isVerboseEnabled?: boolean; isVerboseEnabled?: boolean
schemaPath?: string; schemaPath?: string
baseDir?: string; baseDir?: string
storageAdataper?: any; storageAdataper?: any
} }
/** /**
@ -47,69 +47,69 @@ export interface ServerConfig {
*/ */
export const startServer = async (configs?: ServerConfig) => { export const startServer = async (configs?: ServerConfig) => {
// Update server settings // Update server settings
isVerbose = configs?.isVerboseEnabled ?? true; isVerbose = configs?.isVerboseEnabled ?? true
hostSetting = configs?.host ?? JAN_API_HOST; hostSetting = configs?.host ?? JAN_API_HOST
portSetting = configs?.port ?? JAN_API_PORT; portSetting = configs?.port ?? JAN_API_PORT
corsEnabled = configs?.isCorsEnabled ?? true; corsEnabled = configs?.isCorsEnabled ?? true
const serverLogPath = getServerLogPath(); const serverLogPath = getServerLogPath()
// Start the server // Start the server
try { try {
// Log server start // Log server start
if (isVerbose) logServer(`Debug: Starting JAN API server...`); if (isVerbose) logServer(`Debug: Starting JAN API server...`)
// Initialize Fastify server with logging // Initialize Fastify server with logging
server = fastify({ server = fastify({
logger: { logger: {
level: "info", level: 'info',
file: serverLogPath, file: serverLogPath,
}, },
}); })
// Register CORS if enabled // Register CORS if enabled
if (corsEnabled) await server.register(require("@fastify/cors"), {}); if (corsEnabled) await server.register(require('@fastify/cors'), {})
// Register Swagger for API documentation // Register Swagger for API documentation
await server.register(require("@fastify/swagger"), { await server.register(require('@fastify/swagger'), {
mode: "static", mode: 'static',
specification: { specification: {
path: configs?.schemaPath ?? "./../docs/openapi/jan.yaml", path: configs?.schemaPath ?? './../docs/openapi/jan.yaml',
baseDir: configs?.baseDir ?? "./../docs/openapi", baseDir: configs?.baseDir ?? './../docs/openapi',
}, },
}); })
// Register Swagger UI // Register Swagger UI
await server.register(require("@fastify/swagger-ui"), { await server.register(require('@fastify/swagger-ui'), {
routePrefix: "/", routePrefix: '/',
baseDir: configs?.baseDir ?? join(__dirname, "../..", "./docs/openapi"), baseDir: configs?.baseDir ?? join(__dirname, '../..', './docs/openapi'),
uiConfig: { uiConfig: {
docExpansion: "full", docExpansion: 'full',
deepLinking: false, deepLinking: false,
}, },
staticCSP: false, staticCSP: false,
transformSpecificationClone: true, transformSpecificationClone: true,
}); })
// Register static file serving for extensions // Register static file serving for extensions
// TODO: Watch extension files changes and reload // TODO: Watch extension files changes and reload
await server.register( await server.register(
(childContext: any, _: any, done: any) => { (childContext: any, _: any, done: any) => {
childContext.register(require("@fastify/static"), { childContext.register(require('@fastify/static'), {
root: getJanExtensionsPath(), root: getJanExtensionsPath(),
wildcard: false, wildcard: false,
}); })
done(); done()
}, },
{ prefix: "extensions" } { prefix: 'extensions' }
); )
// Register proxy middleware // Register proxy middleware
if (configs?.storageAdataper) if (configs?.storageAdataper)
server.addHook("preHandler", configs.storageAdataper); server.addHook('preHandler', configs.storageAdataper)
// Register API routes // Register API routes
await server.register(v1Router, { prefix: "/v1" }); await server.register(v1Router, { prefix: '/v1' })
// Start listening for requests // Start listening for requests
await server await server
.listen({ .listen({
@ -121,13 +121,13 @@ export const startServer = async (configs?: ServerConfig) => {
if (isVerbose) if (isVerbose)
logServer( logServer(
`Debug: JAN API listening at: http://${hostSetting}:${portSetting}` `Debug: JAN API listening at: http://${hostSetting}:${portSetting}`
); )
}); })
} catch (e) { } catch (e) {
// Log any errors // Log any errors
if (isVerbose) logServer(`Error: ${e}`); if (isVerbose) logServer(`Error: ${e}`)
} }
}; }
/** /**
* Function to stop the server * Function to stop the server
@ -135,11 +135,11 @@ export const startServer = async (configs?: ServerConfig) => {
export const stopServer = async () => { export const stopServer = async () => {
try { try {
// Log server stop // Log server stop
if (isVerbose) logServer(`Debug: Server stopped`); if (isVerbose) logServer(`Debug: Server stopped`)
// Stop the server // Stop the server
await server.close(); await server.close()
} catch (e) { } catch (e) {
// Log any errors // Log any errors
if (isVerbose) logServer(`Error: ${e}`); if (isVerbose) logServer(`Error: ${e}`)
} }
}; }

8
server/main.ts
View File

@ -1,7 +1,7 @@
import { s3 } from "./middleware/s3"; import { s3 } from './middleware/s3'
import { setup } from "./helpers/setup"; import { setup } from './helpers/setup'
import { startServer as start } from "./index"; import { startServer as start } from './index'
/** /**
* Setup extensions and start the server * Setup extensions and start the server
*/ */
setup().then(() => start({ storageAdataper: s3 })); setup().then(() => start({ storageAdataper: s3 }))

52
server/middleware/s3.ts
View File

@ -1,4 +1,4 @@
import { join } from "path"; import { join } from 'path'
// Middleware to intercept requests and proxy if certain conditions are met // Middleware to intercept requests and proxy if certain conditions are met
const config = { const config = {
@ -8,63 +8,63 @@ const config = {
accessKeyId: process.env.AWS_ACCESS_KEY_ID, accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
}, },
}; }
const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME; const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME
const fs = require("@cyclic.sh/s3fs")(S3_BUCKET_NAME, config); const fs = require('@cyclic.sh/s3fs')(S3_BUCKET_NAME, config)
const PROXY_PREFIX = "/v1/fs"; const PROXY_PREFIX = '/v1/fs'
const PROXY_ROUTES = ["/threads", "/messages"]; const PROXY_ROUTES = ['/threads', '/messages']
export const s3 = (req: any, reply: any, done: any) => { export const s3 = (req: any, reply: any, done: any) => {
// Proxy FS requests to S3 using S3FS // Proxy FS requests to S3 using S3FS
if (req.url.startsWith(PROXY_PREFIX)) { if (req.url.startsWith(PROXY_PREFIX)) {
const route = req.url.split("/").pop(); const route = req.url.split('/').pop()
const args = parseRequestArgs(req); const args = parseRequestArgs(req)
// Proxy matched requests to the s3fs module // Proxy matched requests to the s3fs module
if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) { if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) {
try { try {
// Handle customized route // Handle customized route
// S3FS does not handle appendFileSync // S3FS does not handle appendFileSync
if (route === "appendFileSync") { if (route === 'appendFileSync') {
let result = handAppendFileSync(args); let result = handAppendFileSync(args)
reply.status(200).send(result); reply.status(200).send(result)
return; return
} }
// Reroute the other requests to the s3fs module // Reroute the other requests to the s3fs module
const result = fs[route](...args); const result = fs[route](...args)
reply.status(200).send(result); reply.status(200).send(result)
return; return
} catch (ex) { } catch (ex) {
console.log(ex); console.log(ex)
} }
} }
} }
// Let other requests go through // Let other requests go through
done(); done()
}; }
const parseRequestArgs = (req: Request) => { const parseRequestArgs = (req: Request) => {
const { const {
getJanDataFolderPath, getJanDataFolderPath,
normalizeFilePath, normalizeFilePath,
} = require("@janhq/core/node"); } = require('@janhq/core/node')
return JSON.parse(req.body as any).map((arg: any) => return JSON.parse(req.body as any).map((arg: any) =>
typeof arg === "string" && typeof arg === 'string' &&
(arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`))
? join(getJanDataFolderPath(), normalizeFilePath(arg)) ? join(getJanDataFolderPath(), normalizeFilePath(arg))
: arg : arg
); )
}; }
const handAppendFileSync = (args: any[]) => { const handAppendFileSync = (args: any[]) => {
if (fs.existsSync(args[0])) { if (fs.existsSync(args[0])) {
const data = fs.readFileSync(args[0], "utf-8"); const data = fs.readFileSync(args[0], 'utf-8')
return fs.writeFileSync(args[0], data + args[1]); return fs.writeFileSync(args[0], data + args[1])
} else { } else {
return fs.writeFileSync(args[0], args[1]); return fs.writeFileSync(args[0], args[1])
} }
}; }

6
uikit/types/declaration.d.ts vendored
View File

@ -1,4 +1,4 @@
declare module '*.scss' { declare module '*.scss' {
const content: Record<string, string>; const content: Record<string, string>
export default content; export default content
} }

1
web/app/error.tsx
View File

@ -4,7 +4,6 @@ import { useEffect, useState } from 'react'
export default function Error({ export default function Error({
error, error,
reset,
}: { }: {
error: Error & { digest?: string } error: Error & { digest?: string }
reset: () => void reset: () => void

7
web/containers/Layout/BottomBar/index.tsx
View File

@ -56,13 +56,6 @@ const BottomBar = () => {
const setShowSelectModelModal = useSetAtom(showSelectModelModalAtom) const setShowSelectModelModal = useSetAtom(showSelectModelModalAtom)
const [serverEnabled] = useAtom(serverEnabledAtom) const [serverEnabled] = useAtom(serverEnabledAtom)
const calculateGpuMemoryUsage = (gpu: Record<string, never>) => {
const total = parseInt(gpu.memoryTotal)
const free = parseInt(gpu.memoryFree)
if (!total || !free) return 0
return Math.round(((total - free) / total) * 100)
}
const calculateUtilization = () => { const calculateUtilization = () => {
let sum = 0 let sum = 0
const util = gpus.map((x) => { const util = gpus.map((x) => {

3
web/screens/Chat/index.tsx
View File

@ -13,7 +13,7 @@ import GenerateResponse from '@/containers/Loader/GenerateResponse'
import ModelReload from '@/containers/Loader/ModelReload' import ModelReload from '@/containers/Loader/ModelReload'
import ModelStart from '@/containers/Loader/ModelStart' import ModelStart from '@/containers/Loader/ModelStart'
import { currentPromptAtom, fileUploadAtom } from '@/containers/Providers/Jotai' import { fileUploadAtom } from '@/containers/Providers/Jotai'
import { showLeftSideBarAtom } from '@/containers/Providers/KeyListener' import { showLeftSideBarAtom } from '@/containers/Providers/KeyListener'
import { snackbar } from '@/containers/Toast' import { snackbar } from '@/containers/Toast'
@ -54,7 +54,6 @@ const renderError = (code: string) => {
} }
const ChatScreen: React.FC = () => { const ChatScreen: React.FC = () => {
const setCurrentPrompt = useSetAtom(currentPromptAtom)
const activeThread = useAtomValue(activeThreadAtom) const activeThread = useAtomValue(activeThreadAtom)
const showLeftSideBar = useAtomValue(showLeftSideBarAtom) const showLeftSideBar = useAtomValue(showLeftSideBarAtom)
const engineParamsUpdate = useAtomValue(engineParamsUpdateAtom) const engineParamsUpdate = useAtomValue(engineParamsUpdateAtom)