Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: prettier fix (#2019)

Browse Source
This commit is contained in:
Louis 2024-02-15 08:38:05 +07:00 committed by GitHub
parent b11f51d80f
commit 3412a23654
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
65 changed files with 995 additions and 992 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.devcontainer/devcontainer.json
View File

@ -1,4 +1,4 @@
{
"name": "jan",
"image": "node:20"
"name": "jan",
"image": "node:20"
}

2
core/rollup.config.ts
View File

@ -55,7 +55,7 @@ export default [
'http',
'os',
'util',
'child_process'
'child_process',
],
watch: {
include: 'src/node/**',

5
core/src/api/index.ts
View File

@ -125,8 +125,5 @@ export const CoreRoutes = [
...Object.values(FileManagerRoute),
]
export const APIRoutes = [
...CoreRoutes,
...Object.values(NativeRoute),
]
export const APIRoutes = [...CoreRoutes, ...Object.values(NativeRoute)]
export const APIEvents = [...Object.values(AppEvent), ...Object.values(DownloadEvent)]

18
core/src/extension.ts
View File

@ -1,13 +1,13 @@
export enum ExtensionTypeEnum {
Assistant = "assistant",
Conversational = "conversational",
Inference = "inference",
Model = "model",
SystemMonitoring = "systemMonitoring",
Assistant = 'assistant',
Conversational = 'conversational',
Inference = 'inference',
Model = 'model',
SystemMonitoring = 'systemMonitoring',
}
export interface ExtensionType {
type(): ExtensionTypeEnum | undefined;
type(): ExtensionTypeEnum | undefined
}
/**
* Represents a base extension.
@ -20,16 +20,16 @@ export abstract class BaseExtension implements ExtensionType {
* Undefined means its not extending any known extension by the application.
*/
type(): ExtensionTypeEnum | undefined {
return undefined;
return undefined
}
/**
* Called when the extension is loaded.
* Any initialization logic for the extension should be put here.
*/
abstract onLoad(): void;
abstract onLoad(): void
/**
* Called when the extension is unloaded.
* Any cleanup logic for the extension should be put here.
*/
abstract onUnload(): void;
abstract onUnload(): void
}

12
core/src/extensions/assistant.ts
View File

@ -1,5 +1,5 @@
import { Assistant, AssistantInterface } from "../index";
import { BaseExtension, ExtensionTypeEnum } from "../extension";
import { Assistant, AssistantInterface } from '../index'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Assistant extension for managing assistants.
@ -10,10 +10,10 @@ export abstract class AssistantExtension extends BaseExtension implements Assist
* Assistant extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Assistant;
return ExtensionTypeEnum.Assistant
}
abstract createAssistant(assistant: Assistant): Promise<void>;
abstract deleteAssistant(assistant: Assistant): Promise<void>;
abstract getAssistants(): Promise<Assistant[]>;
abstract createAssistant(assistant: Assistant): Promise<void>
abstract deleteAssistant(assistant: Assistant): Promise<void>
abstract getAssistants(): Promise<Assistant[]>
}

2
core/src/extensions/conversational.ts
View File

@ -14,7 +14,7 @@ export abstract class ConversationalExtension
* Conversation extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Conversational;
return ExtensionTypeEnum.Conversational
}
abstract getThreads(): Promise<Thread[]>

10
core/src/extensions/index.ts
View File

@ -2,24 +2,24 @@
* Conversational extension. Persists and retrieves conversations.
* @module
*/
export { ConversationalExtension } from "./conversational";
export { ConversationalExtension } from './conversational'
/**
* Inference extension. Start, stop and inference models.
*/
export { InferenceExtension } from "./inference";
export { InferenceExtension } from './inference'
/**
* Monitoring extension for system monitoring.
*/
export { MonitoringExtension } from "./monitoring";
export { MonitoringExtension } from './monitoring'
/**
* Assistant extension for managing assistants.
*/
export { AssistantExtension } from "./assistant";
export { AssistantExtension } from './assistant'
/**
* Model extension for managing models.
*/
export { ModelExtension } from "./model";
export { ModelExtension } from './model'

8
core/src/extensions/inference.ts
View File

@ -1,5 +1,5 @@
import { InferenceInterface, MessageRequest, ThreadMessage } from "../index";
import { BaseExtension, ExtensionTypeEnum } from "../extension";
import { InferenceInterface, MessageRequest, ThreadMessage } from '../index'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Inference extension. Start, stop and inference models.
@ -9,8 +9,8 @@ export abstract class InferenceExtension extends BaseExtension implements Infere
* Inference extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Inference;
return ExtensionTypeEnum.Inference
}
abstract inference(data: MessageRequest): Promise<ThreadMessage>;
abstract inference(data: MessageRequest): Promise<ThreadMessage>
}

20
core/src/extensions/model.ts
View File

@ -1,5 +1,5 @@
import { BaseExtension, ExtensionTypeEnum } from "../extension";
import { Model, ModelInterface } from "../index";
import { BaseExtension, ExtensionTypeEnum } from '../extension'
import { Model, ModelInterface } from '../index'
/**
* Model extension for managing models.
@ -9,16 +9,16 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter
* Model extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Model;
return ExtensionTypeEnum.Model
}
abstract downloadModel(
model: Model,
network?: { proxy: string; ignoreSSL?: boolean },
): Promise<void>;
abstract cancelModelDownload(modelId: string): Promise<void>;
abstract deleteModel(modelId: string): Promise<void>;
abstract saveModel(model: Model): Promise<void>;
abstract getDownloadedModels(): Promise<Model[]>;
abstract getConfiguredModels(): Promise<Model[]>;
network?: { proxy: string; ignoreSSL?: boolean }
): Promise<void>
abstract cancelModelDownload(modelId: string): Promise<void>
abstract deleteModel(modelId: string): Promise<void>
abstract saveModel(model: Model): Promise<void>
abstract getDownloadedModels(): Promise<Model[]>
abstract getConfiguredModels(): Promise<Model[]>
}

10
core/src/extensions/monitoring.ts
View File

@ -1,5 +1,5 @@
import { BaseExtension, ExtensionTypeEnum } from "../extension";
import { MonitoringInterface } from "../index";
import { BaseExtension, ExtensionTypeEnum } from '../extension'
import { MonitoringInterface } from '../index'
/**
* Monitoring extension for system monitoring.
@ -10,9 +10,9 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit
* Monitoring extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.SystemMonitoring;
return ExtensionTypeEnum.SystemMonitoring
}
abstract getResourcesInfo(): Promise<any>;
abstract getCurrentLoad(): Promise<any>;
abstract getResourcesInfo(): Promise<any>
abstract getCurrentLoad(): Promise<any>
}

8
core/src/node/api/common/adapter.ts
View File

@ -1,4 +1,10 @@
import { AppRoute, DownloadRoute, ExtensionRoute, FileManagerRoute, FileSystemRoute } from '../../../api'
import {
AppRoute,
DownloadRoute,
ExtensionRoute,
FileManagerRoute,
FileSystemRoute,
} from '../../../api'
import { Downloader } from '../processors/download'
import { FileSystem } from '../processors/fs'
import { Extension } from '../processors/extension'

2
core/src/node/extension/extension.ts
View File

@ -104,7 +104,7 @@ export default class Extension {
await pacote.extract(
this.specifier,
join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''),
this.installOptions,
this.installOptions
)
// Set the url using the custom extensions protocol

61
core/src/node/extension/store.ts
View File

@ -1,6 +1,6 @@
import { writeFileSync } from "fs";
import Extension from "./extension";
import { ExtensionManager } from "./manager";
import { writeFileSync } from 'fs'
import Extension from './extension'
import { ExtensionManager } from './manager'
/**
* @module store
@ -11,7 +11,7 @@ import { ExtensionManager } from "./manager";
* Register of installed extensions
* @type {Object.<string, Extension>} extension - List of installed extensions
*/
const extensions: Record<string, Extension> = {};
const extensions: Record<string, Extension> = {}
/**
* Get a extension from the stored extensions.
@ -21,10 +21,10 @@ const extensions: Record<string, Extension> = {};
*/
export function getExtension(name: string) {
if (!Object.prototype.hasOwnProperty.call(extensions, name)) {
throw new Error(`Extension ${name} does not exist`);
throw new Error(`Extension ${name} does not exist`)
}
return extensions[name];
return extensions[name]
}
/**
@ -33,7 +33,7 @@ export function getExtension(name: string) {
* @alias extensionManager.getAllExtensions
*/
export function getAllExtensions() {
return Object.values(extensions);
return Object.values(extensions)
}
/**
@ -42,7 +42,7 @@ export function getAllExtensions() {
* @alias extensionManager.getActiveExtensions
*/
export function getActiveExtensions() {
return Object.values(extensions).filter((extension) => extension.active);
return Object.values(extensions).filter((extension) => extension.active)
}
/**
@ -53,9 +53,9 @@ export function getActiveExtensions() {
* @alias extensionManager.removeExtension
*/
export function removeExtension(name: string, persist = true) {
const del = delete extensions[name];
if (persist) persistExtensions();
return del;
const del = delete extensions[name]
if (persist) persistExtensions()
return del
}
/**
@ -65,10 +65,10 @@ export function removeExtension(name: string, persist = true) {
* @returns {void}
*/
export function addExtension(extension: Extension, persist = true) {
if (extension.name) extensions[extension.name] = extension;
if (extension.name) extensions[extension.name] = extension
if (persist) {
persistExtensions();
extension.subscribe("pe-persist", persistExtensions);
persistExtensions()
extension.subscribe('pe-persist', persistExtensions)
}
}
@ -77,14 +77,11 @@ export function addExtension(extension: Extension, persist = true) {
* @returns {void}
*/
export function persistExtensions() {
const persistData: Record<string, Extension> = {};
const persistData: Record<string, Extension> = {}
for (const name in extensions) {
persistData[name] = extensions[name];
persistData[name] = extensions[name]
}
writeFileSync(
ExtensionManager.instance.getExtensionsFile(),
JSON.stringify(persistData),
);
writeFileSync(ExtensionManager.instance.getExtensionsFile(), JSON.stringify(persistData))
}
/**
@ -95,26 +92,28 @@ export function persistExtensions() {
* @alias extensionManager.installExtensions
*/
export async function installExtensions(extensions: any) {
const installed: Extension[] = [];
const installed: Extension[] = []
for (const ext of extensions) {
// Set install options and activation based on input type
const isObject = typeof ext === "object";
const spec = isObject ? [ext.specifier, ext] : [ext];
const activate = isObject ? ext.activate !== false : true;
const isObject = typeof ext === 'object'
const spec = isObject ? [ext.specifier, ext] : [ext]
const activate = isObject ? ext.activate !== false : true
// Install and possibly activate extension
const extension = new Extension(...spec);
if(!extension.origin) { continue }
await extension._install();
if (activate) extension.setActive(true);
const extension = new Extension(...spec)
if (!extension.origin) {
continue
}
await extension._install()
if (activate) extension.setActive(true)
// Add extension to store if needed
addExtension(extension);
installed.push(extension);
addExtension(extension)
installed.push(extension)
}
// Return list of all installed extensions
return installed;
return installed
}
/**

2
core/src/node/helper/path.ts
View File

@ -1,4 +1,4 @@
import { join } from "path"
import { join } from 'path'
/**
* Normalize file path

6
core/src/node/helper/resource.ts
View File

@ -1,6 +1,6 @@
import { SystemResourceInfo } from "../../types"
import { physicalCpuCount } from "./config"
import { log, logServer } from "./log"
import { SystemResourceInfo } from '../../types'
import { physicalCpuCount } from './config'
import { log, logServer } from './log'
export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
const cpu = await physicalCpuCount()

7
core/src/types/assistant/assistantEvent.ts
View File

@ -2,7 +2,6 @@
* The `EventName` enumeration contains the names of all the available events in the Jan platform.
*/
export enum AssistantEvent {
/** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */
OnAssistantsUpdate = 'OnAssistantsUpdate',
}
/** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */
OnAssistantsUpdate = 'OnAssistantsUpdate',
}

6
core/src/types/message/messageRequestType.ts
View File

@ -1,5 +1,5 @@
export enum MessageRequestType {
Thread = 'Thread',
Assistant = 'Assistant',
Summary = 'Summary',
Thread = 'Thread',
Assistant = 'Assistant',
Summary = 'Summary',
}

2
core/src/types/model/modelInterface.ts
View File

@ -10,7 +10,7 @@ export interface ModelInterface {
* @param network - Optional object to specify proxy/whether to ignore SSL certificates.
* @returns A Promise that resolves when the model has been downloaded.
*/
downloadModel(model: Model, network?: { ignoreSSL?: boolean, proxy?: string }): Promise<void>
downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise<void>
/**
* Cancels the download of a specific model.

5
core/tslint.json
View File

@ -1,6 +1,3 @@
{
"extends": [
"tslint-config-standard",
"tslint-config-prettier"
]
"extends": ["tslint-config-standard", "tslint-config-prettier"]
}

4
electron/main.ts
View File

@ -1,4 +1,4 @@
import { app, BrowserWindow } from 'electron'
import { app, BrowserWindow, shell } from 'electron'
import { join } from 'path'
/**
* Managers
@ -77,7 +77,7 @@ function createMainWindow() {
/* Open external links in the default browser */
mainWindow.webContents.setWindowOpenHandler(({ url }) => {
require('electron').shell.openExternal(url)
shell.openExternal(url)
return { action: 'deny' }
})

4
electron/merge-latest-ymls.js
View File

@ -9,7 +9,9 @@ const file3 = args[2]
// check that all arguments are present and throw error instead
if (!file1 || !file2 || !file3) {
throw new Error('Please provide 3 file paths as arguments: path to file1, to file2 and destination path')
throw new Error(
'Please provide 3 file paths as arguments: path to file1, to file2 and destination path'
)
}
const doc1 = yaml.load(fs.readFileSync(file1, 'utf8'))

82
electron/sign.js
View File

@ -1,44 +1,48 @@
const { exec } = require('child_process');
const { exec } = require('child_process')
function sign({
path,
name,
certUrl,
clientId,
tenantId,
clientSecret,
certName,
timestampServer,
version,
}) {
return new Promise((resolve, reject) => {
const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`
function sign({ path, name, certUrl, clientId, tenantId, clientSecret, certName, timestampServer, version }) {
return new Promise((resolve, reject) => {
const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`;
exec(command, (error, stdout, stderr) => {
if (error) {
console.error(`Error: ${error}`);
return reject(error);
}
console.log(`stdout: ${stdout}`);
console.error(`stderr: ${stderr}`);
resolve();
});
});
exec(command, (error, stdout, stderr) => {
if (error) {
console.error(`Error: ${error}`)
return reject(error)
}
console.log(`stdout: ${stdout}`)
console.error(`stderr: ${stderr}`)
resolve()
})
})
}
exports.default = async function (options) {
const certUrl = process.env.AZURE_KEY_VAULT_URI
const clientId = process.env.AZURE_CLIENT_ID
const tenantId = process.env.AZURE_TENANT_ID
const clientSecret = process.env.AZURE_CLIENT_SECRET
const certName = process.env.AZURE_CERT_NAME
const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1'
exports.default = async function(options) {
const certUrl = process.env.AZURE_KEY_VAULT_URI;
const clientId = process.env.AZURE_CLIENT_ID;
const tenantId = process.env.AZURE_TENANT_ID;
const clientSecret = process.env.AZURE_CLIENT_SECRET;
const certName = process.env.AZURE_CERT_NAME;
const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1';
await sign({
path: options.path,
name: "jan-win-x64",
certUrl,
clientId,
tenantId,
clientSecret,
certName,
timestampServer,
version: options.version
});
};
await sign({
path: options.path,
name: 'jan-win-x64',
certUrl,
clientId,
tenantId,
clientSecret,
certName,
timestampServer,
version: options.version,
})
}

6
electron/utils/disposable.ts
View File

@ -1,8 +1,8 @@
export function dispose(requiredModules: Record<string, any>) {
for (const key in requiredModules) {
const module = requiredModules[key];
if (typeof module["dispose"] === "function") {
module["dispose"]();
const module = requiredModules[key]
if (typeof module['dispose'] === 'function') {
module['dispose']()
}
}
}

10
electron/utils/setup.ts
View File

@ -1,9 +1,9 @@
import { app } from 'electron'
export const setupCore = async () => {
// Setup core api for main process
global.core = {
// Define appPath function for app to retrieve app path globaly
appPath: () => app.getPath('userData')
}
// Setup core api for main process
global.core = {
// Define appPath function for app to retrieve app path globaly
appPath: () => app.getPath('userData'),
}
}

36
extensions/assistant-extension/rollup.config.ts
View File

@ -1,22 +1,22 @@
import resolve from "@rollup/plugin-node-resolve";
import commonjs from "@rollup/plugin-commonjs";
import sourceMaps from "rollup-plugin-sourcemaps";
import typescript from "rollup-plugin-typescript2";
import json from "@rollup/plugin-json";
import replace from "@rollup/plugin-replace";
import resolve from '@rollup/plugin-node-resolve'
import commonjs from '@rollup/plugin-commonjs'
import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from 'rollup-plugin-typescript2'
import json from '@rollup/plugin-json'
import replace from '@rollup/plugin-replace'
const packageJson = require("./package.json");
const packageJson = require('./package.json')
const pkg = require("./package.json");
const pkg = require('./package.json')
export default [
{
input: `src/index.ts`,
output: [{ file: pkg.main, format: "es", sourcemap: true }],
output: [{ file: pkg.main, format: 'es', sourcemap: true }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [],
watch: {
include: "src/**",
include: 'src/**',
},
plugins: [
replace({
@ -35,7 +35,7 @@ export default [
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({
extensions: [".js", ".ts", ".svelte"],
extensions: ['.js', '.ts', '.svelte'],
}),
// Resolve source maps to the original source
@ -44,15 +44,11 @@ export default [
},
{
input: `src/node/index.ts`,
output: [{ dir: "dist/node", format: "cjs", sourcemap: false }],
output: [{ dir: 'dist/node', format: 'cjs', sourcemap: false }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [
"@janhq/core/node",
"path",
"hnswlib-node",
],
external: ['@janhq/core/node', 'path', 'hnswlib-node'],
watch: {
include: "src/node/**",
include: 'src/node/**',
},
// inlineDynamicImports: true,
plugins: [
@ -68,11 +64,11 @@ export default [
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({
extensions: [".ts", ".js", ".json"],
extensions: ['.ts', '.js', '.json'],
}),
// Resolve source maps to the original source
// sourceMaps(),
],
},
];
]

6
extensions/assistant-extension/src/@types/global.d.ts vendored
View File

@ -1,3 +1,3 @@
declare const NODE: string;
declare const EXTENSION_NAME: string;
declare const VERSION: string;
declare const NODE: string
declare const EXTENSION_NAME: string
declare const VERSION: string

157
extensions/assistant-extension/src/index.ts
View File

@ -10,57 +10,56 @@ import {
executeOnMain,
AssistantExtension,
AssistantEvent,
} from "@janhq/core";
} from '@janhq/core'
export default class JanAssistantExtension extends AssistantExtension {
private static readonly _homeDir = "file://assistants";
private static readonly _threadDir = "file://threads";
private static readonly _homeDir = 'file://assistants'
private static readonly _threadDir = 'file://threads'
controller = new AbortController();
isCancelled = false;
retrievalThreadId: string | undefined = undefined;
controller = new AbortController()
isCancelled = false
retrievalThreadId: string | undefined = undefined
async onLoad() {
// making the assistant directory
const assistantDirExist = await fs.existsSync(
JanAssistantExtension._homeDir
);
)
if (
localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION ||
!assistantDirExist
) {
if (!assistantDirExist)
await fs.mkdirSync(JanAssistantExtension._homeDir);
if (!assistantDirExist) await fs.mkdirSync(JanAssistantExtension._homeDir)
// Write assistant metadata
await this.createJanAssistant();
await this.createJanAssistant()
// Finished migration
localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION);
localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION)
// Update the assistant list
events.emit(AssistantEvent.OnAssistantsUpdate, {});
events.emit(AssistantEvent.OnAssistantsUpdate, {})
}
// Events subscription
events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
JanAssistantExtension.handleMessageRequest(data, this)
);
)
events.on(InferenceEvent.OnInferenceStopped, () => {
JanAssistantExtension.handleInferenceStopped(this);
});
JanAssistantExtension.handleInferenceStopped(this)
})
}
private static async handleInferenceStopped(instance: JanAssistantExtension) {
instance.isCancelled = true;
instance.controller?.abort();
instance.isCancelled = true
instance.controller?.abort()
}
private static async handleMessageRequest(
data: MessageRequest,
instance: JanAssistantExtension
) {
instance.isCancelled = false;
instance.controller = new AbortController();
instance.isCancelled = false
instance.controller = new AbortController()
if (
data.model?.engine !== InferenceEngine.tool_retrieval_enabled ||
@ -69,26 +68,26 @@ export default class JanAssistantExtension extends AssistantExtension {
// That could lead to an issue where thread stuck at generating response
!data.thread?.assistants[0]?.tools
) {
return;
return
}
const latestMessage = data.messages[data.messages.length - 1];
const latestMessage = data.messages[data.messages.length - 1]
// 1. Ingest the document if needed
if (
latestMessage &&
latestMessage.content &&
typeof latestMessage.content !== "string" &&
typeof latestMessage.content !== 'string' &&
latestMessage.content.length > 1
) {
const docFile = latestMessage.content[1]?.doc_url?.url;
const docFile = latestMessage.content[1]?.doc_url?.url
if (docFile) {
await executeOnMain(
NODE,
"toolRetrievalIngestNewDocument",
'toolRetrievalIngestNewDocument',
docFile,
data.model?.proxyEngine
);
)
}
} else if (
// Check whether we need to ingest document or not
@ -97,7 +96,7 @@ export default class JanAssistantExtension extends AssistantExtension {
await joinPath([
JanAssistantExtension._threadDir,
data.threadId,
"memory",
'memory',
])
))
) {
@ -108,61 +107,61 @@ export default class JanAssistantExtension extends AssistantExtension {
...data.model,
engine: data.model.proxyEngine,
},
};
events.emit(MessageEvent.OnMessageSent, output);
return;
}
events.emit(MessageEvent.OnMessageSent, output)
return
}
// 2. Load agent on thread changed
if (instance.retrievalThreadId !== data.threadId) {
await executeOnMain(NODE, "toolRetrievalLoadThreadMemory", data.threadId);
await executeOnMain(NODE, 'toolRetrievalLoadThreadMemory', data.threadId)
instance.retrievalThreadId = data.threadId;
instance.retrievalThreadId = data.threadId
// Update the text splitter
await executeOnMain(
NODE,
"toolRetrievalUpdateTextSplitter",
'toolRetrievalUpdateTextSplitter',
data.thread.assistants[0].tools[0]?.settings?.chunk_size ?? 4000,
data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200
);
)
}
// 3. Using the retrieval template with the result and query
if (latestMessage.content) {
const prompt =
typeof latestMessage.content === "string"
typeof latestMessage.content === 'string'
? latestMessage.content
: latestMessage.content[0].text;
: latestMessage.content[0].text
// Retrieve the result
const retrievalResult = await executeOnMain(
NODE,
"toolRetrievalQueryResult",
'toolRetrievalQueryResult',
prompt
);
console.debug("toolRetrievalQueryResult", retrievalResult);
)
console.debug('toolRetrievalQueryResult', retrievalResult)
// Update message content
if (data.thread?.assistants[0]?.tools && retrievalResult)
data.messages[data.messages.length - 1].content =
data.thread.assistants[0].tools[0].settings?.retrieval_template
?.replace("{CONTEXT}", retrievalResult)
.replace("{QUESTION}", prompt);
?.replace('{CONTEXT}', retrievalResult)
.replace('{QUESTION}', prompt)
}
// Filter out all the messages that are not text
data.messages = data.messages.map((message) => {
if (
message.content &&
typeof message.content !== "string" &&
typeof message.content !== 'string' &&
(message.content.length ?? 0) > 0
) {
return {
...message,
content: [message.content[0]],
};
}
}
return message;
});
return message
})
// 4. Reroute the result to inference engine
const output = {
@ -171,8 +170,8 @@ export default class JanAssistantExtension extends AssistantExtension {
...data.model,
engine: data.model.proxyEngine,
},
};
events.emit(MessageEvent.OnMessageSent, output);
}
events.emit(MessageEvent.OnMessageSent, output)
}
/**
@ -184,88 +183,88 @@ export default class JanAssistantExtension extends AssistantExtension {
const assistantDir = await joinPath([
JanAssistantExtension._homeDir,
assistant.id,
]);
if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir);
])
if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir)
// store the assistant metadata json
const assistantMetadataPath = await joinPath([
assistantDir,
"assistant.json",
]);
'assistant.json',
])
try {
await fs.writeFileSync(
assistantMetadataPath,
JSON.stringify(assistant, null, 2)
);
)
} catch (err) {
console.error(err);
console.error(err)
}
}
async getAssistants(): Promise<Assistant[]> {
// get all the assistant directories
// get all the assistant metadata json
const results: Assistant[] = [];
const results: Assistant[] = []
const allFileName: string[] = await fs.readdirSync(
JanAssistantExtension._homeDir
);
)
for (const fileName of allFileName) {
const filePath = await joinPath([
JanAssistantExtension._homeDir,
fileName,
]);
])
if (filePath.includes(".DS_Store")) continue;
if (filePath.includes('.DS_Store')) continue
const jsonFiles: string[] = (await fs.readdirSync(filePath)).filter(
(file: string) => file === "assistant.json"
);
(file: string) => file === 'assistant.json'
)
if (jsonFiles.length !== 1) {
// has more than one assistant file -> ignore
continue;
continue
}
const content = await fs.readFileSync(
await joinPath([filePath, jsonFiles[0]]),
"utf-8"
);
'utf-8'
)
const assistant: Assistant =
typeof content === "object" ? content : JSON.parse(content);
typeof content === 'object' ? content : JSON.parse(content)
results.push(assistant);
results.push(assistant)
}
return results;
return results
}
async deleteAssistant(assistant: Assistant): Promise<void> {
if (assistant.id === "jan") {
return Promise.reject("Cannot delete Jan Assistant");
if (assistant.id === 'jan') {
return Promise.reject('Cannot delete Jan Assistant')
}
// remove the directory
const assistantDir = await joinPath([
JanAssistantExtension._homeDir,
assistant.id,
]);
await fs.rmdirSync(assistantDir);
return Promise.resolve();
])
await fs.rmdirSync(assistantDir)
return Promise.resolve()
}
private async createJanAssistant(): Promise<void> {
const janAssistant: Assistant = {
avatar: "",
avatar: '',
thread_location: undefined,
id: "jan",
object: "assistant",
id: 'jan',
object: 'assistant',
created_at: Date.now(),
name: "Jan",
description: "A default assistant that can use all downloaded models",
model: "*",
instructions: "",
name: 'Jan',
description: 'A default assistant that can use all downloaded models',
model: '*',
instructions: '',
tools: [
{
type: "retrieval",
type: 'retrieval',
enabled: false,
settings: {
top_k: 2,
@ -283,8 +282,8 @@ Helpful Answer:`,
],
file_ids: [],
metadata: undefined,
};
}
await this.createAssistant(janAssistant);
await this.createAssistant(janAssistant)
}
}

16
extensions/assistant-extension/src/node/engine.ts
View File

@ -1,13 +1,13 @@
import fs from "fs";
import path from "path";
import { getJanDataFolderPath } from "@janhq/core/node";
import fs from 'fs'
import path from 'path'
import { getJanDataFolderPath } from '@janhq/core/node'
// Sec: Do not send engine settings over requests
// Read it manually instead
export const readEmbeddingEngine = (engineName: string) => {
const engineSettings = fs.readFileSync(
path.join(getJanDataFolderPath(), "engines", `${engineName}.json`),
"utf-8",
);
return JSON.parse(engineSettings);
};
path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`),
'utf-8'
)
return JSON.parse(engineSettings)
}

28
extensions/assistant-extension/src/node/index.ts
View File

@ -1,39 +1,39 @@
import { getJanDataFolderPath, normalizeFilePath } from "@janhq/core/node";
import { retrieval } from "./tools/retrieval";
import path from "path";
import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node'
import { retrieval } from './tools/retrieval'
import path from 'path'
export function toolRetrievalUpdateTextSplitter(
chunkSize: number,
chunkOverlap: number
) {
retrieval.updateTextSplitter(chunkSize, chunkOverlap);
retrieval.updateTextSplitter(chunkSize, chunkOverlap)
}
export async function toolRetrievalIngestNewDocument(
file: string,
engine: string
) {
const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file));
const threadPath = path.dirname(filePath.replace("files", ""));
retrieval.updateEmbeddingEngine(engine);
const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file))
const threadPath = path.dirname(filePath.replace('files', ''))
retrieval.updateEmbeddingEngine(engine)
return retrieval
.ingestAgentKnowledge(filePath, `${threadPath}/memory`)
.catch((err) => {
console.error(err);
});
console.error(err)
})
}
export async function toolRetrievalLoadThreadMemory(threadId: string) {
return retrieval
.loadRetrievalAgent(
path.join(getJanDataFolderPath(), "threads", threadId, "memory")
path.join(getJanDataFolderPath(), 'threads', threadId, 'memory')
)
.catch((err) => {
console.error(err);
});
console.error(err)
})
}
export async function toolRetrievalQueryResult(query: string) {
return retrieval.generateResult(query).catch((err) => {
console.error(err);
});
console.error(err)
})
}

80
extensions/assistant-extension/src/node/tools/retrieval/index.ts
View File

@ -1,80 +1,80 @@
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { formatDocumentsAsString } from "langchain/util/document";
import { PDFLoader } from "langchain/document_loaders/fs/pdf";
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
import { formatDocumentsAsString } from 'langchain/util/document'
import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
import { HNSWLib } from "langchain/vectorstores/hnswlib";
import { HNSWLib } from 'langchain/vectorstores/hnswlib'
import { OpenAIEmbeddings } from "langchain/embeddings/openai";
import { readEmbeddingEngine } from "../../engine";
import { OpenAIEmbeddings } from 'langchain/embeddings/openai'
import { readEmbeddingEngine } from '../../engine'
export class Retrieval {
public chunkSize: number = 100;
public chunkOverlap?: number = 0;
private retriever: any;
public chunkSize: number = 100
public chunkOverlap?: number = 0
private retriever: any
private embeddingModel?: OpenAIEmbeddings = undefined;
private textSplitter?: RecursiveCharacterTextSplitter;
private embeddingModel?: OpenAIEmbeddings = undefined
private textSplitter?: RecursiveCharacterTextSplitter
constructor(chunkSize: number = 4000, chunkOverlap: number = 200) {
this.updateTextSplitter(chunkSize, chunkOverlap);
this.updateTextSplitter(chunkSize, chunkOverlap)
}
public updateTextSplitter(chunkSize: number, chunkOverlap: number): void {
this.chunkSize = chunkSize;
this.chunkOverlap = chunkOverlap;
this.chunkSize = chunkSize
this.chunkOverlap = chunkOverlap
this.textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: chunkSize,
chunkOverlap: chunkOverlap,
});
})
}
public updateEmbeddingEngine(engine: string): void {
// Engine settings are not compatible with the current embedding model params
// Switch case manually for now
const settings = readEmbeddingEngine(engine);
if (engine === "nitro") {
const settings = readEmbeddingEngine(engine)
if (engine === 'nitro') {
this.embeddingModel = new OpenAIEmbeddings(
{ openAIApiKey: "nitro-embedding" },
{ openAIApiKey: 'nitro-embedding' },
// TODO: Raw settings
{ basePath: "http://127.0.0.1:3928/v1" },
);
{ basePath: 'http://127.0.0.1:3928/v1' }
)
} else {
// Fallback to OpenAI Settings
this.embeddingModel = new OpenAIEmbeddings({
openAIApiKey: settings.api_key,
});
})
}
}
public ingestAgentKnowledge = async (
filePath: string,
memoryPath: string,
memoryPath: string
): Promise<any> => {
const loader = new PDFLoader(filePath, {
splitPages: true,
});
if (!this.embeddingModel) return Promise.reject();
const doc = await loader.load();
const docs = await this.textSplitter!.splitDocuments(doc);
const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel);
return vectorStore.save(memoryPath);
};
})
if (!this.embeddingModel) return Promise.reject()
const doc = await loader.load()
const docs = await this.textSplitter!.splitDocuments(doc)
const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel)
return vectorStore.save(memoryPath)
}
public loadRetrievalAgent = async (memoryPath: string): Promise<void> => {
if (!this.embeddingModel) return Promise.reject();
const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel);
this.retriever = vectorStore.asRetriever(2);
return Promise.resolve();
};
if (!this.embeddingModel) return Promise.reject()
const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel)
this.retriever = vectorStore.asRetriever(2)
return Promise.resolve()
}
public generateResult = async (query: string): Promise<string> => {
if (!this.retriever) {
return Promise.resolve(" ");
return Promise.resolve(' ')
}
const relevantDocs = await this.retriever.getRelevantDocuments(query);
const serializedDoc = formatDocumentsAsString(relevantDocs);
return Promise.resolve(serializedDoc);
};
const relevantDocs = await this.retriever.getRelevantDocuments(query)
const serializedDoc = formatDocumentsAsString(relevantDocs)
return Promise.resolve(serializedDoc)
}
}
export const retrieval = new Retrieval();
export const retrieval = new Retrieval()

4
extensions/assistant-extension/tsconfig.json
View File

@ -14,7 +14,7 @@
"outDir": "dist",
"importHelpers": true,
"typeRoots": ["node_modules/@types"],
"skipLibCheck": true,
"skipLibCheck": true
},
"include": ["src"],
"include": ["src"]
}

20
extensions/conversational-extension/webpack.config.js
View File

@ -1,27 +1,27 @@
const path = require("path");
const webpack = require("webpack");
const path = require('path')
const webpack = require('webpack')
module.exports = {
experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file
mode: "production",
entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: 'production',
module: {
rules: [
{
test: /\.tsx?$/,
use: "ts-loader",
use: 'ts-loader',
exclude: /node_modules/,
},
],
},
output: {
filename: "index.js", // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"),
library: { type: "module" }, // Specify ESM output format
filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, 'dist'),
library: { type: 'module' }, // Specify ESM output format
},
plugins: [new webpack.DefinePlugin({})],
resolve: {
extensions: [".ts", ".js"],
extensions: ['.ts', '.js'],
fallback: {
path: require.resolve('path-browserify'),
},
@ -31,4 +31,4 @@ module.exports = {
minimize: false,
},
// Add loaders and other configuration as needed for your project
};
}

5
extensions/inference-nitro-extension/README.md
View File

@ -64,10 +64,10 @@ There are a few things to keep in mind when writing your plugin code:
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { core } from "@janhq/core";
import { core } from '@janhq/core'
function onStart(): Promise<any> {
return core.invokePluginFunc(MODULE_PATH, "run", 0);
return core.invokePluginFunc(MODULE_PATH, 'run', 0)
}
```
@ -75,4 +75,3 @@ There are a few things to keep in mind when writing your plugin code:
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your plugin!

38
extensions/inference-nitro-extension/rollup.config.ts
View File

@ -1,34 +1,34 @@
import resolve from "@rollup/plugin-node-resolve";
import commonjs from "@rollup/plugin-commonjs";
import sourceMaps from "rollup-plugin-sourcemaps";
import typescript from "rollup-plugin-typescript2";
import json from "@rollup/plugin-json";
import replace from "@rollup/plugin-replace";
const packageJson = require("./package.json");
import resolve from '@rollup/plugin-node-resolve'
import commonjs from '@rollup/plugin-commonjs'
import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from 'rollup-plugin-typescript2'
import json from '@rollup/plugin-json'
import replace from '@rollup/plugin-replace'
const packageJson = require('./package.json')
const pkg = require("./package.json");
const pkg = require('./package.json')
export default [
{
input: `src/index.ts`,
output: [{ file: pkg.main, format: "es", sourcemap: true }],
output: [{ file: pkg.main, format: 'es', sourcemap: true }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [],
watch: {
include: "src/**",
include: 'src/**',
},
plugins: [
replace({
NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
INFERENCE_URL: JSON.stringify(
process.env.INFERENCE_URL ||
"http://127.0.0.1:3928/inferences/llamacpp/chat_completion"
'http://127.0.0.1:3928/inferences/llamacpp/chat_completion'
),
TROUBLESHOOTING_URL: JSON.stringify(
"https://jan.ai/guides/troubleshooting"
'https://jan.ai/guides/troubleshooting'
),
JAN_SERVER_INFERENCE_URL: JSON.stringify(
"http://localhost:1337/v1/chat/completions"
'http://localhost:1337/v1/chat/completions'
),
}),
// Allow json resolution
@ -42,7 +42,7 @@ export default [
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({
extensions: [".js", ".ts", ".svelte"],
extensions: ['.js', '.ts', '.svelte'],
}),
// Resolve source maps to the original source
@ -52,12 +52,12 @@ export default [
{
input: `src/node/index.ts`,
output: [
{ file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true },
{ file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: ["@janhq/core/node"],
external: ['@janhq/core/node'],
watch: {
include: "src/node/**",
include: 'src/node/**',
},
plugins: [
// Allow json resolution
@ -70,11 +70,11 @@ export default [
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve({
extensions: [".ts", ".js", ".json"],
extensions: ['.ts', '.js', '.json'],
}),
// Resolve source maps to the original source
sourceMaps(),
],
},
];
]

12
extensions/inference-nitro-extension/src/@types/global.d.ts vendored
View File

@ -1,13 +1,13 @@
declare const NODE: string;
declare const INFERENCE_URL: string;
declare const TROUBLESHOOTING_URL: string;
declare const JAN_SERVER_INFERENCE_URL: string;
declare const NODE: string
declare const INFERENCE_URL: string
declare const TROUBLESHOOTING_URL: string
declare const JAN_SERVER_INFERENCE_URL: string
/**
* The response from the initModel function.
* @property error - An error message if the model fails to load.
*/
interface ModelOperationResponse {
error?: any;
modelFile?: string;
error?: any
modelFile?: string
}

56
extensions/inference-nitro-extension/src/helpers/sse.ts
View File

@ -1,5 +1,5 @@
import { Model } from "@janhq/core";
import { Observable } from "rxjs";
import { Model } from '@janhq/core'
import { Observable } from 'rxjs'
/**
* Sends a request to the inference server to generate a response based on the recent messages.
* @param recentMessages - An array of recent messages to use as context for the inference.
@ -17,50 +17,50 @@ export function requestInference(
model: model.id,
stream: true,
...model.parameters,
});
})
fetch(inferenceUrl, {
method: "POST",
method: 'POST',
headers: {
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*",
Accept: model.parameters.stream
? "text/event-stream"
: "application/json",
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Accept': model.parameters.stream
? 'text/event-stream'
: 'application/json',
},
body: requestBody,
signal: controller?.signal,
})
.then(async (response) => {
if (model.parameters.stream === false) {
const data = await response.json();
subscriber.next(data.choices[0]?.message?.content ?? "");
const data = await response.json()
subscriber.next(data.choices[0]?.message?.content ?? '')
} else {
const stream = response.body;
const decoder = new TextDecoder("utf-8");
const reader = stream?.getReader();
let content = "";
const stream = response.body
const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader()
let content = ''
while (true && reader) {
const { done, value } = await reader.read();
const { done, value } = await reader.read()
if (done) {
break;
break
}
const text = decoder.decode(value);
const lines = text.trim().split("\n");
const text = decoder.decode(value)
const lines = text.trim().split('\n')
for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) {
const data = JSON.parse(line.replace("data: ", ""));
content += data.choices[0]?.delta?.content ?? "";
if (content.startsWith("assistant: ")) {
content = content.replace("assistant: ", "");
if (line.startsWith('data: ') && !line.includes('data: [DONE]')) {
const data = JSON.parse(line.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ''
if (content.startsWith('assistant: ')) {
content = content.replace('assistant: ', '')
}
subscriber.next(content);
subscriber.next(content)
}
}
}
}
subscriber.complete();
subscriber.complete()
})
.catch((err) => subscriber.error(err));
});
.catch((err) => subscriber.error(err))
})
}

175
extensions/inference-nitro-extension/src/index.ts
View File

@ -27,9 +27,9 @@ import {
InferenceEvent,
ModelSettingParams,
getJanDataFolderPath,
} from "@janhq/core";
import { requestInference } from "./helpers/sse";
import { ulid } from "ulid";
} from '@janhq/core'
import { requestInference } from './helpers/sse'
import { ulid } from 'ulid'
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
@ -37,16 +37,16 @@ import { ulid } from "ulid";
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceNitroExtension extends InferenceExtension {
private static readonly _homeDir = "file://engines";
private static readonly _settingsDir = "file://settings";
private static readonly _engineMetadataFileName = "nitro.json";
private static readonly _homeDir = 'file://engines'
private static readonly _settingsDir = 'file://settings'
private static readonly _engineMetadataFileName = 'nitro.json'
/**
* Checking the health for Nitro's process each 5 secs.
*/
private static readonly _intervalHealthCheck = 5 * 1000;
private static readonly _intervalHealthCheck = 5 * 1000
private _currentModel: Model | undefined;
private _currentModel: Model | undefined
private _engineSettings: ModelSettingParams = {
ctx_len: 2048,
@ -54,23 +54,22 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
cpu_threads: 1,
cont_batching: false,
embedding: true,
};
}
controller = new AbortController();
isCancelled = false;
controller = new AbortController()
isCancelled = false
/**
* The interval id for the health check. Used to stop the health check.
*/
private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined =
undefined;
private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = undefined
/**
* Tracking the current state of nitro process.
*/
private nitroProcessInfo: any = undefined;
private nitroProcessInfo: any = undefined
private inferenceUrl = "";
private inferenceUrl = ''
/**
* Subscribes to events emitted by the @janhq/core package.
@ -78,44 +77,40 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
async onLoad() {
if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) {
try {
await fs.mkdirSync(JanInferenceNitroExtension._homeDir);
await fs.mkdirSync(JanInferenceNitroExtension._homeDir)
} catch (e) {
console.debug(e);
console.debug(e)
}
}
// init inference url
// @ts-ignore
const electronApi = window?.electronAPI;
this.inferenceUrl = INFERENCE_URL;
const electronApi = window?.electronAPI
this.inferenceUrl = INFERENCE_URL
if (!electronApi) {
this.inferenceUrl = JAN_SERVER_INFERENCE_URL;
this.inferenceUrl = JAN_SERVER_INFERENCE_URL
}
console.debug("Inference url: ", this.inferenceUrl);
console.debug('Inference url: ', this.inferenceUrl)
if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir)))
await fs.mkdirSync(JanInferenceNitroExtension._settingsDir);
this.writeDefaultEngineSettings();
await fs.mkdirSync(JanInferenceNitroExtension._settingsDir)
this.writeDefaultEngineSettings()
// Events subscription
events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.onMessageRequest(data)
);
)
events.on(ModelEvent.OnModelInit, (model: Model) =>
this.onModelInit(model)
);
events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model))
events.on(ModelEvent.OnModelStop, (model: Model) =>
this.onModelStop(model)
);
events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model))
events.on(InferenceEvent.OnInferenceStopped, () =>
this.onInferenceStopped()
);
)
// Attempt to fetch nvidia info
await executeOnMain(NODE, "updateNvidiaInfo", {});
await executeOnMain(NODE, 'updateNvidiaInfo', {})
}
/**
@ -128,59 +123,59 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
const engineFile = await joinPath([
JanInferenceNitroExtension._homeDir,
JanInferenceNitroExtension._engineMetadataFileName,
]);
])
if (await fs.existsSync(engineFile)) {
const engine = await fs.readFileSync(engineFile, "utf-8");
const engine = await fs.readFileSync(engineFile, 'utf-8')
this._engineSettings =
typeof engine === "object" ? engine : JSON.parse(engine);
typeof engine === 'object' ? engine : JSON.parse(engine)
} else {
await fs.writeFileSync(
engineFile,
JSON.stringify(this._engineSettings, null, 2)
);
)
}
} catch (err) {
console.error(err);
console.error(err)
}
}
private async onModelInit(model: Model) {
if (model.engine !== InferenceEngine.nitro) return;
if (model.engine !== InferenceEngine.nitro) return
const modelFolder = await joinPath([
await getJanDataFolderPath(),
"models",
'models',
model.id,
]);
this._currentModel = model;
const nitroInitResult = await executeOnMain(NODE, "runModel", {
])
this._currentModel = model
const nitroInitResult = await executeOnMain(NODE, 'runModel', {
modelFolder,
model,
});
})
if (nitroInitResult?.error) {
events.emit(ModelEvent.OnModelFail, model);
return;
events.emit(ModelEvent.OnModelFail, model)
return
}
events.emit(ModelEvent.OnModelReady, model);
events.emit(ModelEvent.OnModelReady, model)
this.getNitroProcesHealthIntervalId = setInterval(
() => this.periodicallyGetNitroHealth(),
JanInferenceNitroExtension._intervalHealthCheck
);
)
}
private async onModelStop(model: Model) {
if (model.engine !== "nitro") return;
if (model.engine !== 'nitro') return
await executeOnMain(NODE, "stopModel");
events.emit(ModelEvent.OnModelStopped, {});
await executeOnMain(NODE, 'stopModel')
events.emit(ModelEvent.OnModelStopped, {})
// stop the periocally health check
if (this.getNitroProcesHealthIntervalId) {
clearInterval(this.getNitroProcesHealthIntervalId);
this.getNitroProcesHealthIntervalId = undefined;
clearInterval(this.getNitroProcesHealthIntervalId)
this.getNitroProcesHealthIntervalId = undefined
}
}
@ -188,19 +183,19 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
* Periodically check for nitro process's health.
*/
private async periodicallyGetNitroHealth(): Promise<void> {
const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo");
const health = await executeOnMain(NODE, 'getCurrentNitroProcessInfo')
const isRunning = this.nitroProcessInfo?.isRunning ?? false;
const isRunning = this.nitroProcessInfo?.isRunning ?? false
if (isRunning && health.isRunning === false) {
console.debug("Nitro process is stopped");
events.emit(ModelEvent.OnModelStopped, {});
console.debug('Nitro process is stopped')
events.emit(ModelEvent.OnModelStopped, {})
}
this.nitroProcessInfo = health;
this.nitroProcessInfo = health
}
private async onInferenceStopped() {
this.isCancelled = true;
this.controller?.abort();
this.isCancelled = true
this.controller?.abort()
}
/**
@ -209,20 +204,20 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
* @returns {Promise<any>} A promise that resolves with the inference response.
*/
async inference(data: MessageRequest): Promise<ThreadMessage> {
const timestamp = Date.now();
const timestamp = Date.now()
const message: ThreadMessage = {
thread_id: data.threadId,
created: timestamp,
updated: timestamp,
status: MessageStatus.Ready,
id: "",
id: '',
role: ChatCompletionRole.Assistant,
object: "thread.message",
object: 'thread.message',
content: [],
};
}
return new Promise(async (resolve, reject) => {
if (!this._currentModel) return Promise.reject("No model loaded");
if (!this._currentModel) return Promise.reject('No model loaded')
requestInference(
this.inferenceUrl,
@ -231,13 +226,13 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
).subscribe({
next: (_content: any) => {},
complete: async () => {
resolve(message);
resolve(message)
},
error: async (err: any) => {
reject(err);
reject(err)
},
});
});
})
})
}
/**
@ -248,10 +243,10 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
*/
private async onMessageRequest(data: MessageRequest) {
if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) {
return;
return
}
const timestamp = Date.now();
const timestamp = Date.now()
const message: ThreadMessage = {
id: ulid(),
thread_id: data.threadId,
@ -262,21 +257,21 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
status: MessageStatus.Pending,
created: timestamp,
updated: timestamp,
object: "thread.message",
};
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message);
object: 'thread.message',
}
this.isCancelled = false;
this.controller = new AbortController();
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message)
}
this.isCancelled = false
this.controller = new AbortController()
// @ts-ignore
const model: Model = {
...(this._currentModel || {}),
...(data.model || {}),
};
}
requestInference(
this.inferenceUrl,
data.messages ?? [],
@ -290,26 +285,26 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
value: content.trim(),
annotations: [],
},
};
message.content = [messageContent];
events.emit(MessageEvent.OnMessageUpdate, message);
}
message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message)
},
complete: async () => {
message.status = message.content.length
? MessageStatus.Ready
: MessageStatus.Error;
events.emit(MessageEvent.OnMessageUpdate, message);
: MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
},
error: async (err: any) => {
if (this.isCancelled || message.content.length) {
message.status = MessageStatus.Stopped;
events.emit(MessageEvent.OnMessageUpdate, message);
return;
message.status = MessageStatus.Stopped
events.emit(MessageEvent.OnMessageUpdate, message)
return
}
message.status = MessageStatus.Error;
events.emit(MessageEvent.OnMessageUpdate, message);
log(`[APP]::Error: ${err.message}`);
message.status = MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
log(`[APP]::Error: ${err.message}`)
},
});
})
}
}

60
extensions/inference-nitro-extension/src/node/execute.ts
View File

@ -1,65 +1,65 @@
import { readFileSync } from "fs";
import * as path from "path";
import { NVIDIA_INFO_FILE } from "./nvidia";
import { readFileSync } from 'fs'
import * as path from 'path'
import { NVIDIA_INFO_FILE } from './nvidia'
export interface NitroExecutableOptions {
executablePath: string;
cudaVisibleDevices: string;
executablePath: string
cudaVisibleDevices: string
}
/**
* Find which executable file to run based on the current platform.
* @returns The name of the executable file to run.
*/
export const executableNitroFile = (): NitroExecutableOptions => {
let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default
let cudaVisibleDevices = "";
let binaryName = "nitro";
let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default
let cudaVisibleDevices = ''
let binaryName = 'nitro'
/**
* The binary folder is different for each platform.
*/
if (process.platform === "win32") {
if (process.platform === 'win32') {
/**
* For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0
*/
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
if (nvidiaInfo["run_mode"] === "cpu") {
binaryFolder = path.join(binaryFolder, "win-cpu");
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (nvidiaInfo['run_mode'] === 'cpu') {
binaryFolder = path.join(binaryFolder, 'win-cpu')
} else {
if (nvidiaInfo["cuda"].version === "11") {
binaryFolder = path.join(binaryFolder, "win-cuda-11-7");
if (nvidiaInfo['cuda'].version === '11') {
binaryFolder = path.join(binaryFolder, 'win-cuda-11-7')
} else {
binaryFolder = path.join(binaryFolder, "win-cuda-12-0");
binaryFolder = path.join(binaryFolder, 'win-cuda-12-0')
}
cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(",");
cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
}
binaryName = "nitro.exe";
} else if (process.platform === "darwin") {
binaryName = 'nitro.exe'
} else if (process.platform === 'darwin') {
/**
* For MacOS: mac-arm64 (Silicon), mac-x64 (InteL)
*/
if (process.arch === "arm64") {
binaryFolder = path.join(binaryFolder, "mac-arm64");
if (process.arch === 'arm64') {
binaryFolder = path.join(binaryFolder, 'mac-arm64')
} else {
binaryFolder = path.join(binaryFolder, "mac-x64");
binaryFolder = path.join(binaryFolder, 'mac-x64')
}
} else {
/**
* For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0
*/
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
if (nvidiaInfo["run_mode"] === "cpu") {
binaryFolder = path.join(binaryFolder, "linux-cpu");
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (nvidiaInfo['run_mode'] === 'cpu') {
binaryFolder = path.join(binaryFolder, 'linux-cpu')
} else {
if (nvidiaInfo["cuda"].version === "11") {
binaryFolder = path.join(binaryFolder, "linux-cuda-11-7");
if (nvidiaInfo['cuda'].version === '11') {
binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7')
} else {
binaryFolder = path.join(binaryFolder, "linux-cuda-12-0");
binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0')
}
cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(",");
cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
}
}
return {
executablePath: path.join(binaryFolder, binaryName),
cudaVisibleDevices,
};
};
}
}

160
extensions/inference-nitro-extension/src/node/nvidia.ts
View File

@ -1,47 +1,47 @@
import { writeFileSync, existsSync, readFileSync } from "fs";
import { exec } from "child_process";
import path from "path";
import { getJanDataFolderPath } from "@janhq/core/node";
import { writeFileSync, existsSync, readFileSync } from 'fs'
import { exec } from 'child_process'
import path from 'path'
import { getJanDataFolderPath } from '@janhq/core/node'
/**
* Default GPU settings
**/
const DEFALT_SETTINGS = {
notify: true,
run_mode: "cpu",
run_mode: 'cpu',
nvidia_driver: {
exist: false,
version: "",
version: '',
},
cuda: {
exist: false,
version: "",
version: '',
},
gpus: [],
gpu_highest_vram: "",
gpu_highest_vram: '',
gpus_in_use: [],
is_initial: true,
};
}
/**
* Path to the settings file
**/
export const NVIDIA_INFO_FILE = path.join(
getJanDataFolderPath(),
"settings",
"settings.json"
);
'settings',
'settings.json'
)
/**
* Current nitro process
*/
let nitroProcessInfo: NitroProcessInfo | undefined = undefined;
let nitroProcessInfo: NitroProcessInfo | undefined = undefined
/**
* Nitro process info
*/
export interface NitroProcessInfo {
isRunning: boolean;
isRunning: boolean
}
/**
@ -49,16 +49,16 @@ export interface NitroProcessInfo {
* Will be called when the extension is loaded to turn on GPU acceleration if supported
*/
export async function updateNvidiaInfo() {
if (process.platform !== "darwin") {
let data;
if (process.platform !== 'darwin') {
let data
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
} catch (error) {
data = DEFALT_SETTINGS;
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
data = DEFALT_SETTINGS
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
}
updateNvidiaDriverInfo();
updateGpuInfo();
updateNvidiaDriverInfo()
updateGpuInfo()
}
}
@ -68,31 +68,31 @@ export async function updateNvidiaInfo() {
export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => {
nitroProcessInfo = {
isRunning: subprocess != null,
};
return nitroProcessInfo;
};
}
return nitroProcessInfo
}
/**
* Validate nvidia and cuda for linux and windows
*/
export async function updateNvidiaDriverInfo(): Promise<void> {
exec(
"nvidia-smi --query-gpu=driver_version --format=csv,noheader",
'nvidia-smi --query-gpu=driver_version --format=csv,noheader',
(error, stdout) => {
let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (!error) {
const firstLine = stdout.split("\n")[0].trim();
data["nvidia_driver"].exist = true;
data["nvidia_driver"].version = firstLine;
const firstLine = stdout.split('\n')[0].trim()
data['nvidia_driver'].exist = true
data['nvidia_driver'].version = firstLine
} else {
data["nvidia_driver"].exist = false;
data['nvidia_driver'].exist = false
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
Promise.resolve();
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
Promise.resolve()
}
);
)
}
/**
@ -102,54 +102,56 @@ export function checkFileExistenceInPaths(
file: string,
paths: string[]
): boolean {
return paths.some((p) => existsSync(path.join(p, file)));
return paths.some((p) => existsSync(path.join(p, file)))
}
/**
* Validate cuda for linux and windows
*/
export function updateCudaExistence(data: Record<string, any> = DEFALT_SETTINGS): Record<string, any> {
let filesCuda12: string[];
let filesCuda11: string[];
let paths: string[];
let cudaVersion: string = "";
export function updateCudaExistence(
data: Record<string, any> = DEFALT_SETTINGS
): Record<string, any> {
let filesCuda12: string[]
let filesCuda11: string[]
let paths: string[]
let cudaVersion: string = ''
if (process.platform === "win32") {
filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"];
filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"];
paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [];
if (process.platform === 'win32') {
filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll']
filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll']
paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []
} else {
filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"];
filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"];
filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12']
filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11']
paths = process.env.LD_LIBRARY_PATH
? process.env.LD_LIBRARY_PATH.split(path.delimiter)
: [];
paths.push("/usr/lib/x86_64-linux-gnu/");
: []
paths.push('/usr/lib/x86_64-linux-gnu/')
}
let cudaExists = filesCuda12.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
)
if (!cudaExists) {
cudaExists = filesCuda11.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
)
if (cudaExists) {
cudaVersion = "11";
cudaVersion = '11'
}
} else {
cudaVersion = "12";
cudaVersion = '12'
}
data["cuda"].exist = cudaExists;
data["cuda"].version = cudaVersion;
console.log(data["is_initial"], data["gpus_in_use"]);
if (cudaExists && data["is_initial"] && data["gpus_in_use"].length > 0) {
data.run_mode = "gpu";
data['cuda'].exist = cudaExists
data['cuda'].version = cudaVersion
console.log(data['is_initial'], data['gpus_in_use'])
if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) {
data.run_mode = 'gpu'
}
data.is_initial = false;
return data;
data.is_initial = false
return data
}
/**
@ -157,41 +159,41 @@ export function updateCudaExistence(data: Record<string, any> = DEFALT_SETTINGS)
*/
export async function updateGpuInfo(): Promise<void> {
exec(
"nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits",
'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits',
(error, stdout) => {
let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
if (!error) {
// Get GPU info and gpu has higher memory first
let highestVram = 0;
let highestVramId = "0";
let highestVram = 0
let highestVramId = '0'
let gpus = stdout
.trim()
.split("\n")
.split('\n')
.map((line) => {
let [id, vram, name] = line.split(", ");
vram = vram.replace(/\r/g, "");
let [id, vram, name] = line.split(', ')
vram = vram.replace(/\r/g, '')
if (parseFloat(vram) > highestVram) {
highestVram = parseFloat(vram);
highestVramId = id;
highestVram = parseFloat(vram)
highestVramId = id
}
return { id, vram, name };
});
return { id, vram, name }
})
data.gpus = gpus;
data.gpu_highest_vram = highestVramId;
data.gpus = gpus
data.gpu_highest_vram = highestVramId
} else {
data.gpus = [];
data.gpu_highest_vram = "";
data.gpus = []
data.gpu_highest_vram = ''
}
if (!data["gpus_in_use"] || data["gpus_in_use"].length === 0) {
data.gpus_in_use = [data["gpu_highest_vram"]];
if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) {
data.gpus_in_use = [data['gpu_highest_vram']]
}
data = updateCudaExistence(data);
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
Promise.resolve();
data = updateCudaExistence(data)
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
Promise.resolve()
}
);
)
}

38
extensions/inference-openai-extension/src/@types/global.d.ts vendored
View File

@ -1,26 +1,26 @@
declare const MODULE: string;
declare const OPENAI_DOMAIN: string;
declare const MODULE: string
declare const OPENAI_DOMAIN: string
declare interface EngineSettings {
full_url?: string;
api_key?: string;
full_url?: string
api_key?: string
}
enum OpenAIChatCompletionModelName {
"gpt-3.5-turbo-instruct" = "gpt-3.5-turbo-instruct",
"gpt-3.5-turbo-instruct-0914" = "gpt-3.5-turbo-instruct-0914",
"gpt-4-1106-preview" = "gpt-4-1106-preview",
"gpt-3.5-turbo-0613" = "gpt-3.5-turbo-0613",
"gpt-3.5-turbo-0301" = "gpt-3.5-turbo-0301",
"gpt-3.5-turbo" = "gpt-3.5-turbo",
"gpt-3.5-turbo-16k-0613" = "gpt-3.5-turbo-16k-0613",
"gpt-3.5-turbo-1106" = "gpt-3.5-turbo-1106",
"gpt-4-vision-preview" = "gpt-4-vision-preview",
"gpt-4" = "gpt-4",
"gpt-4-0314" = "gpt-4-0314",
"gpt-4-0613" = "gpt-4-0613",
'gpt-3.5-turbo-instruct' = 'gpt-3.5-turbo-instruct',
'gpt-3.5-turbo-instruct-0914' = 'gpt-3.5-turbo-instruct-0914',
'gpt-4-1106-preview' = 'gpt-4-1106-preview',
'gpt-3.5-turbo-0613' = 'gpt-3.5-turbo-0613',
'gpt-3.5-turbo-0301' = 'gpt-3.5-turbo-0301',
'gpt-3.5-turbo' = 'gpt-3.5-turbo',
'gpt-3.5-turbo-16k-0613' = 'gpt-3.5-turbo-16k-0613',
'gpt-3.5-turbo-1106' = 'gpt-3.5-turbo-1106',
'gpt-4-vision-preview' = 'gpt-4-vision-preview',
'gpt-4' = 'gpt-4',
'gpt-4-0314' = 'gpt-4-0314',
'gpt-4-0613' = 'gpt-4-0613',
}
declare type OpenAIModel = Omit<Model, "id"> & {
id: OpenAIChatCompletionModelName;
};
declare type OpenAIModel = Omit<Model, 'id'> & {
id: OpenAIChatCompletionModelName
}

70
extensions/inference-openai-extension/src/helpers/sse.ts
View File

@ -1,4 +1,4 @@
import { Observable } from "rxjs";
import { Observable } from 'rxjs'
/**
* Sends a request to the inference server to generate a response based on the recent messages.
@ -14,26 +14,26 @@ export function requestInference(
controller?: AbortController
): Observable<string> {
return new Observable((subscriber) => {
let model_id: string = model.id;
let model_id: string = model.id
if (engine.full_url.includes(OPENAI_DOMAIN)) {
model_id = engine.full_url.split("/")[5];
model_id = engine.full_url.split('/')[5]
}
const requestBody = JSON.stringify({
messages: recentMessages,
stream: true,
model: model_id,
...model.parameters,
});
})
fetch(`${engine.full_url}`, {
method: "POST",
method: 'POST',
headers: {
"Content-Type": "application/json",
Accept: model.parameters.stream
? "text/event-stream"
: "application/json",
"Access-Control-Allow-Origin": "*",
Authorization: `Bearer ${engine.api_key}`,
"api-key": `${engine.api_key}`,
'Content-Type': 'application/json',
'Accept': model.parameters.stream
? 'text/event-stream'
: 'application/json',
'Access-Control-Allow-Origin': '*',
'Authorization': `Bearer ${engine.api_key}`,
'api-key': `${engine.api_key}`,
},
body: requestBody,
signal: controller?.signal,
@ -41,41 +41,41 @@ export function requestInference(
.then(async (response) => {
if (!response.ok) {
subscriber.next(
(await response.json()).error?.message ?? "Error occurred."
);
subscriber.complete();
return;
(await response.json()).error?.message ?? 'Error occurred.'
)
subscriber.complete()
return
}
if (model.parameters.stream === false) {
const data = await response.json();
subscriber.next(data.choices[0]?.message?.content ?? "");
const data = await response.json()
subscriber.next(data.choices[0]?.message?.content ?? '')
} else {
const stream = response.body;
const decoder = new TextDecoder("utf-8");
const reader = stream?.getReader();
let content = "";
const stream = response.body
const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader()
let content = ''
while (true && reader) {
const { done, value } = await reader.read();
const { done, value } = await reader.read()
if (done) {
break;
break
}
const text = decoder.decode(value);
const lines = text.trim().split("\n");
const text = decoder.decode(value)
const lines = text.trim().split('\n')
for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) {
const data = JSON.parse(line.replace("data: ", ""));
content += data.choices[0]?.delta?.content ?? "";
if (content.startsWith("assistant: ")) {
content = content.replace("assistant: ", "");
if (line.startsWith('data: ') && !line.includes('data: [DONE]')) {
const data = JSON.parse(line.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ''
if (content.startsWith('assistant: ')) {
content = content.replace('assistant: ', '')
}
subscriber.next(content);
subscriber.next(content)
}
}
}
}
subscriber.complete();
subscriber.complete()
})
.catch((err) => subscriber.error(err));
});
.catch((err) => subscriber.error(err))
})
}

138
extensions/inference-openai-extension/src/index.ts
View File

@ -23,10 +23,10 @@ import {
InferenceEvent,
AppConfigurationEventName,
joinPath,
} from "@janhq/core";
import { requestInference } from "./helpers/sse";
import { ulid } from "ulid";
import { join } from "path";
} from '@janhq/core'
import { requestInference } from './helpers/sse'
import { ulid } from 'ulid'
import { join } from 'path'
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
@ -34,18 +34,18 @@ import { join } from "path";
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceOpenAIExtension extends BaseExtension {
private static readonly _engineDir = "file://engines";
private static readonly _engineMetadataFileName = "openai.json";
private static readonly _engineDir = 'file://engines'
private static readonly _engineMetadataFileName = 'openai.json'
private static _currentModel: OpenAIModel;
private static _currentModel: OpenAIModel
private static _engineSettings: EngineSettings = {
full_url: "https://api.openai.com/v1/chat/completions",
api_key: "sk-<your key here>",
};
full_url: 'https://api.openai.com/v1/chat/completions',
api_key: 'sk-<your key here>',
}
controller = new AbortController();
isCancelled = false;
controller = new AbortController()
isCancelled = false
/**
* Subscribes to events emitted by the @janhq/core package.
@ -54,40 +54,40 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) {
await fs
.mkdirSync(JanInferenceOpenAIExtension._engineDir)
.catch((err) => console.debug(err));
.catch((err) => console.debug(err))
}
JanInferenceOpenAIExtension.writeDefaultEngineSettings();
JanInferenceOpenAIExtension.writeDefaultEngineSettings()
// Events subscription
events.on(MessageEvent.OnMessageSent, (data) =>
JanInferenceOpenAIExtension.handleMessageRequest(data, this),
);
JanInferenceOpenAIExtension.handleMessageRequest(data, this)
)
events.on(ModelEvent.OnModelInit, (model: OpenAIModel) => {
JanInferenceOpenAIExtension.handleModelInit(model);
});
JanInferenceOpenAIExtension.handleModelInit(model)
})
events.on(ModelEvent.OnModelStop, (model: OpenAIModel) => {
JanInferenceOpenAIExtension.handleModelStop(model);
});
JanInferenceOpenAIExtension.handleModelStop(model)
})
events.on(InferenceEvent.OnInferenceStopped, () => {
JanInferenceOpenAIExtension.handleInferenceStopped(this);
});
JanInferenceOpenAIExtension.handleInferenceStopped(this)
})
const settingsFilePath = await joinPath([
JanInferenceOpenAIExtension._engineDir,
JanInferenceOpenAIExtension._engineMetadataFileName,
]);
])
events.on(
AppConfigurationEventName.OnConfigurationUpdate,
(settingsKey: string) => {
// Update settings on changes
if (settingsKey === settingsFilePath)
JanInferenceOpenAIExtension.writeDefaultEngineSettings();
},
);
JanInferenceOpenAIExtension.writeDefaultEngineSettings()
}
)
}
/**
@ -99,45 +99,45 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
try {
const engineFile = join(
JanInferenceOpenAIExtension._engineDir,
JanInferenceOpenAIExtension._engineMetadataFileName,
);
JanInferenceOpenAIExtension._engineMetadataFileName
)
if (await fs.existsSync(engineFile)) {
const engine = await fs.readFileSync(engineFile, "utf-8");
const engine = await fs.readFileSync(engineFile, 'utf-8')
JanInferenceOpenAIExtension._engineSettings =
typeof engine === "object" ? engine : JSON.parse(engine);
typeof engine === 'object' ? engine : JSON.parse(engine)
} else {
await fs.writeFileSync(
engineFile,
JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2),
);
JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2)
)
}
} catch (err) {
console.error(err);
console.error(err)
}
}
private static async handleModelInit(model: OpenAIModel) {
if (model.engine !== InferenceEngine.openai) {
return;
return
} else {
JanInferenceOpenAIExtension._currentModel = model;
JanInferenceOpenAIExtension.writeDefaultEngineSettings();
JanInferenceOpenAIExtension._currentModel = model
JanInferenceOpenAIExtension.writeDefaultEngineSettings()
// Todo: Check model list with API key
events.emit(ModelEvent.OnModelReady, model);
events.emit(ModelEvent.OnModelReady, model)
}
}
private static async handleModelStop(model: OpenAIModel) {
if (model.engine !== "openai") {
return;
if (model.engine !== 'openai') {
return
}
events.emit(ModelEvent.OnModelStopped, model);
events.emit(ModelEvent.OnModelStopped, model)
}
private static async handleInferenceStopped(
instance: JanInferenceOpenAIExtension,
instance: JanInferenceOpenAIExtension
) {
instance.isCancelled = true;
instance.controller?.abort();
instance.isCancelled = true
instance.controller?.abort()
}
/**
@ -148,13 +148,13 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
*/
private static async handleMessageRequest(
data: MessageRequest,
instance: JanInferenceOpenAIExtension,
instance: JanInferenceOpenAIExtension
) {
if (data.model.engine !== "openai") {
return;
if (data.model.engine !== 'openai') {
return
}
const timestamp = Date.now();
const timestamp = Date.now()
const message: ThreadMessage = {
id: ulid(),
thread_id: data.threadId,
@ -165,15 +165,15 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
status: MessageStatus.Pending,
created: timestamp,
updated: timestamp,
object: "thread.message",
};
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message);
object: 'thread.message',
}
instance.isCancelled = false;
instance.controller = new AbortController();
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message)
}
instance.isCancelled = false
instance.controller = new AbortController()
requestInference(
data?.messages ?? [],
@ -182,7 +182,7 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
...JanInferenceOpenAIExtension._currentModel,
parameters: data.model.parameters,
},
instance.controller,
instance.controller
).subscribe({
next: (content) => {
const messageContent: ThreadContent = {
@ -191,33 +191,33 @@ export default class JanInferenceOpenAIExtension extends BaseExtension {
value: content.trim(),
annotations: [],
},
};
message.content = [messageContent];
events.emit(MessageEvent.OnMessageUpdate, message);
}
message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message)
},
complete: async () => {
message.status = message.content.length
? MessageStatus.Ready
: MessageStatus.Error;
events.emit(MessageEvent.OnMessageUpdate, message);
: MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
},
error: async (err) => {
if (instance.isCancelled || message.content.length > 0) {
message.status = MessageStatus.Stopped;
events.emit(MessageEvent.OnMessageUpdate, message);
return;
message.status = MessageStatus.Stopped
events.emit(MessageEvent.OnMessageUpdate, message)
return
}
const messageContent: ThreadContent = {
type: ContentType.Text,
text: {
value: "Error occurred: " + err.message,
value: 'Error occurred: ' + err.message,
annotations: [],
},
};
message.content = [messageContent];
message.status = MessageStatus.Error;
events.emit(MessageEvent.OnMessageUpdate, message);
}
message.content = [messageContent]
message.status = MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
},
});
})
}
}

4
extensions/inference-openai-extension/tsconfig.json
View File

@ -8,7 +8,7 @@
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src",
"rootDir": "./src"
},
"include": ["./src"],
"include": ["./src"]
}

26
extensions/inference-openai-extension/webpack.config.js
View File

@ -1,16 +1,16 @@
const path = require("path");
const webpack = require("webpack");
const packageJson = require("./package.json");
const path = require('path')
const webpack = require('webpack')
const packageJson = require('./package.json')
module.exports = {
experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file
mode: "production",
entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: 'production',
module: {
rules: [
{
test: /\.tsx?$/,
use: "ts-loader",
use: 'ts-loader',
exclude: /node_modules/,
},
],
@ -18,22 +18,22 @@ module.exports = {
plugins: [
new webpack.DefinePlugin({
MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
OPENAI_DOMAIN: JSON.stringify("openai.azure.com"),
OPENAI_DOMAIN: JSON.stringify('openai.azure.com'),
}),
],
output: {
filename: "index.js", // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"),
library: { type: "module" }, // Specify ESM output format
filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, 'dist'),
library: { type: 'module' }, // Specify ESM output format
},
resolve: {
extensions: [".ts", ".js"],
extensions: ['.ts', '.js'],
fallback: {
path: require.resolve("path-browserify"),
path: require.resolve('path-browserify'),
},
},
optimization: {
minimize: false,
},
// Add loaders and other configuration as needed for your project
};
}

4
extensions/inference-triton-trtllm-extension/src/@types/global.d.ts vendored
View File

@ -1,5 +1,5 @@
import { Model } from "@janhq/core";
import { Model } from '@janhq/core'
declare interface EngineSettings {
base_url?: string;
base_url?: string
}

54
extensions/inference-triton-trtllm-extension/src/helpers/sse.ts
View File

@ -1,6 +1,6 @@
import { Observable } from "rxjs";
import { EngineSettings } from "../@types/global";
import { Model } from "@janhq/core";
import { Observable } from 'rxjs'
import { EngineSettings } from '../@types/global'
import { Model } from '@janhq/core'
/**
* Sends a request to the inference server to generate a response based on the recent messages.
@ -16,48 +16,48 @@ export function requestInference(
controller?: AbortController
): Observable<string> {
return new Observable((subscriber) => {
const text_input = recentMessages.map((message) => message.text).join("\n");
const text_input = recentMessages.map((message) => message.text).join('\n')
const requestBody = JSON.stringify({
text_input: text_input,
max_tokens: 4096,
temperature: 0,
bad_words: "",
stop_words: "[DONE]",
stream: true
});
bad_words: '',
stop_words: '[DONE]',
stream: true,
})
fetch(`${engine.base_url}/v2/models/ensemble/generate_stream`, {
method: "POST",
method: 'POST',
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
"Access-Control-Allow-Origin": "*",
'Content-Type': 'application/json',
'Accept': 'text/event-stream',
'Access-Control-Allow-Origin': '*',
},
body: requestBody,
signal: controller?.signal,
})
.then(async (response) => {
const stream = response.body;
const decoder = new TextDecoder("utf-8");
const reader = stream?.getReader();
let content = "";
const stream = response.body
const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader()
let content = ''
while (true && reader) {
const { done, value } = await reader.read();
const { done, value } = await reader.read()
if (done) {
break;
break
}
const text = decoder.decode(value);
const lines = text.trim().split("\n");
const text = decoder.decode(value)
const lines = text.trim().split('\n')
for (const line of lines) {
if (line.startsWith("data: ") && !line.includes("data: [DONE]")) {
const data = JSON.parse(line.replace("data: ", ""));
content += data.choices[0]?.delta?.content ?? "";
subscriber.next(content);
if (line.startsWith('data: ') && !line.includes('data: [DONE]')) {
const data = JSON.parse(line.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ''
subscriber.next(content)
}
}
}
subscriber.complete();
subscriber.complete()
})
.catch((err) => subscriber.error(err));
});
.catch((err) => subscriber.error(err))
})
}

116
extensions/inference-triton-trtllm-extension/src/index.ts
View File

@ -20,51 +20,49 @@ import {
BaseExtension,
MessageEvent,
ModelEvent,
} from "@janhq/core";
import { requestInference } from "./helpers/sse";
import { ulid } from "ulid";
import { join } from "path";
import { EngineSettings } from "./@types/global";
} from '@janhq/core'
import { requestInference } from './helpers/sse'
import { ulid } from 'ulid'
import { join } from 'path'
import { EngineSettings } from './@types/global'
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceTritonTrtLLMExtension
extends BaseExtension
{
private static readonly _homeDir = "file://engines";
private static readonly _engineMetadataFileName = "triton_trtllm.json";
export default class JanInferenceTritonTrtLLMExtension extends BaseExtension {
private static readonly _homeDir = 'file://engines'
private static readonly _engineMetadataFileName = 'triton_trtllm.json'
static _currentModel: Model;
static _currentModel: Model
static _engineSettings: EngineSettings = {
base_url: "",
};
base_url: '',
}
controller = new AbortController();
isCancelled = false;
controller = new AbortController()
isCancelled = false
/**
* Subscribes to events emitted by the @janhq/core package.
*/
async onLoad() {
if (!(await fs.existsSync(JanInferenceTritonTrtLLMExtension._homeDir)))
JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings();
JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings()
// Events subscription
events.on(MessageEvent.OnMessageSent, (data) =>
JanInferenceTritonTrtLLMExtension.handleMessageRequest(data, this)
);
)
events.on(ModelEvent.OnModelInit, (model: Model) => {
JanInferenceTritonTrtLLMExtension.handleModelInit(model);
});
JanInferenceTritonTrtLLMExtension.handleModelInit(model)
})
events.on(ModelEvent.OnModelStop, (model: Model) => {
JanInferenceTritonTrtLLMExtension.handleModelStop(model);
});
JanInferenceTritonTrtLLMExtension.handleModelStop(model)
})
}
/**
@ -81,7 +79,7 @@ export default class JanInferenceTritonTrtLLMExtension
modelId: string,
settings?: ModelSettingParams
): Promise<void> {
return;
return
}
static async writeDefaultEngineSettings() {
@ -89,11 +87,11 @@ export default class JanInferenceTritonTrtLLMExtension
const engine_json = join(
JanInferenceTritonTrtLLMExtension._homeDir,
JanInferenceTritonTrtLLMExtension._engineMetadataFileName
);
)
if (await fs.existsSync(engine_json)) {
const engine = await fs.readFileSync(engine_json, "utf-8");
const engine = await fs.readFileSync(engine_json, 'utf-8')
JanInferenceTritonTrtLLMExtension._engineSettings =
typeof engine === "object" ? engine : JSON.parse(engine);
typeof engine === 'object' ? engine : JSON.parse(engine)
} else {
await fs.writeFileSync(
engine_json,
@ -102,10 +100,10 @@ export default class JanInferenceTritonTrtLLMExtension
null,
2
)
);
)
}
} catch (err) {
console.error(err);
console.error(err)
}
}
/**
@ -119,26 +117,26 @@ export default class JanInferenceTritonTrtLLMExtension
* @returns {Promise<void>} A promise that resolves when the streaming is stopped.
*/
async stopInference(): Promise<void> {
this.isCancelled = true;
this.controller?.abort();
this.isCancelled = true
this.controller?.abort()
}
private static async handleModelInit(model: Model) {
if (model.engine !== "triton_trtllm") {
return;
if (model.engine !== 'triton_trtllm') {
return
} else {
JanInferenceTritonTrtLLMExtension._currentModel = model;
JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings();
JanInferenceTritonTrtLLMExtension._currentModel = model
JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings()
// Todo: Check model list with API key
events.emit(ModelEvent.OnModelReady, model);
events.emit(ModelEvent.OnModelReady, model)
}
}
private static async handleModelStop(model: Model) {
if (model.engine !== "triton_trtllm") {
return;
if (model.engine !== 'triton_trtllm') {
return
}
events.emit(ModelEvent.OnModelStopped, model);
events.emit(ModelEvent.OnModelStopped, model)
}
/**
@ -151,11 +149,11 @@ export default class JanInferenceTritonTrtLLMExtension
data: MessageRequest,
instance: JanInferenceTritonTrtLLMExtension
) {
if (data.model.engine !== "triton_trtllm") {
return;
if (data.model.engine !== 'triton_trtllm') {
return
}
const timestamp = Date.now();
const timestamp = Date.now()
const message: ThreadMessage = {
id: ulid(),
thread_id: data.threadId,
@ -165,12 +163,12 @@ export default class JanInferenceTritonTrtLLMExtension
status: MessageStatus.Pending,
created: timestamp,
updated: timestamp,
object: "thread.message",
};
events.emit(MessageEvent.OnMessageResponse, message);
object: 'thread.message',
}
events.emit(MessageEvent.OnMessageResponse, message)
instance.isCancelled = false;
instance.controller = new AbortController();
instance.isCancelled = false
instance.controller = new AbortController()
requestInference(
data?.messages ?? [],
@ -188,33 +186,33 @@ export default class JanInferenceTritonTrtLLMExtension
value: content.trim(),
annotations: [],
},
};
message.content = [messageContent];
events.emit(MessageEvent.OnMessageUpdate, message);
}
message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message)
},
complete: async () => {
message.status = message.content.length
? MessageStatus.Ready
: MessageStatus.Error;
events.emit(MessageEvent.OnMessageUpdate, message);
: MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
},
error: async (err) => {
if (instance.isCancelled || message.content.length) {
message.status = MessageStatus.Error;
events.emit(MessageEvent.OnMessageUpdate, message);
return;
message.status = MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
return
}
const messageContent: ThreadContent = {
type: ContentType.Text,
text: {
value: "Error occurred: " + err.message,
value: 'Error occurred: ' + err.message,
annotations: [],
},
};
message.content = [messageContent];
message.status = MessageStatus.Ready;
events.emit(MessageEvent.OnMessageUpdate, message);
}
message.content = [messageContent]
message.status = MessageStatus.Ready
events.emit(MessageEvent.OnMessageUpdate, message)
},
});
})
}
}

4
extensions/inference-triton-trtllm-extension/tsconfig.json
View File

@ -8,7 +8,7 @@
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src",
"rootDir": "./src"
},
"include": ["./src"],
"include": ["./src"]
}

24
extensions/inference-triton-trtllm-extension/webpack.config.js
View File

@ -1,16 +1,16 @@
const path = require("path");
const webpack = require("webpack");
const packageJson = require("./package.json");
const path = require('path')
const webpack = require('webpack')
const packageJson = require('./package.json')
module.exports = {
experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file
mode: "production",
entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: 'production',
module: {
rules: [
{
test: /\.tsx?$/,
use: "ts-loader",
use: 'ts-loader',
exclude: /node_modules/,
},
],
@ -21,18 +21,18 @@ module.exports = {
}),
],
output: {
filename: "index.js", // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"),
library: { type: "module" }, // Specify ESM output format
filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, 'dist'),
library: { type: 'module' }, // Specify ESM output format
},
resolve: {
extensions: [".ts", ".js"],
extensions: ['.ts', '.js'],
fallback: {
path: require.resolve("path-browserify"),
path: require.resolve('path-browserify'),
},
},
optimization: {
minimize: false,
},
// Add loaders and other configuration as needed for your project
};
}

4
extensions/model-extension/tsconfig.json
View File

@ -8,7 +8,7 @@
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src",
"rootDir": "./src"
},
"include": ["./src"],
"include": ["./src"]
}

2
extensions/model-extension/webpack.config.js
View File

@ -19,7 +19,7 @@ module.exports = {
new webpack.DefinePlugin({
EXTENSION_NAME: JSON.stringify(packageJson.name),
MODULE_PATH: JSON.stringify(`${packageJson.name}/${packageJson.module}`),
VERSION: JSON.stringify(packageJson.version)
VERSION: JSON.stringify(packageJson.version),
}),
],
output: {

2
extensions/monitoring-extension/src/@types/global.d.ts vendored
View File

@ -1 +1 @@
declare const MODULE: string;
declare const MODULE: string

6
extensions/monitoring-extension/src/index.ts
View File

@ -1,4 +1,4 @@
import { MonitoringExtension, executeOnMain } from "@janhq/core";
import { MonitoringExtension, executeOnMain } from '@janhq/core'
/**
* JanMonitoringExtension is a extension that provides system monitoring functionality.
@ -20,7 +20,7 @@ export default class JanMonitoringExtension extends MonitoringExtension {
* @returns A Promise that resolves to an object containing information about the system resources.
*/
getResourcesInfo(): Promise<any> {
return executeOnMain(MODULE, "getResourcesInfo");
return executeOnMain(MODULE, 'getResourcesInfo')
}
/**
@ -28,6 +28,6 @@ export default class JanMonitoringExtension extends MonitoringExtension {
* @returns A Promise that resolves to an object containing information about the current system load.
*/
getCurrentLoad(): Promise<any> {
return executeOnMain(MODULE, "getCurrentLoad");
return executeOnMain(MODULE, 'getCurrentLoad')
}
}

91
extensions/monitoring-extension/src/module.ts
View File

@ -1,73 +1,92 @@
const nodeOsUtils = require("node-os-utils");
const getJanDataFolderPath = require("@janhq/core/node").getJanDataFolderPath;
const path = require("path");
const { readFileSync } = require("fs");
const exec = require("child_process").exec;
const nodeOsUtils = require('node-os-utils')
const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath
const path = require('path')
const { readFileSync } = require('fs')
const exec = require('child_process').exec
const NVIDIA_INFO_FILE = path.join(
getJanDataFolderPath(),
"settings",
"settings.json"
);
'settings',
'settings.json'
)
const getResourcesInfo = () =>
new Promise((resolve) => {
nodeOsUtils.mem.used().then((ramUsedInfo) => {
const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024;
const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024;
const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024
const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024
const response = {
mem: {
totalMemory,
usedMemory,
},
};
resolve(response);
});
});
}
resolve(response)
})
})
const getCurrentLoad = () =>
new Promise((resolve, reject) => {
nodeOsUtils.cpu.usage().then((cpuPercentage) => {
let data = {
run_mode: "cpu",
run_mode: 'cpu',
gpus_in_use: [],
};
if (process.platform !== "darwin") {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
}
if (data.run_mode === "gpu" && data.gpus_in_use.length > 0) {
const gpuIds = data["gpus_in_use"].join(",");
if (gpuIds !== "") {
if (process.platform !== 'darwin') {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
}
if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) {
const gpuIds = data['gpus_in_use'].join(',')
if (gpuIds !== '') {
exec(
`nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`,
(error, stdout, stderr) => {
if (error) {
console.error(`exec error: ${error}`);
reject(error);
return;
console.error(`exec error: ${error}`)
reject(error)
return
}
const gpuInfo = stdout.trim().split("\n").map((line) => {
const [id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization] = line.split(", ").map(item => item.replace(/\r/g, ""));
return { id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization };
});
const gpuInfo = stdout
.trim()
.split('\n')
.map((line) => {
const [
id,
name,
temperature,
utilization,
memoryTotal,
memoryFree,
memoryUtilization,
] = line.split(', ').map((item) => item.replace(/\r/g, ''))
return {
id,
name,
temperature,
utilization,
memoryTotal,
memoryFree,
memoryUtilization,
}
})
resolve({
cpu: { usage: cpuPercentage },
gpu: gpuInfo
});
gpu: gpuInfo,
})
}
);
)
} else {
// Handle the case where gpuIds is empty
resolve({ cpu: { usage: cpuPercentage }, gpu: [] });
resolve({ cpu: { usage: cpuPercentage }, gpu: [] })
}
} else {
// Handle the case where run_mode is not 'gpu' or no GPUs are in use
resolve({ cpu: { usage: cpuPercentage }, gpu: [] });
resolve({ cpu: { usage: cpuPercentage }, gpu: [] })
}
});
});
})
})
module.exports = {
getResourcesInfo,
getCurrentLoad,
};
}

22
extensions/monitoring-extension/webpack.config.js
View File

@ -1,24 +1,24 @@
const path = require("path");
const webpack = require("webpack");
const packageJson = require("./package.json");
const path = require('path')
const webpack = require('webpack')
const packageJson = require('./package.json')
module.exports = {
experiments: { outputModule: true },
entry: "./src/index.ts", // Adjust the entry point to match your project's main file
mode: "production",
entry: './src/index.ts', // Adjust the entry point to match your project's main file
mode: 'production',
module: {
rules: [
{
test: /\.tsx?$/,
use: "ts-loader",
use: 'ts-loader',
exclude: /node_modules/,
},
],
},
output: {
filename: "index.js", // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"),
library: { type: "module" }, // Specify ESM output format
filename: 'index.js', // Adjust the output file name as needed
path: path.resolve(__dirname, 'dist'),
library: { type: 'module' }, // Specify ESM output format
},
plugins: [
new webpack.DefinePlugin({
@ -26,10 +26,10 @@ module.exports = {
}),
],
resolve: {
extensions: [".ts", ".js"],
extensions: ['.ts', '.js'],
},
optimization: {
minimize: false,
},
// Add loaders and other configuration as needed for your project
};
}

38
server/helpers/setup.ts
View File

@ -1,47 +1,47 @@
import { join, extname } from "path";
import { existsSync, readdirSync, writeFileSync, mkdirSync } from "fs";
import { init, installExtensions } from "@janhq/core/node";
import { join, extname } from 'path'
import { existsSync, readdirSync, writeFileSync, mkdirSync } from 'fs'
import { init, installExtensions } from '@janhq/core/node'
export async function setup() {
/**
* Setup Jan Data Directory
*/
const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, "..", "jan");
const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, '..', 'jan')
console.debug(`Create app data directory at ${appDir}...`);
if (!existsSync(appDir)) mkdirSync(appDir);
console.debug(`Create app data directory at ${appDir}...`)
if (!existsSync(appDir)) mkdirSync(appDir)
//@ts-ignore
global.core = {
// Define appPath function for app to retrieve app path globaly
appPath: () => appDir,
};
}
init({
extensionsPath: join(appDir, "extensions"),
});
extensionsPath: join(appDir, 'extensions'),
})
/**
* Write app configurations. See #1619
*/
console.debug("Writing config file...");
console.debug('Writing config file...')
writeFileSync(
join(appDir, "settings.json"),
join(appDir, 'settings.json'),
JSON.stringify({
data_folder: appDir,
}),
"utf-8"
);
'utf-8'
)
/**
* Install extensions
*/
console.debug("Installing extensions...");
console.debug('Installing extensions...')
const baseExtensionPath = join(__dirname, "../../..", "pre-install");
const baseExtensionPath = join(__dirname, '../../..', 'pre-install')
const extensions = readdirSync(baseExtensionPath)
.filter((file) => extname(file) === ".tgz")
.map((file) => join(baseExtensionPath, file));
.filter((file) => extname(file) === '.tgz')
.map((file) => join(baseExtensionPath, file))
await installExtensions(extensions);
console.debug("Extensions installed");
await installExtensions(extensions)
console.debug('Extensions installed')
}

106
server/index.ts
View File

@ -1,26 +1,26 @@
import fastify from "fastify";
import dotenv from "dotenv";
import fastify from 'fastify'
import dotenv from 'dotenv'
import {
getServerLogPath,
v1Router,
logServer,
getJanExtensionsPath,
} from "@janhq/core/node";
import { join } from "path";
} from '@janhq/core/node'
import { join } from 'path'
// Load environment variables
dotenv.config();
dotenv.config()
// Define default settings
const JAN_API_HOST = process.env.JAN_API_HOST || "127.0.0.1";
const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || "1337");
const JAN_API_HOST = process.env.JAN_API_HOST || '127.0.0.1'
const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || '1337')
// Initialize server settings
let server: any | undefined = undefined;
let hostSetting: string = JAN_API_HOST;
let portSetting: number = JAN_API_PORT;
let corsEnabled: boolean = true;
let isVerbose: boolean = true;
let server: any | undefined = undefined
let hostSetting: string = JAN_API_HOST
let portSetting: number = JAN_API_PORT
let corsEnabled: boolean = true
let isVerbose: boolean = true
/**
* Server configurations
@ -32,13 +32,13 @@ let isVerbose: boolean = true;
* @param baseDir - Base directory for the OpenAPI schema file
*/
export interface ServerConfig {
host?: string;
port?: number;
isCorsEnabled?: boolean;
isVerboseEnabled?: boolean;
schemaPath?: string;
baseDir?: string;
storageAdataper?: any;
host?: string
port?: number
isCorsEnabled?: boolean
isVerboseEnabled?: boolean
schemaPath?: string
baseDir?: string
storageAdataper?: any
}
/**
@ -47,69 +47,69 @@ export interface ServerConfig {
*/
export const startServer = async (configs?: ServerConfig) => {
// Update server settings
isVerbose = configs?.isVerboseEnabled ?? true;
hostSetting = configs?.host ?? JAN_API_HOST;
portSetting = configs?.port ?? JAN_API_PORT;
corsEnabled = configs?.isCorsEnabled ?? true;
const serverLogPath = getServerLogPath();
isVerbose = configs?.isVerboseEnabled ?? true
hostSetting = configs?.host ?? JAN_API_HOST
portSetting = configs?.port ?? JAN_API_PORT
corsEnabled = configs?.isCorsEnabled ?? true
const serverLogPath = getServerLogPath()
// Start the server
try {
// Log server start
if (isVerbose) logServer(`Debug: Starting JAN API server...`);
if (isVerbose) logServer(`Debug: Starting JAN API server...`)
// Initialize Fastify server with logging
server = fastify({
logger: {
level: "info",
level: 'info',
file: serverLogPath,
},
});
})
// Register CORS if enabled
if (corsEnabled) await server.register(require("@fastify/cors"), {});
if (corsEnabled) await server.register(require('@fastify/cors'), {})
// Register Swagger for API documentation
await server.register(require("@fastify/swagger"), {
mode: "static",
await server.register(require('@fastify/swagger'), {
mode: 'static',
specification: {
path: configs?.schemaPath ?? "./../docs/openapi/jan.yaml",
baseDir: configs?.baseDir ?? "./../docs/openapi",
path: configs?.schemaPath ?? './../docs/openapi/jan.yaml',
baseDir: configs?.baseDir ?? './../docs/openapi',
},
});
})
// Register Swagger UI
await server.register(require("@fastify/swagger-ui"), {
routePrefix: "/",
baseDir: configs?.baseDir ?? join(__dirname, "../..", "./docs/openapi"),
await server.register(require('@fastify/swagger-ui'), {
routePrefix: '/',
baseDir: configs?.baseDir ?? join(__dirname, '../..', './docs/openapi'),
uiConfig: {
docExpansion: "full",
docExpansion: 'full',
deepLinking: false,
},
staticCSP: false,
transformSpecificationClone: true,
});
})
// Register static file serving for extensions
// TODO: Watch extension files changes and reload
await server.register(
(childContext: any, _: any, done: any) => {
childContext.register(require("@fastify/static"), {
childContext.register(require('@fastify/static'), {
root: getJanExtensionsPath(),
wildcard: false,
});
})
done();
done()
},
{ prefix: "extensions" }
);
{ prefix: 'extensions' }
)
// Register proxy middleware
if (configs?.storageAdataper)
server.addHook("preHandler", configs.storageAdataper);
server.addHook('preHandler', configs.storageAdataper)
// Register API routes
await server.register(v1Router, { prefix: "/v1" });
await server.register(v1Router, { prefix: '/v1' })
// Start listening for requests
await server
.listen({
@ -121,13 +121,13 @@ export const startServer = async (configs?: ServerConfig) => {
if (isVerbose)
logServer(
`Debug: JAN API listening at: http://${hostSetting}:${portSetting}`
);
});
)
})
} catch (e) {
// Log any errors
if (isVerbose) logServer(`Error: ${e}`);
if (isVerbose) logServer(`Error: ${e}`)
}
};
}
/**
* Function to stop the server
@ -135,11 +135,11 @@ export const startServer = async (configs?: ServerConfig) => {
export const stopServer = async () => {
try {
// Log server stop
if (isVerbose) logServer(`Debug: Server stopped`);
if (isVerbose) logServer(`Debug: Server stopped`)
// Stop the server
await server.close();
await server.close()
} catch (e) {
// Log any errors
if (isVerbose) logServer(`Error: ${e}`);
if (isVerbose) logServer(`Error: ${e}`)
}
};
}

8
server/main.ts
View File

@ -1,7 +1,7 @@
import { s3 } from "./middleware/s3";
import { setup } from "./helpers/setup";
import { startServer as start } from "./index";
import { s3 } from './middleware/s3'
import { setup } from './helpers/setup'
import { startServer as start } from './index'
/**
* Setup extensions and start the server
*/
setup().then(() => start({ storageAdataper: s3 }));
setup().then(() => start({ storageAdataper: s3 }))

52
server/middleware/s3.ts
View File

@ -1,4 +1,4 @@
import { join } from "path";
import { join } from 'path'
// Middleware to intercept requests and proxy if certain conditions are met
const config = {
@ -8,63 +8,63 @@ const config = {
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
},
};
}
const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME;
const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME
const fs = require("@cyclic.sh/s3fs")(S3_BUCKET_NAME, config);
const PROXY_PREFIX = "/v1/fs";
const PROXY_ROUTES = ["/threads", "/messages"];
const fs = require('@cyclic.sh/s3fs')(S3_BUCKET_NAME, config)
const PROXY_PREFIX = '/v1/fs'
const PROXY_ROUTES = ['/threads', '/messages']
export const s3 = (req: any, reply: any, done: any) => {
// Proxy FS requests to S3 using S3FS
if (req.url.startsWith(PROXY_PREFIX)) {
const route = req.url.split("/").pop();
const args = parseRequestArgs(req);
const route = req.url.split('/').pop()
const args = parseRequestArgs(req)
// Proxy matched requests to the s3fs module
if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) {
try {
// Handle customized route
// S3FS does not handle appendFileSync
if (route === "appendFileSync") {
let result = handAppendFileSync(args);
if (route === 'appendFileSync') {
let result = handAppendFileSync(args)
reply.status(200).send(result);
return;
reply.status(200).send(result)
return
}
// Reroute the other requests to the s3fs module
const result = fs[route](...args);
reply.status(200).send(result);
return;
const result = fs[route](...args)
reply.status(200).send(result)
return
} catch (ex) {
console.log(ex);
console.log(ex)
}
}
}
// Let other requests go through
done();
};
done()
}
const parseRequestArgs = (req: Request) => {
const {
getJanDataFolderPath,
normalizeFilePath,
} = require("@janhq/core/node");
} = require('@janhq/core/node')
return JSON.parse(req.body as any).map((arg: any) =>
typeof arg === "string" &&
typeof arg === 'string' &&
(arg.startsWith(`file:/`) || arg.startsWith(`file:\\`))
? join(getJanDataFolderPath(), normalizeFilePath(arg))
: arg
);
};
)
}
const handAppendFileSync = (args: any[]) => {
if (fs.existsSync(args[0])) {
const data = fs.readFileSync(args[0], "utf-8");
return fs.writeFileSync(args[0], data + args[1]);
const data = fs.readFileSync(args[0], 'utf-8')
return fs.writeFileSync(args[0], data + args[1])
} else {
return fs.writeFileSync(args[0], args[1]);
return fs.writeFileSync(args[0], args[1])
}
};
}

4
uikit/types/declaration.d.ts vendored
View File

@ -1,4 +1,4 @@
declare module '*.scss' {
const content: Record<string, string>;
export default content;
const content: Record<string, string>
export default content
}

1
web/app/error.tsx
View File

@ -4,7 +4,6 @@ import { useEffect, useState } from 'react'
export default function Error({
error,
reset,
}: {
error: Error & { digest?: string }
reset: () => void

7
web/containers/Layout/BottomBar/index.tsx
View File

@ -56,13 +56,6 @@ const BottomBar = () => {
const setShowSelectModelModal = useSetAtom(showSelectModelModalAtom)
const [serverEnabled] = useAtom(serverEnabledAtom)
const calculateGpuMemoryUsage = (gpu: Record<string, never>) => {
const total = parseInt(gpu.memoryTotal)
const free = parseInt(gpu.memoryFree)
if (!total || !free) return 0
return Math.round(((total - free) / total) * 100)
}
const calculateUtilization = () => {
let sum = 0
const util = gpus.map((x) => {

3
web/screens/Chat/index.tsx
View File

@ -13,7 +13,7 @@ import GenerateResponse from '@/containers/Loader/GenerateResponse'
import ModelReload from '@/containers/Loader/ModelReload'
import ModelStart from '@/containers/Loader/ModelStart'
import { currentPromptAtom, fileUploadAtom } from '@/containers/Providers/Jotai'
import { fileUploadAtom } from '@/containers/Providers/Jotai'
import { showLeftSideBarAtom } from '@/containers/Providers/KeyListener'
import { snackbar } from '@/containers/Toast'
@ -54,7 +54,6 @@ const renderError = (code: string) => {
}
const ChatScreen: React.FC = () => {
const setCurrentPrompt = useSetAtom(currentPromptAtom)
const activeThread = useAtomValue(activeThreadAtom)
const showLeftSideBar = useAtomValue(showLeftSideBarAtom)
const engineParamsUpdate = useAtomValue(engineParamsUpdateAtom)