diff --git a/core/src/browser/extension.ts b/core/src/browser/extension.ts index 973d4778a..13f0e94f3 100644 --- a/core/src/browser/extension.ts +++ b/core/src/browser/extension.ts @@ -1,3 +1,7 @@ +import { SettingComponentProps } from '../types' +import { getJanDataFolderPath, joinPath } from './core' +import { fs } from './fs' + export enum ExtensionTypeEnum { Assistant = 'assistant', Conversational = 'conversational', @@ -32,6 +36,38 @@ export type InstallationState = InstallationStateTuple[number] * This class should be extended by any class that represents an extension. */ export abstract class BaseExtension implements ExtensionType { + protected settingFolderName = 'settings' + protected settingFileName = 'settings.json' + + /** @type {string} Name of the extension. */ + name?: string + + /** @type {string} The URL of the extension to load. */ + url: string + + /** @type {boolean} Whether the extension is activated or not. */ + active + + /** @type {string} Extension's description. */ + description + + /** @type {string} Extension's version. */ + version + + constructor( + url: string, + name?: string, + active?: boolean, + description?: string, + version?: string + ) { + this.name = name + this.url = url + this.active = active + this.description = description + this.version = version + } + /** * Returns the type of the extension. * @returns {ExtensionType} The type of the extension @@ -40,11 +76,13 @@ export abstract class BaseExtension implements ExtensionType { type(): ExtensionTypeEnum | undefined { return undefined } + /** * Called when the extension is loaded. * Any initialization logic for the extension should be put here. */ abstract onLoad(): void + /** * Called when the extension is unloaded. * Any cleanup logic for the extension should be put here. @@ -67,6 +105,42 @@ export abstract class BaseExtension implements ExtensionType { return false } + async registerSettings(settings: SettingComponentProps[]): Promise { + if (!this.name) { + console.error('Extension name is not defined') + return + } + + const extensionSettingFolderPath = await joinPath([ + await getJanDataFolderPath(), + 'settings', + this.name, + ]) + settings.forEach((setting) => { + setting.extensionName = this.name + }) + try { + await fs.mkdir(extensionSettingFolderPath) + const settingFilePath = await joinPath([extensionSettingFolderPath, this.settingFileName]) + + if (await fs.existsSync(settingFilePath)) return + await fs.writeFileSync(settingFilePath, JSON.stringify(settings, null, 2)) + } catch (err) { + console.error(err) + } + } + + async getSetting(key: string, defaultValue: T) { + const keySetting = (await this.getSettings()).find((setting) => setting.key === key) + + const value = keySetting?.controllerProps.value + return (value as T) ?? defaultValue + } + + onSettingUpdate(key: string, value: T) { + return + } + /** * Determine if the prerequisites for the extension are installed. * @@ -81,8 +155,59 @@ export abstract class BaseExtension implements ExtensionType { * * @returns {Promise} */ - // @ts-ignore - async install(...args): Promise { + async install(): Promise { return } + + async getSettings(): Promise { + if (!this.name) return [] + + const settingPath = await joinPath([ + await getJanDataFolderPath(), + this.settingFolderName, + this.name, + this.settingFileName, + ]) + + try { + const content = await fs.readFileSync(settingPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + return settings + } catch (err) { + console.warn(err) + return [] + } + } + + async updateSettings(componentProps: Partial[]): Promise { + if (!this.name) return + + const settings = await this.getSettings() + + const updatedSettings = settings.map((setting) => { + const updatedSetting = componentProps.find( + (componentProp) => componentProp.key === setting.key + ) + if (updatedSetting && updatedSetting.controllerProps) { + setting.controllerProps.value = updatedSetting.controllerProps.value + } + return setting + }) + + const settingPath = await joinPath([ + await getJanDataFolderPath(), + this.settingFolderName, + this.name, + this.settingFileName, + ]) + + await fs.writeFileSync(settingPath, JSON.stringify(updatedSettings, null, 2)) + + updatedSettings.forEach((setting) => { + this.onSettingUpdate( + setting.key, + setting.controllerProps.value + ) + }) + } } diff --git a/core/src/browser/extensions/engines/OAIEngine.ts b/core/src/browser/extensions/engines/OAIEngine.ts index 41b08f459..12bf81d36 100644 --- a/core/src/browser/extensions/engines/OAIEngine.ts +++ b/core/src/browser/extensions/engines/OAIEngine.ts @@ -48,7 +48,7 @@ export abstract class OAIEngine extends AIEngine { /* * Inference request */ - override inference(data: MessageRequest) { + override async inference(data: MessageRequest) { if (data.model?.engine?.toString() !== this.provider) return const timestamp = Date.now() @@ -77,12 +77,14 @@ export abstract class OAIEngine extends AIEngine { ...data.model, } + const header = await this.headers() + requestInference( this.inferenceUrl, data.messages ?? [], model, this.controller, - this.headers() + header ).subscribe({ next: (content: any) => { const messageContent: ThreadContent = { @@ -123,7 +125,7 @@ export abstract class OAIEngine extends AIEngine { /** * Headers for the inference request */ - headers(): HeadersInit { + async headers(): Promise { return {} } } diff --git a/core/src/browser/extensions/engines/RemoteOAIEngine.ts b/core/src/browser/extensions/engines/RemoteOAIEngine.ts index 2d5126c6b..b11235370 100644 --- a/core/src/browser/extensions/engines/RemoteOAIEngine.ts +++ b/core/src/browser/extensions/engines/RemoteOAIEngine.ts @@ -5,8 +5,7 @@ import { OAIEngine } from './OAIEngine' * Added the implementation of loading and unloading model (applicable to local inference providers) */ export abstract class RemoteOAIEngine extends OAIEngine { - // The inference engine - abstract apiKey: string + apiKey?: string /** * On extension load, subscribe to events. */ @@ -17,10 +16,12 @@ export abstract class RemoteOAIEngine extends OAIEngine { /** * Headers for the inference request */ - override headers(): HeadersInit { + override async headers(): Promise { return { - 'Authorization': `Bearer ${this.apiKey}`, - 'api-key': `${this.apiKey}`, + ...(this.apiKey && { + 'Authorization': `Bearer ${this.apiKey}`, + 'api-key': `${this.apiKey}`, + }), } } } diff --git a/core/src/node/api/common/handler.ts b/core/src/node/api/common/handler.ts index fb958dbd1..5cf232d8a 100644 --- a/core/src/node/api/common/handler.ts +++ b/core/src/node/api/common/handler.ts @@ -5,19 +5,16 @@ export type Handler = (route: string, args: any) => any export class RequestHandler { handler: Handler - adataper: RequestAdapter + adapter: RequestAdapter constructor(handler: Handler, observer?: Function) { this.handler = handler - this.adataper = new RequestAdapter(observer) + this.adapter = new RequestAdapter(observer) } handle() { CoreRoutes.map((route) => { - this.handler(route, async (...args: any[]) => { - const values = await this.adataper.process(route, ...args) - return values - }) + this.handler(route, async (...args: any[]) => this.adapter.process(route, ...args)) }) } } diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index e34fb606b..01ab26394 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -316,6 +316,7 @@ export const chatCompletions = async (request: any, reply: any) => { } const requestedModel = matchedModels[0] + const engineConfiguration = await getEngineConfiguration(requestedModel.engine) let apiKey: string | undefined = undefined @@ -323,7 +324,7 @@ export const chatCompletions = async (request: any, reply: any) => { if (engineConfiguration) { apiKey = engineConfiguration.api_key - apiUrl = engineConfiguration.full_url + apiUrl = engineConfiguration.full_url ?? DEFAULT_CHAT_COMPLETION_URL } const headers: Record = { diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index b5ec2e029..2b828b576 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -1,4 +1,4 @@ -import { AppConfiguration } from '../../types' +import { AppConfiguration, SettingComponentProps } from '../../types' import { join } from 'path' import fs from 'fs' import os from 'os' @@ -125,14 +125,30 @@ const exec = async (command: string): Promise => { }) } +// a hacky way to get the api key. we should comes up with a better +// way to handle this export const getEngineConfiguration = async (engineId: string) => { - if (engineId !== 'openai' && engineId !== 'groq') { - return undefined + if (engineId !== 'openai' && engineId !== 'groq') return undefined + + const settingDirectoryPath = join( + getJanDataFolderPath(), + 'settings', + engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension', + 'settings.json' + ) + + const content = fs.readFileSync(settingDirectoryPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key' + const keySetting = settings.find((setting) => setting.key === apiKeyId) + + let apiKey = keySetting?.controllerProps.value + if (typeof apiKey !== 'string') apiKey = '' + + return { + api_key: apiKey, + full_url: undefined, } - const directoryPath = join(getJanDataFolderPath(), 'engines') - const filePath = join(directoryPath, `${engineId}.json`) - const data = fs.readFileSync(filePath, 'utf-8') - return JSON.parse(data) } /** diff --git a/core/src/types/index.ts b/core/src/types/index.ts index 291c73524..6627ebff9 100644 --- a/core/src/types/index.ts +++ b/core/src/types/index.ts @@ -9,3 +9,4 @@ export * from './config' export * from './huggingface' export * from './miscellaneous' export * from './api' +export * from './setting' diff --git a/core/src/types/setting/index.ts b/core/src/types/setting/index.ts new file mode 100644 index 000000000..b3407460c --- /dev/null +++ b/core/src/types/setting/index.ts @@ -0,0 +1 @@ +export * from './settingComponent' diff --git a/core/src/types/setting/settingComponent.ts b/core/src/types/setting/settingComponent.ts new file mode 100644 index 000000000..4d9526505 --- /dev/null +++ b/core/src/types/setting/settingComponent.ts @@ -0,0 +1,34 @@ +export type SettingComponentProps = { + key: string + title: string + description: string + controllerType: ControllerType + controllerProps: SliderComponentProps | CheckboxComponentProps | InputComponentProps + + extensionName?: string + requireModelReload?: boolean + configType?: ConfigType +} + +export type ConfigType = 'runtime' | 'setting' + +export type ControllerType = 'slider' | 'checkbox' | 'input' + +export type InputType = 'password' | 'text' | 'email' | 'number' | 'tel' | 'url' + +export type InputComponentProps = { + placeholder: string + value: string + type?: InputType +} + +export type SliderComponentProps = { + min: number + max: number + step: number + value: number +} + +export type CheckboxComponentProps = { + value: boolean +} diff --git a/extensions/assistant-extension/rollup.config.ts b/extensions/assistant-extension/rollup.config.ts index 0d1e4832c..744ef2b97 100644 --- a/extensions/assistant-extension/rollup.config.ts +++ b/extensions/assistant-extension/rollup.config.ts @@ -7,12 +7,10 @@ import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') -const pkg = require('./package.json') - export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -36,7 +34,7 @@ export default [ // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ extensions: ['.js', '.ts', '.svelte'], - browser: true + browser: true, }), // Resolve source maps to the original source diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 70d02af1f..17094ffbc 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -1,13 +1,36 @@ import fs from 'fs' import path from 'path' -import { getJanDataFolderPath } from '@janhq/core/node' +import { SettingComponentProps, getJanDataFolderPath } from '@janhq/core/node' // Sec: Do not send engine settings over requests // Read it manually instead export const readEmbeddingEngine = (engineName: string) => { - const engineSettings = fs.readFileSync( - path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), - 'utf-8' - ) - return JSON.parse(engineSettings) + if (engineName !== 'openai' && engineName !== 'groq') { + const engineSettings = fs.readFileSync( + path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), + 'utf-8' + ) + return JSON.parse(engineSettings) + } else { + const settingDirectoryPath = path.join( + getJanDataFolderPath(), + 'settings', + engineName === 'openai' + ? 'inference-openai-extension' + : 'inference-groq-extension', + 'settings.json' + ) + + const content = fs.readFileSync(settingDirectoryPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + const apiKeyId = engineName === 'openai' ? 'openai-api-key' : 'groq-api-key' + const keySetting = settings.find((setting) => setting.key === apiKeyId) + + let apiKey = keySetting?.controllerProps.value + if (typeof apiKey !== 'string') apiKey = '' + + return { + api_key: apiKey, + } + } } diff --git a/extensions/inference-groq-extension/resources/settings.json b/extensions/inference-groq-extension/resources/settings.json new file mode 100644 index 000000000..2e6ca413b --- /dev/null +++ b/extensions/inference-groq-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [Groq Documentation](https://console.groq.com/docs/openai) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "Chat Completions Endpoint", + "value": "https://api.groq.com/openai/v1/chat/completions" + } + }, + { + "key": "groq-api-key", + "title": "API Key", + "description": "The Groq API uses API keys for authentication. Visit your [API Keys](https://console.groq.com/keys) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "API Key", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-groq-extension/src/index.ts b/extensions/inference-groq-extension/src/index.ts index cd22c62f9..ea62aa14a 100644 --- a/extensions/inference-groq-extension/src/index.ts +++ b/extensions/inference-groq-extension/src/index.ts @@ -6,78 +6,41 @@ * @module inference-groq-extension/src/index */ -import { - events, - fs, - AppConfigurationEventName, - joinPath, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine } from '@janhq/core' -declare const COMPLETION_URL: string +declare const SETTINGS: Array +enum Settings { + apiKey = 'groq-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceGroqExtension extends RemoteOAIEngine { - private readonly _engineDir = 'file://engines' - private readonly _engineMetadataFileName = 'groq.json' - - inferenceUrl: string = COMPLETION_URL + inferenceUrl: string = '' provider = 'groq' - apiKey = '' - private _engineSettings = { - full_url: COMPLETION_URL, - api_key: 'gsk-', - } - - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { + override async onLoad(): Promise { super.onLoad() - if (!(await fs.existsSync(this._engineDir))) { - await fs.mkdir(this._engineDir) - } + // Register Settings + this.registerSettings(SETTINGS) - this.writeDefaultEngineSettings() - - const settingsFilePath = await joinPath([ - this._engineDir, - this._engineMetadataFileName, - ]) - - // Events subscription - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + // Retrieve API Key Setting + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) } - async writeDefaultEngineSettings() { - try { - const engineFile = join(this._engineDir, this._engineMetadataFileName) - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.full_url - this.apiKey = this._engineSettings.api_key - } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ) - } - } catch (err) { - console.error(err) + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = value as string } } } diff --git a/extensions/inference-groq-extension/webpack.config.js b/extensions/inference-groq-extension/webpack.config.js index 5352b56b7..13d32c52d 100644 --- a/extensions/inference-groq-extension/webpack.config.js +++ b/extensions/inference-groq-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,8 +18,8 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - COMPLETION_URL: JSON.stringify('https://api.groq.com/openai/v1/chat/completions'), }), ], output: { diff --git a/extensions/inference-nitro-extension/resources/default_settings.json b/extensions/inference-nitro-extension/resources/default_settings.json new file mode 100644 index 000000000..39f0880b0 --- /dev/null +++ b/extensions/inference-nitro-extension/resources/default_settings.json @@ -0,0 +1,33 @@ +[ + { + "key": "test", + "title": "Test", + "description": "Test", + "controllerType": "input", + "controllerProps": { + "placeholder": "Test", + "value": "" + } + }, + { + "key": "embedding", + "title": "Embedding", + "description": "Whether to enable embedding.", + "controllerType": "checkbox", + "controllerProps": { + "value": true + } + }, + { + "key": "ctx_len", + "title": "Context Length", + "description": "The context length for model operations varies; the maximum depends on the specific model used.", + "controllerType": "slider", + "controllerProps": { + "min": 0, + "max": 4096, + "step": 128, + "value": 4096 + } + } +] diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts index 396c40d08..ea7f1b14d 100644 --- a/extensions/inference-nitro-extension/rollup.config.ts +++ b/extensions/inference-nitro-extension/rollup.config.ts @@ -5,13 +5,12 @@ import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') - -const pkg = require('./package.json') +const defaultSettingJson = require('./resources/default_settings.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -19,7 +18,9 @@ export default [ }, plugins: [ replace({ + EXTENSION_NAME: JSON.stringify(packageJson.name), NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson), INFERENCE_URL: JSON.stringify( process.env.INFERENCE_URL || 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion' diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index 3a3d2aa32..e16f77b31 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -2,6 +2,8 @@ declare const NODE: string declare const INFERENCE_URL: string declare const TROUBLESHOOTING_URL: string declare const JAN_SERVER_INFERENCE_URL: string +declare const EXTENSION_NAME: string +declare const DEFAULT_SETTINGS: Array /** * The response from the initModel function. diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 313b67365..3d08efdfa 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -58,8 +58,6 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine { this.inferenceUrl = `${window.core?.api?.baseApiUrl}/v1/chat/completions` } - console.debug('Inference url: ', this.inferenceUrl) - this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), JanInferenceNitroExtension._intervalHealthCheck diff --git a/extensions/inference-openai-extension/resources/settings.json b/extensions/inference-openai-extension/resources/settings.json new file mode 100644 index 000000000..1a19e30a7 --- /dev/null +++ b/extensions/inference-openai-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "Chat Completions Endpoint", + "value": "https://api.openai.com/v1/chat/completions" + } + }, + { + "key": "openai-api-key", + "title": "API Key", + "description": "The OpenAI API uses API keys for authentication. Visit your [API Keys](https://platform.openai.com/account/api-keys) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "API Key", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index eb60540fa..0853fe0f6 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -5,85 +5,41 @@ * @version 1.0.0 * @module inference-openai-extension/src/index */ -declare const ENGINE: string -import { - events, - fs, - AppConfigurationEventName, - joinPath, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' - -declare const COMPLETION_URL: string +import { RemoteOAIEngine } from '@janhq/core' +declare const SETTINGS: Array +enum Settings { + apiKey = 'openai-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension extends RemoteOAIEngine { - private static readonly _engineDir = 'file://engines' - private static readonly _engineMetadataFileName = `${ENGINE}.json` - - private _engineSettings = { - full_url: COMPLETION_URL, - api_key: 'sk-', - } - - inferenceUrl: string = COMPLETION_URL + inferenceUrl: string = '' provider: string = 'openai' - apiKey: string = '' - // TODO: Just use registerSettings from BaseExtension - // Remove these methods - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { + override async onLoad(): Promise { super.onLoad() - if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { - await fs.mkdir(JanInferenceOpenAIExtension._engineDir) - } + // Register Settings + this.registerSettings(SETTINGS) - this.writeDefaultEngineSettings() - - const settingsFilePath = await joinPath([ - JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName, - ]) - - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) } - async writeDefaultEngineSettings() { - try { - const engineFile = join( - JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName - ) - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.full_url - this.apiKey = this._engineSettings.api_key - } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ) - } - } catch (err) { - console.error(err) + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = value as string } } } diff --git a/extensions/inference-openai-extension/webpack.config.js b/extensions/inference-openai-extension/webpack.config.js index ee18035f2..3954d031d 100644 --- a/extensions/inference-openai-extension/webpack.config.js +++ b/extensions/inference-openai-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,8 +18,8 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), ENGINE: JSON.stringify(packageJson.engine), - COMPLETION_URL: JSON.stringify('https://api.openai.com/v1/chat/completions'), }), ], output: { diff --git a/extensions/inference-triton-trtllm-extension/resources/settings.json b/extensions/inference-triton-trtllm-extension/resources/settings.json new file mode 100644 index 000000000..bba69805e --- /dev/null +++ b/extensions/inference-triton-trtllm-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions.", + "controllerType": "input", + "controllerProps": { + "placeholder": "Chat Completions Endpoint", + "value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate" + } + }, + { + "key": "tritonllm-api-key", + "title": "Triton LLM API Key", + "description": "The Triton LLM API uses API keys for authentication.", + "controllerType": "input", + "controllerProps": { + "placeholder": "API Key", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts index 6f9a01d9b..a3032f01d 100644 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ b/extensions/inference-triton-trtllm-extension/src/index.ts @@ -6,77 +6,44 @@ * @module inference-nvidia-triton-trt-llm-extension/src/index */ -import { - AppConfigurationEventName, - events, - fs, - joinPath, - Model, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine } from '@janhq/core' +declare const SETTINGS: Array +enum Settings { + apiKey = 'tritonllm-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine { - private readonly _engineDir = 'file://engines' - private readonly _engineMetadataFileName = 'triton_trtllm.json' - inferenceUrl: string = '' provider: string = 'triton_trtllm' - apiKey: string = '' - - _engineSettings: { - base_url: '' - api_key: '' - } /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { super.onLoad() - if (!(await fs.existsSync(this._engineDir))) { - await fs.mkdir(this._engineDir) - } - this.writeDefaultEngineSettings() + // Register Settings + this.registerSettings(SETTINGS) - const settingsFilePath = await joinPath([ - this._engineDir, - this._engineMetadataFileName, - ]) - - // Events subscription - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + // Retrieve API Key Setting + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) } - async writeDefaultEngineSettings() { - try { - const engine_json = join(this._engineDir, this._engineMetadataFileName) - if (await fs.existsSync(engine_json)) { - const engine = await fs.readFileSync(engine_json, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.base_url - this.apiKey = this._engineSettings.api_key - } else { - await fs.writeFileSync( - engine_json, - JSON.stringify(this._engineSettings, null, 2) - ) - } - } catch (err) { - console.error(err) + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = value as string } } } diff --git a/extensions/inference-triton-trtllm-extension/webpack.config.js b/extensions/inference-triton-trtllm-extension/webpack.config.js index e83370a1a..13d32c52d 100644 --- a/extensions/inference-triton-trtllm-extension/webpack.config.js +++ b/extensions/inference-triton-trtllm-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,6 +18,7 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), }), ], diff --git a/extensions/model-extension/rollup.config.ts b/extensions/model-extension/rollup.config.ts index 722785aa3..0f5ad04df 100644 --- a/extensions/model-extension/rollup.config.ts +++ b/extensions/model-extension/rollup.config.ts @@ -1,5 +1,4 @@ import resolve from '@rollup/plugin-node-resolve' -import commonjs from '@rollup/plugin-commonjs' import sourceMaps from 'rollup-plugin-sourcemaps' import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' @@ -7,12 +6,10 @@ import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') -const pkg = require('./package.json') - export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index ce89beca2..d099edfe9 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -251,7 +251,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { return Promise.resolve() } - override inference(data: MessageRequest): void { + override async inference(data: MessageRequest) { if (!this.loadedModel) return // TensorRT LLM Extension supports streaming only if (data.model) data.model.parameters.stream = true diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index b0953cdea..76cbd05dd 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -38,7 +38,6 @@ import useUpdateModelParameters from '@/hooks/useUpdateModelParameters' import { toGibibytes } from '@/utils/converter' import ModelLabel from '../ModelLabel' -import OpenAiKeyInput from '../OpenAiKeyInput' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom' @@ -144,7 +143,7 @@ const DropdownListSidebar = ({ // Update model parameter to the thread file if (model) - updateModelParameter(activeThread.id, { + updateModelParameter(activeThread, { params: modelParams, modelId: model.id, engine: model.engine, @@ -170,172 +169,164 @@ const DropdownListSidebar = ({ stateModel.model === selectedModel?.id && stateModel.loading return ( - <> -
+ - - - {selectedModelLoading && ( -
- )} - - {selectedModel?.name} - - - - - + + {selectedModelLoading && ( +
+ )} + -
-
    - {engineOptions.map((name, i) => { - return ( -
  • + + + + +
    +
      + {engineOptions.map((name, i) => { + return ( +
    • setIsTabActive(i)} + > + {i === 0 ? ( + + ) : ( + + )} + setIsTabActive(i)} > - {i === 0 ? ( - - ) : ( - - )} - - {name} - -
    • - ) - })} -
    -
    + {name} + +
  • + ) + })} +
+
-
- {downloadedModels.length === 0 ? ( -
-

{`Oops, you don't have a model yet.`}

-
- ) : ( - - <> - {modelOptions.map((x, i) => ( -
+ {downloadedModels.length === 0 ? ( +
+

{`Oops, you don't have a model yet.`}

+
+ ) : ( + + <> + {modelOptions.map((x, i) => ( +
+ - -
-
- - {x.name} - -
- - {toGibibytes(x.metadata.size)} - - {x.metadata.size && ( - - )} -
-
+
+ {x.name} +
+ + {toGibibytes(x.metadata.size)} + + {x.metadata.size && ( + + )}
- -
- {x.id} - {clipboard.copied && copyId === x.id ? ( - - ) : ( - { - clipboard.copy(x.id) - setCopyId(x.id) - }} - /> - )}
+ +
+ {x.id} + {clipboard.copied && copyId === x.id ? ( + + ) : ( + { + clipboard.copy(x.id) + setCopyId(x.id) + }} + /> + )}
- ))} - - - )} -
-
- - -
- - - -
- - - +
+ ))} + + + )} +
+
+ + +
+ + + +
) } diff --git a/web/containers/ModelConfigInput/index.tsx b/web/containers/ModelConfigInput/index.tsx index d573a0bf9..0c16c916c 100644 --- a/web/containers/ModelConfigInput/index.tsx +++ b/web/containers/ModelConfigInput/index.tsx @@ -26,33 +26,31 @@ const ModelConfigInput: React.FC = ({ description, placeholder, onValueChanged, -}) => { - return ( -
-
-

- {title} -

- - - - - - - {description} - - - - -
-