Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: local engine management (#4334)

Browse Source 
* feat: local engine management

* chore: move remote engine into engine page instead extension page

* chore: set default engine from extension

* chore: update endpoint update engine

* chore: update event onEngineUpdate

* chore: filter out engine download

* chore: update version env

* chore: select default engine variant base on user device specs

* chore: symlink engine variants

* chore: rolldown.config in mjs format

* chore: binary codesign

* fix: download state in footer bar and variant status

* chore: update yarn.lock

* fix: rimraf failure

* fix: setup-node@v3 for built-in cache

* fix: cov pipeline

* fix: build syntax

* chore: fix build step

* fix: create engines folder on launch

* chore: update ui delete engine variant with modal confirmation

* chore: fix linter

* chore: add installing progress for Local Engine download

* chore: wording

---------

Co-authored-by: Louis <louis@jan.ai>
This commit is contained in:
Faisal Amir 2024-12-30 18:27:51 +08:00 committed by GitHub
parent bd0e525d66
commit a6a0cb325b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 1962 additions and 418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/jan-electron-linter-and-test.yml vendored
View File

@ -44,16 +44,16 @@ jobs:
- uses: actions/checkout@v3
with:
ref: ${{ github.base_ref }}
- name: Use Node.js v20.9.0
- name: Use Node.js 20.x
uses: actions/setup-node@v3
with:
node-version: v20.9.0
node-version: 20
- name: Install dependencies
run: |
yarn
make build-joi
yarn build:core
yarn build:joi
yarn
- name: Run test coverage
run: yarn test:coverage
@ -187,7 +187,7 @@ jobs:
fetch-depth: 0
- name: Installing node
uses: actions/setup-node@v1
uses: actions/setup-node@v3
with:
node-version: 20

1
core/src/browser/extension.ts
View File

@ -11,6 +11,7 @@ export enum ExtensionTypeEnum {
Model = 'model',
SystemMonitoring = 'systemMonitoring',
HuggingFace = 'huggingFace',
Engine = 'engine',
}
export interface ExtensionType {

91
core/src/browser/extensions/enginesManagement.ts Normal file
View File

@ -0,0 +1,91 @@
import {
InferenceEngine,
Engines,
EngineVariant,
EngineReleased,
DefaultEngineVariant,
} from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Engine management extension. Persists and retrieves engine management.
* @abstract
* @extends BaseExtension
*/
export abstract class EngineManagementExtension extends BaseExtension {
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Engine
}
/**
* @returns A Promise that resolves to an object of list engines.
*/
abstract getEngines(): Promise<Engines>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an array of installed engine.
*/
abstract getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]>
/**
* @param name - Inference engine name.
* @param version - Version of the engine.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
abstract getReleasedEnginesByVersion(
name: InferenceEngine,
version: string,
platform?: string
): Promise<EngineReleased[]>
/**
* @param name - Inference engine name.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine.
*/
abstract getLatestReleasedEngine(
name: InferenceEngine,
platform?: string
): Promise<EngineReleased[]>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to intall of engine.
*/
abstract installEngine(
name: InferenceEngine,
engineConfig: { variant: string; version?: string }
): Promise<{ messages: string }>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to unintall of engine.
*/
abstract uninstallEngine(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
): Promise<{ messages: string }>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
*/
abstract getDefaultEngineVariant(name: InferenceEngine): Promise<DefaultEngineVariant>
/**
* @body variant - string
* @body version - string
* @returns A Promise that resolves to set default engine.
*/
abstract setDefaultEngineVariant(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
): Promise<{ messages: string }>
/**
* @returns A Promise that resolves to update engine.
*/
abstract updateEngine(name: InferenceEngine): Promise<{ messages: string }>
}

5
core/src/browser/extensions/index.ts
View File

@ -28,3 +28,8 @@ export { ModelExtension } from './model'
* Base AI Engines.
*/
export * from './engines'
/**
* Engines Management
*/
export * from './enginesManagement'

28
core/src/types/engine/index.ts Normal file
View File

@ -0,0 +1,28 @@
import { InferenceEngine } from '../../types'
export type Engines = {
[key in InferenceEngine]: EngineVariant[]
}
export type EngineVariant = {
engine: InferenceEngine
name: string
version: string
}
export type DefaultEngineVariant = {
engine: InferenceEngine
variant: string
version: string
}
export type EngineReleased = {
created_at: string
download_count: number
name: string
size: number
}
export enum EngineEvent {
OnEngineUpdate = 'OnEngineUpdate',
}

1
core/src/types/index.ts
View File

@ -10,3 +10,4 @@ export * from './huggingface'
export * from './miscellaneous'
export * from './api'
export * from './setting'
export * from './engine'

5
extensions/engine-management-extension/jest.config.js Normal file
View File

@ -0,0 +1,5 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
}

47
extensions/engine-management-extension/package.json Normal file
View File

@ -0,0 +1,47 @@
{
"name": "@janhq/engine-management-extension",
"productName": "Engine Management",
"version": "1.0.0",
"description": "Extension for managing engines and their configurations",
"main": "dist/index.js",
"node": "dist/node/index.cjs.js",
"author": "Jan <service@jan.ai>",
"license": "MIT",
"scripts": {
"test": "jest",
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install"
},
"exports": {
".": "./dist/index.js",
"./main": "./dist/module.js"
},
"devDependencies": {
"@rollup/plugin-replace": "^6.0.2",
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "^1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.3.3",
"webpack": "^5.88.2",
"webpack-cli": "^5.1.4"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"cpu-instructions": "^0.0.13",
"ky": "^1.7.2",
"p-queue": "^8.0.1"
},
"bundledDependencies": [
"cpu-instructions",
"@janhq/core"
],
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
]
}

45
extensions/engine-management-extension/rolldown.config.mjs Normal file
View File

@ -0,0 +1,45 @@
import { defineConfig } from 'rolldown'
import replace from '@rollup/plugin-replace'
import pkgJson from './package.json' with { type: 'json' }
export default defineConfig([
{
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
plugins: [
replace({
NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
API_URL: JSON.stringify('http://127.0.0.1:39291'),
SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'),
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'),
}),
],
},
{
input: 'src/node/index.ts',
external: ['@janhq/core/node'],
output: {
format: 'cjs',
file: 'dist/node/index.cjs.js',
},
plugins: [
replace({
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'),
}),
],
},
{
input: 'src/node/cpuInfo.ts',
output: {
format: 'cjs',
file: 'dist/node/cpuInfo.js',
},
external: ['cpu-instructions'],
resolve: {
extensions: ['.ts', '.js', '.svg'],
},
},
])

16
extensions/engine-management-extension/src/@types/global.d.ts vendored Normal file
View File

@ -0,0 +1,16 @@
export {}
declare global {
declare const API_URL: string
declare const CORTEX_ENGINE_VERSION: string
declare const SOCKET_URL: string
declare const NODE: string
interface Core {
api: APIFunctions
events: EventEmitter
}
interface Window {
core?: Core | undefined
electronAPI?: any | undefined
}
}

10
extensions/engine-management-extension/src/error.ts Normal file
View File

@ -0,0 +1,10 @@
/**
* Custom Engine Error
*/
export class EngineError extends Error {
message: string
constructor(message: string) {
super()
this.message = message
}
}

219
extensions/engine-management-extension/src/index.ts Normal file
View File

@ -0,0 +1,219 @@
import {
EngineManagementExtension,
InferenceEngine,
DefaultEngineVariant,
Engines,
EngineVariant,
EngineReleased,
executeOnMain,
systemInformation,
} from '@janhq/core'
import ky, { HTTPError } from 'ky'
import PQueue from 'p-queue'
import { EngineError } from './error'
/**
* JSONEngineManagementExtension is a EngineManagementExtension implementation that provides
* functionality for managing engines.
*/
export default class JSONEngineManagementExtension extends EngineManagementExtension {
queue = new PQueue({ concurrency: 1 })
/**
* Called when the extension is loaded.
*/
async onLoad() {
// Symlink Engines Directory
await executeOnMain(NODE, 'symlinkEngines')
// Run Healthcheck
this.queue.add(() => this.healthz())
try {
const variant = await this.getDefaultEngineVariant(
InferenceEngine.cortex_llamacpp
)
// Check whether should use bundled version or installed version
// Only use larger version
if (this.compareVersions(CORTEX_ENGINE_VERSION, variant.version) > 0) {
throw new EngineError(
'Default engine version is smaller than bundled version'
)
}
} catch (error) {
if (
(error instanceof HTTPError && error.response.status === 400) ||
error instanceof EngineError
) {
const systemInfo = await systemInformation()
const variant = await executeOnMain(
NODE,
'engineVariant',
systemInfo.gpuSetting
)
await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, {
variant: variant,
version: `${CORTEX_ENGINE_VERSION}`,
})
} else {
console.error('An unexpected error occurred:', error)
}
}
}
/**
* Called when the extension is unloaded.
*/
onUnload() {}
/**
* @returns A Promise that resolves to an object of list engines.
*/
async getEngines(): Promise<Engines> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines`)
.json<Engines>()
.then((e) => e)
) as Promise<Engines>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an array of installed engine.
*/
async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}`)
.json<EngineVariant[]>()
.then((e) => e)
) as Promise<EngineVariant[]>
}
/**
* @param name - Inference engine name.
* @param version - Version of the engine.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
async getReleasedEnginesByVersion(
name: InferenceEngine,
version: string,
platform?: string
) {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}/releases/${version}`)
.json<EngineReleased[]>()
.then((e) =>
platform ? e.filter((r) => r.name.includes(platform)) : e
)
) as Promise<EngineReleased[]>
}
/**
* @param name - Inference engine name.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
async getLatestReleasedEngine(name: InferenceEngine, platform?: string) {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}/releases/latest`)
.json<EngineReleased[]>()
.then((e) =>
platform ? e.filter((r) => r.name.includes(platform)) : e
)
) as Promise<EngineReleased[]>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to intall of engine.
*/
async installEngine(
name: InferenceEngine,
engineConfig: { variant: string; version?: string }
) {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to unintall of engine.
*/
async uninstallEngine(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
) {
return this.queue.add(() =>
ky
.delete(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
*/
async getDefaultEngineVariant(name: InferenceEngine) {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}/default`)
.json<{ messages: string }>()
.then((e) => e)
) as Promise<DefaultEngineVariant>
}
/**
* @body variant - string
* @body version - string
* @returns A Promise that resolves to set default engine.
*/
async setDefaultEngineVariant(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
) {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/engines/${name}/default`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* @returns A Promise that resolves to update engine.
*/
async updateEngine(name: InferenceEngine) {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/engines/${name}/update`).then((e) => e)
) as Promise<{ messages: string }>
}
/**
* Do health check on cortex.cpp
* @returns
*/
async healthz(): Promise<void> {
return ky
.get(`${API_URL}/healthz`, {
retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
.then(() => {})
}
private compareVersions(version1: string, version2: string): number {
const parseVersion = (version: string) => version.split('.').map(Number)
const [major1, minor1, patch1] = parseVersion(version1.replace(/^v/, ''))
const [major2, minor2, patch2] = parseVersion(version2.replace(/^v/, ''))
if (major1 !== major2) return major1 - major2
if (minor1 !== minor2) return minor1 - minor2
return patch1 - patch2
}
}

0
extensions/inference-cortex-extension/src/node/cpuInfo.ts → extensions/engine-management-extension/src/node/cpuInfo.ts
View File

181
extensions/inference-cortex-extension/src/node/execute.test.ts → extensions/engine-management-extension/src/node/index.test.ts
View File

@ -1,5 +1,5 @@
import { describe, expect, it } from '@jest/globals'
import { engineVariant, executableCortexFile } from './execute'
import engine from './index'
import { GpuSetting } from '@janhq/core/node'
import { cpuInfo } from 'cpu-instructions'
import { fork } from 'child_process'
@ -62,20 +62,9 @@ describe('test executable cortex file', () => {
Object.defineProperty(process, 'arch', {
value: 'arm64',
})
expect(executableCortexFile(testSettings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath:
originalPlatform === 'darwin'
? expect.stringContaining(`cortex-server`)
: expect.anything(),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
mockFork.mockReturnValue(mockProcess)
expect(engineVariant(testSettings)).resolves.toEqual('mac-arm64')
expect(engine.engineVariant(testSettings)).resolves.toEqual('mac-arm64')
})
it('executes on MacOS', () => {
@ -99,18 +88,7 @@ describe('test executable cortex file', () => {
value: 'x64',
})
expect(executableCortexFile(testSettings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath:
originalPlatform === 'darwin'
? expect.stringContaining(`cortex-server`)
: expect.anything(),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant(testSettings)).resolves.toEqual('mac-amd64')
expect(engine.engineVariant(testSettings)).resolves.toEqual('mac-amd64')
})
it('executes on Windows CPU', () => {
@ -131,15 +109,7 @@ describe('test executable cortex file', () => {
}
mockFork.mockReturnValue(mockProcess)
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant()).resolves.toEqual('windows-amd64-avx')
expect(engine.engineVariant()).resolves.toEqual('windows-amd64-avx')
})
it('executes on Windows Cuda 11', () => {
@ -176,15 +146,8 @@ describe('test executable cortex file', () => {
send: jest.fn(),
}
mockFork.mockReturnValue(mockProcess)
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
'windows-amd64-avx2-cuda-11-7'
)
})
@ -221,15 +184,8 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
'windows-amd64-noavx-cuda-12-0'
)
mockFork.mockReturnValue({
@ -240,7 +196,7 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
'windows-amd64-avx2-cuda-12-0'
)
})
@ -261,15 +217,8 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant()).resolves.toEqual('linux-amd64-noavx')
expect(engine.engineVariant()).resolves.toEqual('linux-amd64-noavx')
})
it('executes on Linux Cuda 11', () => {
@ -306,15 +255,9 @@ describe('test executable cortex file', () => {
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
expect(engine.engineVariant(settings)).resolves.toBe(
'linux-amd64-avx2-cuda-11-7'
)
expect(engineVariant(settings)).resolves.toBe('linux-amd64-avx2-cuda-11-7')
})
it('executes on Linux Cuda 12', () => {
@ -349,15 +292,8 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
'linux-amd64-avx2-cuda-12-0'
)
})
@ -383,16 +319,7 @@ describe('test executable cortex file', () => {
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
`linux-amd64-${instruction}`
)
})
@ -416,15 +343,7 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
`windows-amd64-${instruction}`
)
})
@ -465,15 +384,7 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
`windows-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0`
)
})
@ -514,15 +425,7 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).resolves.toEqual(
expect(engine.engineVariant(settings)).resolves.toEqual(
`linux-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0`
)
})
@ -564,50 +467,8 @@ describe('test executable cortex file', () => {
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).resolves.toEqual(`linux-amd64-vulkan`)
})
})
// Generate test for different cpu instructions on MacOS
it(`executes on MacOS with different instructions`, () => {
Object.defineProperty(process, 'platform', {
value: 'darwin',
})
const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
cpuInstructions.forEach(() => {
Object.defineProperty(process, 'platform', {
value: 'darwin',
})
const settings: GpuSetting = {
...testSettings,
run_mode: 'cpu',
}
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('noavx')
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
executablePath:
originalPlatform === 'darwin'
? expect.stringContaining(`cortex-server`)
: expect.anything(),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
expect(engine.engineVariant(settings)).resolves.toEqual(
`linux-amd64-vulkan`
)
})
})

86
extensions/inference-cortex-extension/src/node/execute.ts → extensions/engine-management-extension/src/node/index.ts
View File

@ -1,13 +1,13 @@
import * as path from 'path'
import { GpuSetting, appResourcePath, log } from '@janhq/core/node'
import {
appResourcePath,
getJanDataFolderPath,
GpuSetting,
log,
} from '@janhq/core/node'
import { fork } from 'child_process'
import { mkdir, readdir, symlink } from 'fs/promises'
export interface CortexExecutableOptions {
enginePath: string
executablePath: string
cudaVisibleDevices: string
vkVisibleDevices: string
}
/**
* The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu.
* @param settings
@ -37,14 +37,6 @@ const os = (): string => {
: 'linux-amd64'
}
/**
* The cortex.cpp extension based on the current platform.
* @returns .exe if on Windows, otherwise an empty string.
*/
const extension = (): '.exe' | '' => {
return process.platform === 'win32' ? '.exe' : ''
}
/**
* The CUDA version that will be set - either '11-7' or '12-0'.
* @param settings
@ -89,30 +81,10 @@ const cpuInstructions = async (): Promise<string> => {
})
}
/**
* The executable options for the cortex.cpp extension.
*/
export const executableCortexFile = (
gpuSetting?: GpuSetting
): CortexExecutableOptions => {
let cudaVisibleDevices = gpuSetting?.gpus_in_use.join(',') ?? ''
let vkVisibleDevices = gpuSetting?.gpus_in_use.join(',') ?? ''
let binaryName = `cortex-server${extension()}`
const binPath = path.join(__dirname, '..', 'bin')
return {
enginePath: path.join(appResourcePath(), 'shared'),
executablePath: path.join(binPath, binaryName),
cudaVisibleDevices,
vkVisibleDevices,
}
}
/**
* Find which variant to run based on the current platform.
*/
export const engineVariant = async (
gpuSetting?: GpuSetting
): Promise<string> => {
const engineVariant = async (gpuSetting?: GpuSetting): Promise<string> => {
const cpuInstruction = await cpuInstructions()
log(`[CORTEX]: CPU instruction: ${cpuInstruction}`)
let engineVariant = [
@ -135,3 +107,45 @@ export const engineVariant = async (
log(`[CORTEX]: Engine variant: ${engineVariant}`)
return engineVariant
}
/**
* Create symlink to each variant for the default bundled version
*/
const symlinkEngines = async () => {
const sourceEnginePath = path.join(
appResourcePath(),
'shared',
'engines',
'cortex.llamacpp'
)
const symlinkEnginePath = path.join(
getJanDataFolderPath(),
'engines',
'cortex.llamacpp'
)
const variantFolders = await readdir(sourceEnginePath)
for (const variant of variantFolders) {
const targetVariantPath = path.join(
sourceEnginePath,
variant,
CORTEX_ENGINE_VERSION
)
const symlinkVariantPath = path.join(
symlinkEnginePath,
variant,
CORTEX_ENGINE_VERSION
)
await mkdir(path.join(symlinkEnginePath, variant), {
recursive: true,
}).catch(console.error)
await symlink(targetVariantPath, symlinkVariantPath).catch(console.error)
console.log(`Symlink created: ${targetVariantPath} -> ${symlinkEnginePath}`)
}
}
export default {
engineVariant,
symlinkEngines,
}

15
extensions/engine-management-extension/tsconfig.json Normal file
View File

@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"],
"exclude": ["src/**/*.test.ts", "rolldown.config.mjs"]
}

2
extensions/inference-cortex-extension/download.sh
View File

@ -30,8 +30,6 @@ if [ "$OS_TYPE" == "Linux" ]; then
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1
download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1
download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1
mkdir -p "${SHARED_PATH}/engines/cortex.llamacpp/deps"
touch "${SHARED_PATH}/engines/cortex.llamacpp/deps/keep"
elif [ "$OS_TYPE" == "Darwin" ]; then
# macOS downloads

4
extensions/inference-cortex-extension/package.json
View File

@ -47,7 +47,6 @@
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"cpu-instructions": "^0.0.13",
"decompress": "^4.2.1",
"fetch-retry": "^5.0.6",
"ky": "^1.7.2",
@ -69,8 +68,7 @@
"tcp-port-used",
"fetch-retry",
"@janhq/core",
"decompress",
"cpu-instructions"
"decompress"
],
"installConfig": {
"hoistingLimits": "workspaces"

10
extensions/inference-cortex-extension/src/@types/global.d.ts vendored
View File

@ -13,3 +13,13 @@ interface ModelOperationResponse {
error?: any
modelFile?: string
}
/**
* Cortex Executable Options Interface
*/
interface CortexExecutableOptions {
enginePath: string
executablePath: string
cudaVisibleDevices: string
vkVisibleDevices: string
}

70
extensions/inference-cortex-extension/src/index.ts
View File

@ -9,6 +9,7 @@
import {
Model,
executeOnMain,
EngineEvent,
systemInformation,
joinPath,
LocalOAIEngine,
@ -18,9 +19,7 @@ import {
fs,
events,
ModelEvent,
SystemInformation,
dirName,
AppConfigurationEventName,
} from '@janhq/core'
import PQueue from 'p-queue'
import ky from 'ky'
@ -112,21 +111,11 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
const systemInfo = await systemInformation()
this.queue.add(() => executeOnMain(NODE, 'run', systemInfo))
this.queue.add(() => this.healthz())
this.queue.add(() => this.setDefaultEngine(systemInfo))
this.subscribeToEvents()
window.addEventListener('beforeunload', () => {
this.clean()
})
const currentMode = systemInfo.gpuSetting?.run_mode
events.on(AppConfigurationEventName.OnConfigurationUpdate, async () => {
const systemInfo = await systemInformation()
// Update run mode on settings update
if (systemInfo.gpuSetting?.run_mode !== currentMode)
this.queue.add(() => this.setDefaultEngine(systemInfo))
})
}
async onUnload() {
@ -236,7 +225,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* Do health check on cortex.cpp
* @returns
*/
private healthz(): Promise<void> {
private async healthz(): Promise<void> {
return ky
.get(`${CORTEX_API_URL}/healthz`, {
retry: {
@ -248,36 +237,11 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
.then(() => {})
}
/**
* Set default engine variant on launch
*/
private async setDefaultEngine(systemInfo: SystemInformation) {
const variant = await executeOnMain(
NODE,
'engineVariant',
systemInfo.gpuSetting
)
return (
ky
// Fallback support for legacy API
.post(
`${CORTEX_API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/default?version=${CORTEX_ENGINE_VERSION}&variant=${variant}`,
{
json: {
version: CORTEX_ENGINE_VERSION,
variant,
},
}
)
.then(() => {})
)
}
/**
* Clean cortex processes
* @returns
*/
private clean(): Promise<any> {
private async clean(): Promise<any> {
return ky
.delete(`${CORTEX_API_URL}/processmanager/destroy`, {
timeout: 2000, // maximum 2 seconds
@ -301,6 +265,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
this.socket.addEventListener('message', (event) => {
const data = JSON.parse(event.data)
const transferred = data.task.items.reduce(
(acc: number, cur: any) => acc + cur.downloadedBytes,
0
@ -320,17 +285,26 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
transferred: transferred,
total: total,
},
downloadType: data.task.type,
}
)
// Update models list from Hub
if (data.type === DownloadTypes.DownloadSuccess) {
// Delay for the state update from cortex.cpp
// Just to be sure
setTimeout(() => {
events.emit(ModelEvent.OnModelsUpdate, {
fetch: true,
})
}, 500)
if (data.task.type === 'Engine') {
events.emit(EngineEvent.OnEngineUpdate, {
type: DownloadTypes[data.type as keyof typeof DownloadTypes],
percent: percent,
id: data.task.id,
})
} else {
if (data.type === DownloadTypes.DownloadSuccess) {
// Delay for the state update from cortex.cpp
// Just to be sure
setTimeout(() => {
events.emit(ModelEvent.OnModelsUpdate, {
fetch: true,
})
}, 500)
}
}
})

11
extensions/inference-cortex-extension/src/node/index.test.ts
View File

@ -54,17 +54,6 @@ jest.mock('child_process', () => ({
},
}))
jest.mock('./execute', () => ({
executableCortexFile: () => {
return {
enginePath: 'enginePath',
executablePath: 'executablePath',
cudaVisibleDevices: 'cudaVisibleDevices',
vkVisibleDevices: 'vkVisibleDevices',
}
},
}))
import index from './index'
describe('dispose', () => {

31
extensions/inference-cortex-extension/src/node/index.ts
View File

@ -1,6 +1,5 @@
import path from 'path'
import { getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node'
import { engineVariant, executableCortexFile } from './execute'
import { ProcessWatchdog } from './watchdog'
// The HOST address to use for the Nitro subprocess
@ -15,21 +14,12 @@ function run(systemInfo?: SystemInformation): Promise<any> {
log(`[CORTEX]:: Spawning cortex subprocess...`)
return new Promise<void>(async (resolve, reject) => {
let executableOptions = executableCortexFile(
// If ngl is not set or equal to 0, run on CPU with correct instructions
systemInfo?.gpuSetting
? {
...systemInfo.gpuSetting,
run_mode: systemInfo.gpuSetting.run_mode,
}
: undefined
)
let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? ''
let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}`
const binPath = path.join(__dirname, '..', 'bin')
const executablePath = path.join(binPath, binaryName)
// Execute the binary
log(`[CORTEX]:: Spawn cortex at path: ${executableOptions.executablePath}`)
log(`[CORTEX]:: Cortex engine path: ${executableOptions.enginePath}`)
addEnvPaths(executableOptions.enginePath)
log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`)
const dataFolderPath = getJanDataFolderPath()
if (watchdog) {
@ -37,7 +27,7 @@ function run(systemInfo?: SystemInformation): Promise<any> {
}
watchdog = new ProcessWatchdog(
executableOptions.executablePath,
executablePath,
[
'--start-server',
'--port',
@ -48,14 +38,12 @@ function run(systemInfo?: SystemInformation): Promise<any> {
dataFolderPath,
],
{
cwd: executableOptions.enginePath,
env: {
...process.env,
ENGINE_PATH: executableOptions.enginePath,
CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices,
CUDA_VISIBLE_DEVICES: gpuVisibleDevices,
// Vulkan - Support 1 device at a time for now
...(executableOptions.vkVisibleDevices?.length > 0 && {
GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0],
...(gpuVisibleDevices?.length > 0 && {
GGML_VK_VISIBLE_DEVICES: gpuVisibleDevices,
}),
},
}
@ -96,5 +84,4 @@ export interface CortexProcessInfo {
export default {
run,
dispose,
engineVariant,
}

2
extensions/model-extension/package.json
View File

@ -2,7 +2,7 @@
"name": "@janhq/model-extension",
"productName": "Model Management",
"version": "1.0.35",
"description": "Model Management Extension provides model exploration and seamless downloads",
"description": "This extension manages model lists, model details, and model configurations",
"main": "dist/index.js",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",

296
extensions/yarn.lock
View File

@ -518,6 +518,34 @@ __metadata:
languageName: node
linkType: hard
"@emnapi/core@npm:^1.3.1":
version: 1.3.1
resolution: "@emnapi/core@npm:1.3.1"
dependencies:
"@emnapi/wasi-threads": "npm:1.0.1"
tslib: "npm:^2.4.0"
checksum: 10c0/d3be1044ad704e2c486641bc18908523490f28c7d38bd12d9c1d4ce37d39dae6c4aecd2f2eaf44c6e3bd90eaf04e0591acc440b1b038cdf43cce078a355a0ea0
languageName: node
linkType: hard
"@emnapi/runtime@npm:^1.3.1":
version: 1.3.1
resolution: "@emnapi/runtime@npm:1.3.1"
dependencies:
tslib: "npm:^2.4.0"
checksum: 10c0/060ffede50f1b619c15083312b80a9e62a5b0c87aa8c1b54854c49766c9d69f8d1d3d87bd963a647071263a320db41b25eaa50b74d6a80dcc763c23dbeaafd6c
languageName: node
linkType: hard
"@emnapi/wasi-threads@npm:1.0.1":
version: 1.0.1
resolution: "@emnapi/wasi-threads@npm:1.0.1"
dependencies:
tslib: "npm:^2.4.0"
checksum: 10c0/1e0c8036b8d53e9b07cc9acf021705ef6c86ab6b13e1acda7fffaf541a2d3565072afb92597419173ced9ea14f6bf32fce149106e669b5902b825e8b499e5c6c
languageName: node
linkType: hard
"@isaacs/cliui@npm:^8.0.2":
version: 8.0.2
resolution: "@isaacs/cliui@npm:8.0.2"
@ -606,154 +634,183 @@ __metadata:
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-anthropic-extension%40workspace%3Ainference-anthropic-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-anthropic-extension%40workspace%3Ainference-anthropic-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-anthropic-extension%40workspace%3Ainference-anthropic-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-cohere-extension%40workspace%3Ainference-cohere-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-cohere-extension%40workspace%3Ainference-cohere-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-cohere-extension%40workspace%3Ainference-cohere-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-groq-extension%40workspace%3Ainference-groq-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-groq-extension%40workspace%3Ainference-groq-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-groq-extension%40workspace%3Ainference-groq-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-martian-extension%40workspace%3Ainference-martian-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-martian-extension%40workspace%3Ainference-martian-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-martian-extension%40workspace%3Ainference-martian-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-mistral-extension%40workspace%3Ainference-mistral-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-mistral-extension%40workspace%3Ainference-mistral-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-mistral-extension%40workspace%3Ainference-mistral-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-nvidia-extension%40workspace%3Ainference-nvidia-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-nvidia-extension%40workspace%3Ainference-nvidia-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-nvidia-extension%40workspace%3Ainference-nvidia-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-openai-extension%40workspace%3Ainference-openai-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-openai-extension%40workspace%3Ainference-openai-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-openai-extension%40workspace%3Ainference-openai-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-openrouter-extension%40workspace%3Ainference-openrouter-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-openrouter-extension%40workspace%3Ainference-openrouter-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-openrouter-extension%40workspace%3Ainference-openrouter-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-triton-trt-llm-extension%40workspace%3Ainference-triton-trtllm-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Finference-triton-trt-llm-extension%40workspace%3Ainference-triton-trtllm-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Finference-triton-trt-llm-extension%40workspace%3Ainference-triton-trtllm-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Ftensorrt-llm-extension%40workspace%3Atensorrt-llm-extension":
version: 0.1.10
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=14cf2e&locator=%40janhq%2Ftensorrt-llm-extension%40workspace%3Atensorrt-llm-extension"
resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=048f32&locator=%40janhq%2Ftensorrt-llm-extension%40workspace%3Atensorrt-llm-extension"
dependencies:
rxjs: "npm:^7.8.1"
ulidx: "npm:^2.3.0"
checksum: 10c0/d928ad13f353990eefcfbaa13539e49416d396c1bc3a5826f052942b04c266eb2d1335e072302b11e3d17cc1098b8026aee650105a4c9412f271383fc389a3a1
checksum: 10c0/b48796ca697fffa5aeb33b5c20927a2c3c0b6080a17fccdfa4030919baa8d5c9e3d68d45f87da1eda7842285d7b642361f3793af8aac8d4399c5b13d940a6a42
languageName: node
linkType: hard
"@janhq/engine-management-extension@workspace:engine-management-extension":
version: 0.0.0-use.local
resolution: "@janhq/engine-management-extension@workspace:engine-management-extension"
dependencies:
"@janhq/core": ../../core/package.tgz
"@rollup/plugin-replace": "npm:^6.0.2"
cpu-instructions: "npm:^0.0.13"
cpx: "npm:^1.5.0"
ky: "npm:^1.7.2"
p-queue: "npm:^8.0.1"
rimraf: "npm:^3.0.2"
rolldown: "npm:^1.0.0-beta.1"
ts-loader: "npm:^9.5.0"
typescript: "npm:^5.3.3"
webpack: "npm:^5.88.2"
webpack-cli: "npm:^5.1.4"
languageName: unknown
linkType: soft
"@janhq/inference-anthropic-extension@workspace:inference-anthropic-extension":
version: 0.0.0-use.local
resolution: "@janhq/inference-anthropic-extension@workspace:inference-anthropic-extension"
@ -802,7 +859,6 @@ __metadata:
"@types/node": "npm:^20.11.4"
"@types/os-utils": "npm:^0.0.4"
"@types/tcp-port-used": "npm:^1.0.4"
cpu-instructions: "npm:^0.0.13"
cpx: "npm:^1.5.0"
decompress: "npm:^4.2.1"
download-cli: "npm:^1.1.1"
@ -1907,6 +1963,17 @@ __metadata:
languageName: node
linkType: hard
"@napi-rs/wasm-runtime@npm:^0.2.4":
version: 0.2.6
resolution: "@napi-rs/wasm-runtime@npm:0.2.6"
dependencies:
"@emnapi/core": "npm:^1.3.1"
"@emnapi/runtime": "npm:^1.3.1"
"@tybys/wasm-util": "npm:^0.9.0"
checksum: 10c0/f921676c1d5c75494bd704c6c0837fd05fe95f5d1cb7373e32987ef5e00c3a1e90b5052352bd4b60ee20c3fe592af2dbba3b0de0c637218c25590828dbc4067e
languageName: node
linkType: hard
"@npmcli/agent@npm:^3.0.0":
version: 3.0.0
resolution: "@npmcli/agent@npm:3.0.0"
@ -1936,6 +2003,92 @@ __metadata:
languageName: node
linkType: hard
"@rolldown/binding-darwin-arm64@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-darwin-arm64@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=darwin & cpu=arm64
languageName: node
linkType: hard
"@rolldown/binding-darwin-x64@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-darwin-x64@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
"@rolldown/binding-freebsd-x64@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-freebsd-x64@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=freebsd & cpu=x64
languageName: node
linkType: hard
"@rolldown/binding-linux-arm-gnueabihf@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-linux-arm-gnueabihf@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=linux & cpu=arm
languageName: node
linkType: hard
"@rolldown/binding-linux-arm64-gnu@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-linux-arm64-gnu@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=linux & cpu=arm64 & libc=glibc
languageName: node
linkType: hard
"@rolldown/binding-linux-arm64-musl@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-linux-arm64-musl@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=linux & cpu=arm64 & libc=musl
languageName: node
linkType: hard
"@rolldown/binding-linux-x64-gnu@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-linux-x64-gnu@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=linux & cpu=x64 & libc=glibc
languageName: node
linkType: hard
"@rolldown/binding-linux-x64-musl@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-linux-x64-musl@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=linux & cpu=x64 & libc=musl
languageName: node
linkType: hard
"@rolldown/binding-wasm32-wasi@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-wasm32-wasi@npm:1.0.0-beta.1-commit.f90856a"
dependencies:
"@napi-rs/wasm-runtime": "npm:^0.2.4"
conditions: cpu=wasm32
languageName: node
linkType: hard
"@rolldown/binding-win32-arm64-msvc@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-win32-arm64-msvc@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=win32 & cpu=arm64
languageName: node
linkType: hard
"@rolldown/binding-win32-ia32-msvc@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-win32-ia32-msvc@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=win32 & cpu=ia32
languageName: node
linkType: hard
"@rolldown/binding-win32-x64-msvc@npm:1.0.0-beta.1-commit.f90856a":
version: 1.0.0-beta.1-commit.f90856a
resolution: "@rolldown/binding-win32-x64-msvc@npm:1.0.0-beta.1-commit.f90856a"
conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
"@rollup/plugin-commonjs@npm:^25.0.7":
version: 25.0.8
resolution: "@rollup/plugin-commonjs@npm:25.0.8"
@ -2002,6 +2155,21 @@ __metadata:
languageName: node
linkType: hard
"@rollup/plugin-replace@npm:^6.0.2":
version: 6.0.2
resolution: "@rollup/plugin-replace@npm:6.0.2"
dependencies:
"@rollup/pluginutils": "npm:^5.0.1"
magic-string: "npm:^0.30.3"
peerDependencies:
rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0
peerDependenciesMeta:
rollup:
optional: true
checksum: 10c0/71c0dea46f560c8dff59853446d43fa0e8258139a74d2af09fce5790d0540ff3d874c8fd9962cb049577d25327262bfc97485ef90b2a0a21bf28a9d3bd8c6d44
languageName: node
linkType: hard
"@rollup/plugin-typescript@npm:^11.1.6":
version: 11.1.6
resolution: "@rollup/plugin-typescript@npm:11.1.6"
@ -2085,6 +2253,15 @@ __metadata:
languageName: node
linkType: hard
"@tybys/wasm-util@npm:^0.9.0":
version: 0.9.0
resolution: "@tybys/wasm-util@npm:0.9.0"
dependencies:
tslib: "npm:^2.4.0"
checksum: 10c0/f9fde5c554455019f33af6c8215f1a1435028803dc2a2825b077d812bed4209a1a64444a4ca0ce2ea7e1175c8d88e2f9173a36a33c199e8a5c671aa31de8242d
languageName: node
linkType: hard
"@types/babel__core@npm:^7.1.14":
version: 7.20.5
resolution: "@types/babel__core@npm:7.20.5"
@ -7837,6 +8014,59 @@ __metadata:
languageName: node
linkType: hard
"rolldown@npm:^1.0.0-beta.1":
version: 1.0.0-beta.1-commit.f90856a
resolution: "rolldown@npm:1.0.0-beta.1-commit.f90856a"
dependencies:
"@rolldown/binding-darwin-arm64": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-darwin-x64": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-freebsd-x64": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-linux-arm-gnueabihf": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-linux-arm64-gnu": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-linux-arm64-musl": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-linux-x64-gnu": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-linux-x64-musl": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-wasm32-wasi": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-win32-arm64-msvc": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-win32-ia32-msvc": "npm:1.0.0-beta.1-commit.f90856a"
"@rolldown/binding-win32-x64-msvc": "npm:1.0.0-beta.1-commit.f90856a"
zod: "npm:^3.23.8"
peerDependencies:
"@babel/runtime": ">=7"
dependenciesMeta:
"@rolldown/binding-darwin-arm64":
optional: true
"@rolldown/binding-darwin-x64":
optional: true
"@rolldown/binding-freebsd-x64":
optional: true
"@rolldown/binding-linux-arm-gnueabihf":
optional: true
"@rolldown/binding-linux-arm64-gnu":
optional: true
"@rolldown/binding-linux-arm64-musl":
optional: true
"@rolldown/binding-linux-x64-gnu":
optional: true
"@rolldown/binding-linux-x64-musl":
optional: true
"@rolldown/binding-wasm32-wasi":
optional: true
"@rolldown/binding-win32-arm64-msvc":
optional: true
"@rolldown/binding-win32-ia32-msvc":
optional: true
"@rolldown/binding-win32-x64-msvc":
optional: true
peerDependenciesMeta:
"@babel/runtime":
optional: true
bin:
rolldown: bin/cli.js
checksum: 10c0/e9018052d305374b85b330357c65ad41ee94c3dd4f92827f03eaddc6998fb9fb6611fa21be248f1a18d233dfca9c6d268e5305403a1d0d61789ae646555bded6
languageName: node
linkType: hard
"rollup-plugin-define@npm:^1.0.1":
version: 1.0.1
resolution: "rollup-plugin-define@npm:1.0.1"
@ -8799,7 +9029,7 @@ __metadata:
languageName: node
linkType: hard
"tslib@npm:^2.1.0, tslib@npm:^2.6.2":
"tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2":
version: 2.8.1
resolution: "tslib@npm:2.8.1"
checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62
@ -9376,7 +9606,7 @@ __metadata:
languageName: node
linkType: hard
"zod@npm:^3.22.3, zod@npm:^3.22.4":
"zod@npm:^3.22.3, zod@npm:^3.22.4, zod@npm:^3.23.8":
version: 3.24.1
resolution: "zod@npm:3.24.1"
checksum: 10c0/0223d21dbaa15d8928fe0da3b54696391d8e3e1e2d0283a1a070b5980a1dbba945ce631c2d1eccc088fdbad0f2dfa40155590bf83732d3ac4fcca2cc9237591b

6
web/containers/Providers/EventListener.tsx
View File

@ -108,6 +108,12 @@ const EventListener = () => {
...model.parameters,
} as Partial<Model>)
.catch((e) => console.debug(e))
toaster({
title: 'Download Completed',
description: `Download ${state.modelId} completed`,
type: 'success',
})
}
state.downloadState = 'end'
setDownloadState(state)

12
web/helpers/atoms/Extension.atom.ts
View File

@ -43,5 +43,15 @@ export const removeInstallingExtensionAtom = atom(
const INACTIVE_ENGINE_PROVIDER = 'inActiveEngineProvider'
export const inActiveEngineProviderAtom = atomWithStorage<string[]>(
INACTIVE_ENGINE_PROVIDER,
[]
[],
undefined,
{ getOnInit: true }
)
const SHOW_SETTING_ACTIVE_LOCAL_ENGINE = 'showSettingActiveLocalEngine'
export const showSettingActiveLocalEngineAtom = atomWithStorage<string[]>(
SHOW_SETTING_ACTIVE_LOCAL_ENGINE,
[],
undefined,
{ getOnInit: true }
)

5
web/hooks/useDownloadState.ts
View File

@ -54,11 +54,6 @@ export const setDownloadStateAtom = atom(
(e) => e.id === state.modelId
)
if (model) set(downloadedModelsAtom, (prev) => [...prev, model])
toaster({
title: 'Download Completed',
description: `Download ${state.modelId} completed`,
type: 'success',
})
}
} else if (state.downloadState === 'error') {
// download error

328
web/hooks/useEngineManagement.ts Normal file
View File

@ -0,0 +1,328 @@
import { useMemo } from 'react'
import {
ExtensionTypeEnum,
EngineManagementExtension,
InferenceEngine,
EngineReleased,
} from '@janhq/core'
import { useAtom } from 'jotai'
import { atomWithStorage } from 'jotai/utils'
import useSWR from 'swr'
import { extensionManager } from '@/extension/ExtensionManager'
export const releasedEnginesCacheAtom = atomWithStorage<{
data: EngineReleased[]
timestamp: number
} | null>('releasedEnginesCache', null, undefined, { getOnInit: true })
export const releasedEnginesLatestCacheAtom = atomWithStorage<{
data: EngineReleased[]
timestamp: number
} | null>('releasedEnginesLatestCache', null, undefined, { getOnInit: true })
// fetcher function
async function fetchExtensionData<T>(
extension: EngineManagementExtension | null,
method: (extension: EngineManagementExtension) => Promise<T>
): Promise<T> {
if (!extension) {
throw new Error('Extension not found')
}
return method(extension)
}
/**
* @returns A Promise that resolves to an object of list engines.
*/
export function useGetEngines() {
const extension = useMemo(
() =>
extensionManager.get<EngineManagementExtension>(
ExtensionTypeEnum.Engine
) ?? null,
[]
)
const {
data: engines,
error,
mutate,
} = useSWR(
extension ? 'engines' : null,
() => fetchExtensionData(extension, (ext) => ext.getEngines()),
{
revalidateOnFocus: false,
revalidateOnReconnect: true,
}
)
return { engines, error, mutate }
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an array of installed engine.
*/
export function useGetInstalledEngines(name: InferenceEngine) {
const extension = useMemo(
() =>
extensionManager.get<EngineManagementExtension>(
ExtensionTypeEnum.Engine
) ?? null,
[]
)
const {
data: installedEngines,
error,
mutate,
} = useSWR(
extension ? 'installedEngines' : null,
() => fetchExtensionData(extension, (ext) => ext.getInstalledEngines(name)),
{
revalidateOnFocus: false,
revalidateOnReconnect: true,
}
)
return { installedEngines, error, mutate }
}
/**
* @param name - Inference engine name.
* @param version - Version of the engine.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
export function useGetReleasedEnginesByVersion(
engine: InferenceEngine,
version: string | undefined,
platform: string
) {
const extension = useMemo(
() =>
extensionManager.get<EngineManagementExtension>(
ExtensionTypeEnum.Engine
) ?? null,
[]
)
const [cache, setCache] = useAtom(releasedEnginesCacheAtom)
const shouldFetch = Boolean(extension && version)
const fetcher = async () => {
const now = Date.now()
const fifteenMinutes = 15 * 60 * 1000
if (cache && cache.timestamp + fifteenMinutes > now) {
return cache.data // Use cached data
}
const newData = await fetchExtensionData(extension, (ext) =>
ext.getReleasedEnginesByVersion(engine, version!, platform)
)
setCache({ data: newData, timestamp: now })
return newData
}
const { data, error, mutate } = useSWR(
shouldFetch
? `releasedEnginesByVersion-${engine}-${version}-${platform}`
: null,
fetcher,
{
revalidateOnFocus: false,
revalidateOnReconnect: true,
}
)
return {
releasedEnginesByVersion: data,
error,
mutate,
}
}
/**
* @param name - Inference engine name.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine.
*/
export function useGetLatestReleasedEngine(
engine: InferenceEngine,
platform: string
) {
const extension = useMemo(
() =>
extensionManager.get<EngineManagementExtension>(
ExtensionTypeEnum.Engine
) ?? null,
[]
)
const [cache, setCache] = useAtom(releasedEnginesLatestCacheAtom)
const fetcher = async () => {
const now = Date.now()
const fifteenMinutes = 15 * 60 * 1000
if (cache && cache.timestamp + fifteenMinutes > now) {
return cache.data // Use cached data
}
const newData = await fetchExtensionData(extension, (ext) =>
ext.getLatestReleasedEngine(engine, platform)
)
setCache({ data: newData, timestamp: now })
return newData
}
const { data, error, mutate } = useSWR(
extension ? 'latestReleasedEngine' : null,
fetcher,
{
revalidateOnFocus: false,
revalidateOnReconnect: true,
}
)
return {
latestReleasedEngine: data,
error,
mutate,
}
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
*/
export function useGetDefaultEngineVariant(name: InferenceEngine) {
const extension = useMemo(
() =>
extensionManager.get<EngineManagementExtension>(ExtensionTypeEnum.Engine),
[]
)
const {
data: defaultEngineVariant,
error,
mutate,
} = useSWR(
extension ? 'defaultEngineVariant' : null,
() =>
fetchExtensionData(extension ?? null, (ext) =>
ext.getDefaultEngineVariant(name)
),
{
revalidateOnFocus: false,
revalidateOnReconnect: true,
}
)
return { defaultEngineVariant, error, mutate }
}
const getExtension = () =>
extensionManager.get<EngineManagementExtension>(ExtensionTypeEnum.Engine) ??
null
/**
* @body variant - string
* @body version - string
* @returns A Promise that resolves to set default engine.
*/
export const setDefaultEngineVariant = async (
name: InferenceEngine,
engineConfig: { variant: string; version: string }
) => {
const extension = getExtension()
if (!extension) {
throw new Error('Extension is not available')
}
try {
// Call the extension's method
const response = await extension.setDefaultEngineVariant(name, engineConfig)
return response
} catch (error) {
console.error('Failed to set default engine variant:', error)
throw error
}
}
/**
* @body variant - string
* @body version - string
* @returns A Promise that resolves to set default engine.
*/
export const updateEngine = async (name: InferenceEngine) => {
const extension = getExtension()
if (!extension) {
throw new Error('Extension is not available')
}
try {
// Call the extension's method
const response = await extension.updateEngine(name)
return response
} catch (error) {
console.error('Failed to set default engine variant:', error)
throw error
}
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to intall of engine.
*/
export const installEngine = async (
name: InferenceEngine,
engineConfig: { variant: string; version?: string }
) => {
const extension = getExtension()
if (!extension) {
throw new Error('Extension is not available')
}
try {
// Call the extension's method
const response = await extension.installEngine(name, engineConfig)
return response
} catch (error) {
console.error('Failed to install engine variant:', error)
throw error
}
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to unintall of engine.
*/
export const uninstallEngine = async (
name: InferenceEngine,
engineConfig: { variant: string; version: string }
) => {
const extension = getExtension()
if (!extension) {
throw new Error('Extension is not available')
}
try {
// Call the extension's method
const response = await extension.uninstallEngine(name, engineConfig)
return response
} catch (error) {
console.error('Failed to install engine variant:', error)
throw error
}
}

1
web/package.json
View File

@ -52,6 +52,7 @@
"slate-dom": "0.111.0",
"slate-history": "0.110.3",
"slate-react": "0.110.3",
"swr": "^2.2.5",
"tailwind-merge": "^2.0.0",
"tailwindcss": "3.3.5",
"ulidx": "^2.3.0",

90
web/screens/Settings/CoreExtensions/index.tsx
View File

@ -1,34 +1,23 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import React, { useState, useEffect, useRef, useCallback } from 'react'
import React, { useState, useEffect, useRef } from 'react'
import { InferenceEngine } from '@janhq/core'
import { Button, ScrollArea, Badge, Input } from '@janhq/joi'
import { Button, ScrollArea, Badge, Switch, Input } from '@janhq/joi'
import { useAtom } from 'jotai'
import { SearchIcon } from 'lucide-react'
import { Marked, Renderer } from 'marked'
import Loader from '@/containers/Loader'
import SetupRemoteModel from '@/containers/SetupRemoteModel'
import { formatExtensionsName } from '@/utils/converter'
import { extensionManager } from '@/extension'
import Extension from '@/extension/Extension'
import { inActiveEngineProviderAtom } from '@/helpers/atoms/Extension.atom'
type EngineExtension = {
provider: InferenceEngine
} & Extension
const ExtensionCatalog = () => {
const [coreActiveExtensions, setCoreActiveExtensions] = useState<Extension[]>(
[]
)
const [engineActiveExtensions, setEngineActiveExtensions] = useState<
EngineExtension[]
>([])
const [searchText, setSearchText] = useState('')
const [showLoading, setShowLoading] = useState(false)
const fileInputRef = useRef<HTMLInputElement | null>(null)
@ -67,7 +56,6 @@ const ExtensionCatalog = () => {
}
setCoreActiveExtensions(extensionsMenu)
setEngineActiveExtensions(engineMenu as any)
}
getAllSettings()
}, [])
@ -113,23 +101,6 @@ const ExtensionCatalog = () => {
}
}
const [inActiveEngineProvider, setInActiveEngineProvider] = useAtom(
inActiveEngineProviderAtom
)
const onSwitchChange = useCallback(
(name: string) => {
if (inActiveEngineProvider.includes(name)) {
setInActiveEngineProvider(
[...inActiveEngineProvider].filter((x) => x !== name)
)
} else {
setInActiveEngineProvider([...inActiveEngineProvider, name])
}
},
[inActiveEngineProvider, setInActiveEngineProvider]
)
return (
<>
<ScrollArea className="h-full w-full">
@ -158,61 +129,6 @@ const ExtensionCatalog = () => {
</div>
<div className="block w-full px-4">
{engineActiveExtensions.length !== 0 && (
<div className="mb-3 mt-4 border-b border-[hsla(var(--app-border))] pb-4">
<h6 className="text-base font-semibold text-[hsla(var(--text-primary))]">
Model Providers
</h6>
</div>
)}
{engineActiveExtensions
.filter((x) => x.name.includes(searchText.toLowerCase().trim()))
.sort((a, b) => a.provider.localeCompare(b.provider))
.map((item, i) => {
return (
<div
key={i}
className="flex w-full flex-col items-start justify-between py-3 sm:flex-row"
>
<div className="w-full flex-shrink-0 space-y-1.5">
<div className="flex items-center justify-between gap-x-2">
<div className="flex items-center gap-x-2">
<h6 className="line-clamp-1 font-semibold">
{item.productName?.replace('Inference Engine', '') ??
formatExtensionsName(item.name)}
</h6>
<Badge variant="outline" theme="secondary">
v{item.version}
</Badge>
<p>{item.provider}</p>
</div>
<div className="flex items-center gap-x-2">
{!inActiveEngineProvider.includes(item.provider) && (
<SetupRemoteModel engine={item.provider} />
)}
<Switch
checked={
!inActiveEngineProvider.includes(item.provider)
}
onChange={() => onSwitchChange(item.provider)}
/>
</div>
</div>
{
<div
className="w-full font-medium leading-relaxed text-[hsla(var(--text-secondary))] sm:w-4/5"
dangerouslySetInnerHTML={{
__html: marked.parse(item.description ?? '', {
async: false,
}),
}}
/>
}
</div>
</div>
)
})}
{coreActiveExtensions.length > 0 && (
<div className="mb-3 mt-8 border-b border-[hsla(var(--app-border))] pb-4">
<h6 className="text-base font-semibold text-[hsla(var(--text-primary))]">

77
web/screens/Settings/Engines/DeleteEngineVariant.tsx Normal file
View File

@ -0,0 +1,77 @@
import { memo, useState } from 'react'
import { EngineReleased, InferenceEngine } from '@janhq/core'
import { Button, Modal, ModalClose } from '@janhq/joi'
import { Trash2Icon } from 'lucide-react'
import {
uninstallEngine,
useGetDefaultEngineVariant,
useGetInstalledEngines,
} from '@/hooks/useEngineManagement'
const DeleteEngineVariant = ({
variant,
engine,
}: {
variant: EngineReleased
engine: InferenceEngine
}) => {
const [open, setOpen] = useState(false)
const { mutate: mutateInstalledEngines } = useGetInstalledEngines(engine)
const { defaultEngineVariant, mutate: mutateDefaultEngineVariant } =
useGetDefaultEngineVariant(engine)
return (
<Modal
title={<span>Delete {variant.name}</span>}
open={open}
onOpenChange={() => setOpen(!open)}
trigger={
<Button theme="icon" variant="outline" onClick={() => setOpen(!open)}>
<Trash2Icon
size={14}
className="text-[hsla(var(--text-secondary))]"
/>
</Button>
}
content={
<div>
<p className="text-[hsla(var(--text-secondary))]">
Are you sure you want to delete this variant?
</p>
<div className="mt-4 flex justify-end gap-x-2">
<ModalClose
asChild
onClick={(e) => {
setOpen(!open)
e.stopPropagation()
}}
>
<Button theme="ghost">No</Button>
</ModalClose>
<ModalClose asChild>
<Button
theme="destructive"
onClick={() => {
uninstallEngine(engine, {
variant: variant.name,
version: String(defaultEngineVariant?.version),
})
mutateInstalledEngines()
}}
autoFocus
>
Yes
</Button>
</ModalClose>
</div>
</div>
}
/>
)
}
export default memo(DeleteEngineVariant)

349
web/screens/Settings/Engines/Settings.tsx Normal file
View File

@ -0,0 +1,349 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import React, { useCallback, useEffect, useMemo, useState } from 'react'
import {
DownloadEvent,
EngineEvent,
events,
InferenceEngine,
} from '@janhq/core'
import { Button, ScrollArea, Badge, Select, Progress } from '@janhq/joi'
import { Trash2Icon } from 'lucide-react'
import { twMerge } from 'tailwind-merge'
import {
useGetDefaultEngineVariant,
useGetInstalledEngines,
useGetLatestReleasedEngine,
setDefaultEngineVariant,
installEngine,
updateEngine,
uninstallEngine,
useGetReleasedEnginesByVersion,
} from '@/hooks/useEngineManagement'
import { formatDownloadPercentage } from '@/utils/converter'
const os = () => {
switch (PLATFORM) {
case 'win32':
return 'windows'
case 'linux':
return 'linux'
default:
return 'mac'
}
}
const EngineSettings = ({ engine }: { engine: InferenceEngine }) => {
const { installedEngines, mutate: mutateInstalledEngines } =
useGetInstalledEngines(engine)
const { defaultEngineVariant, mutate: mutateDefaultEngineVariant } =
useGetDefaultEngineVariant(engine)
const { latestReleasedEngine } = useGetLatestReleasedEngine(engine, os())
const { releasedEnginesByVersion } = useGetReleasedEnginesByVersion(
engine,
defaultEngineVariant?.version as string,
os()
)
const [installingEngines, setInstallingEngines] = useState<
Map<string, number>
>(new Map())
const isEngineUpdated =
latestReleasedEngine &&
latestReleasedEngine.every((item) =>
item.name.includes(
defaultEngineVariant?.version.replace(/^v/, '') as string
)
)
const availableVariants = useMemo(
() =>
latestReleasedEngine?.map((e) =>
e.name.replace(
`${defaultEngineVariant?.version.replace(/^v/, '') as string}-`,
''
)
),
[latestReleasedEngine, defaultEngineVariant]
)
const options =
installedEngines &&
installedEngines
.filter((x: any) => x.version === defaultEngineVariant?.version)
.map((x: any) => ({
name: x.name,
value: x.name,
}))
const installedEngineByVersion = installedEngines?.filter(
(x: any) => x.version === defaultEngineVariant?.version
)
const [selectedVariants, setSelectedVariants] = useState(
defaultEngineVariant?.variant
)
const selectedVariant = useMemo(
() =>
options?.map((e) => e.value).includes(selectedVariants)
? selectedVariants
: undefined,
[selectedVariants, options]
)
useEffect(() => {
if (defaultEngineVariant?.variant) {
setSelectedVariants(defaultEngineVariant.variant || '')
}
}, [defaultEngineVariant])
const handleEngineUpdate = useCallback(
(event: { id: string; type: DownloadEvent; percent: number }) => {
mutateInstalledEngines()
mutateDefaultEngineVariant()
// Backward compatible support - cortex.cpp returns full variant file name
const variant: string | undefined = event.id.includes('.tar.gz')
? availableVariants?.find((e) => event.id.includes(`${e}.tar.gz`))
: availableVariants?.find((e) => event.id.includes(e))
if (!variant) return
setInstallingEngines((prev) => {
prev.set(variant, event.percent)
return prev
})
if (
event.type === DownloadEvent.onFileDownloadError ||
event.type === DownloadEvent.onFileDownloadStopped ||
event.type === DownloadEvent.onFileDownloadSuccess
) {
setInstallingEngines((prev) => {
prev.delete(variant)
return prev
})
}
},
[
mutateDefaultEngineVariant,
mutateInstalledEngines,
setInstallingEngines,
availableVariants,
]
)
useEffect(() => {
events.on(EngineEvent.OnEngineUpdate, handleEngineUpdate)
return () => {
events.off(EngineEvent.OnEngineUpdate, handleEngineUpdate)
}
}, [handleEngineUpdate])
const handleChangeVariant = (e: string) => {
setSelectedVariants(e)
setDefaultEngineVariant(engine, {
variant: e,
version: String(defaultEngineVariant?.version),
})
}
return (
<ScrollArea className="h-full w-full">
<div className="block w-full px-4">
<div className="mb-3 mt-4 border-b border-[hsla(var(--app-border))] pb-4">
<div className="flex w-full flex-col items-start justify-between sm:flex-row">
<div className="w-full flex-shrink-0 space-y-1.5">
<div className="flex items-center justify-between gap-x-2">
<div>
<h6 className="line-clamp-1 font-semibold">Engine Version</h6>
</div>
<div className="flex items-center gap-x-3">
<Badge variant="outline" theme="secondary">
{defaultEngineVariant?.version}
</Badge>
</div>
</div>
</div>
</div>
</div>
</div>
<div className="block w-full px-4">
<div className="mb-3 mt-4 border-b border-[hsla(var(--app-border))] pb-4">
<div className="flex w-full flex-col items-start justify-between sm:flex-row">
<div className="w-full flex-shrink-0 space-y-1.5">
<div className="flex items-center justify-between gap-x-2">
<div>
<h6 className="line-clamp-1 font-semibold">Check Updates</h6>
</div>
<div className="flex items-center gap-x-3">
<Button
disabled={isEngineUpdated}
onClick={() => {
updateEngine(engine)
}}
>
{!isEngineUpdated ? 'Update now' : 'Updated'}
</Button>
</div>
</div>
</div>
</div>
</div>
</div>
<div className="block w-full px-4">
<div className="mb-3 mt-4 border-b border-[hsla(var(--app-border))] pb-4">
<div className="flex w-full flex-col items-start justify-between sm:flex-row">
<div className="w-full flex-shrink-0 space-y-1.5">
<div className="flex items-center justify-between gap-x-4">
<div>
<h6 className="line-clamp-1 font-semibold">
{engine} Backend
</h6>
<div className="mt-2 w-full font-medium leading-relaxed text-[hsla(var(--text-secondary))]">
<p>
Choose the default variant that best suited for your
hardware. See more information here.
</p>
</div>
</div>
<div className="flex flex-shrink-0 items-center gap-x-3">
<div className="flex w-full min-w-[180px]">
<Select
value={selectedVariant}
placeholder="Select variant"
onValueChange={(e) => handleChangeVariant(e)}
options={options}
block
/>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div className="block w-full px-4">
<div className="mb-3 mt-4 pb-4">
<div className="flex w-full flex-col items-start justify-between sm:flex-row">
<div className="w-full flex-shrink-0 ">
<div className="flex items-center justify-between gap-x-2">
<div>
<h6 className="mb-2 line-clamp-1 font-semibold">Backends</h6>
</div>
</div>
<div>
{releasedEnginesByVersion &&
releasedEnginesByVersion?.map((item, i) => {
return (
<div
key={i}
className={twMerge(
'border border-b-0 border-[hsla(var(--app-border))] bg-[hsla(var(--tertiary-bg))] p-4 first:rounded-t-lg last:rounded-b-lg last:border-b',
releasedEnginesByVersion?.length === 1 && 'rounded-lg'
)}
>
<div className="flex flex-col items-start justify-start gap-4 sm:flex-row sm:items-center sm:justify-between">
<div className="flex w-full gap-x-8">
<div className="flex h-full w-full items-center justify-between gap-2">
<h6
className={twMerge(
'font-medium lg:line-clamp-1 lg:min-w-[280px] lg:max-w-[280px]',
'max-w-none text-[hsla(var(--text-secondary))]'
)}
>
{item.name}
</h6>
{installedEngineByVersion?.some(
(x) => x.name === item.name
) ? (
<Button
theme="icon"
variant="outline"
onClick={() => {
uninstallEngine(engine, {
variant: item.name,
version: String(
defaultEngineVariant?.version
),
})
if (selectedVariants === item.name) {
setSelectedVariants('')
}
mutateInstalledEngines()
}}
>
<Trash2Icon
size={14}
className="text-[hsla(var(--text-secondary))]"
/>
</Button>
) : (
<>
{installingEngines.has(item.name) ? (
<Button variant="soft">
<div className="flex items-center space-x-2">
<Progress
className="inline-block h-2 w-[80px]"
value={
formatDownloadPercentage(
installingEngines.get(
item.name
) ?? 0,
{
hidePercentage: true,
}
) as number
}
/>
<span className="tabular-nums">
{formatDownloadPercentage(
installingEngines.get(item.name) ??
0
)}
</span>
</div>
</Button>
) : (
<Button
variant="soft"
onClick={() => {
setInstallingEngines((prev) => {
prev.set(item.name, 0)
return prev
})
installEngine(engine, {
variant: item.name,
version: String(
defaultEngineVariant?.version
),
}).then(() => {
if (selectedVariants === '') {
setSelectedVariants(item.name)
}
})
}}
>
Download
</Button>
)}
</>
)}
</div>
</div>
</div>
</div>
)
})}
</div>
</div>
</div>
</div>
</div>
</ScrollArea>
)
}
export default EngineSettings

261
web/screens/Settings/Engines/index.tsx Normal file
View File

@ -0,0 +1,261 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import React, { useCallback, useEffect, useState } from 'react'
import { InferenceEngine } from '@janhq/core'
import { Button, ScrollArea, Input, Switch, Badge } from '@janhq/joi'
import { useAtom, useSetAtom } from 'jotai'
import { SearchIcon, SettingsIcon } from 'lucide-react'
import { marked } from 'marked'
import SetupRemoteModel from '@/containers/SetupRemoteModel'
import {
useGetDefaultEngineVariant,
useGetEngines,
} from '@/hooks/useEngineManagement'
import { formatExtensionsName } from '@/utils/converter'
import { extensionManager } from '@/extension'
import Extension from '@/extension/Extension'
import {
inActiveEngineProviderAtom,
showSettingActiveLocalEngineAtom,
} from '@/helpers/atoms/Extension.atom'
import { selectedSettingAtom } from '@/helpers/atoms/Setting.atom'
type EngineExtension = {
provider: InferenceEngine
} & Extension
const EngineItems = ({ engine }: { engine: InferenceEngine }) => {
const { defaultEngineVariant } = useGetDefaultEngineVariant(engine)
const manualDescription = (engine: string) => {
switch (engine) {
case InferenceEngine.cortex_llamacpp:
return 'Fast, efficient local inference engine that runs GGUFmodels directly on your device'
default:
break
}
}
const setSelectedSetting = useSetAtom(selectedSettingAtom)
const [showSettingActiveLocalEngine, setShowSettingActiveLocalEngineAtom] =
useAtom(showSettingActiveLocalEngineAtom)
const onSwitchChange = useCallback(
(name: string) => {
if (showSettingActiveLocalEngine.includes(name)) {
setShowSettingActiveLocalEngineAtom(
[...showSettingActiveLocalEngine].filter((x) => x !== name)
)
} else {
setShowSettingActiveLocalEngineAtom([
...showSettingActiveLocalEngine,
name,
])
}
},
[showSettingActiveLocalEngine, setShowSettingActiveLocalEngineAtom]
)
return (
<div className="flex w-full flex-col items-start justify-between border-b border-[hsla(var(--app-border))] py-3 sm:flex-row">
<div className="w-full flex-shrink-0 space-y-1.5">
<div className="flex items-center justify-between gap-x-2">
<div>
<div className="flex items-center gap-2">
<h6 className="line-clamp-1 font-semibold">{engine}</h6>
<Badge variant="outline" theme="secondary">
{defaultEngineVariant?.version}
</Badge>
</div>
<div className="mt-2 w-full font-medium leading-relaxed text-[hsla(var(--text-secondary))]">
<p>{manualDescription(engine)}</p>
</div>
</div>
<div className="flex items-center gap-x-3">
<Switch
checked={!showSettingActiveLocalEngine.includes(engine)}
onChange={() => onSwitchChange(engine)}
/>
<Button
theme="icon"
variant="outline"
onClick={() => setSelectedSetting(engine)}
>
<SettingsIcon
size={14}
className="text-[hsla(var(--text-secondary))]"
/>
</Button>
</div>
</div>
</div>
</div>
)
}
const Engines = () => {
const [searchText, setSearchText] = useState('')
const { engines } = useGetEngines()
const [engineActiveExtensions, setEngineActiveExtensions] = useState<
EngineExtension[]
>([])
const [inActiveEngineProvider, setInActiveEngineProvider] = useAtom(
inActiveEngineProviderAtom
)
useEffect(() => {
const getAllSettings = async () => {
const extensionsMenu = []
const engineMenu = []
const extensions = extensionManager.getAll()
for (const extension of extensions) {
const settings = await extension.getSettings()
if (
typeof extension.getSettings === 'function' &&
'provider' in extension &&
typeof extension.provider === 'string'
) {
if (
(settings && settings.length > 0) ||
(await extension.installationState()) !== 'NotRequired'
) {
engineMenu.push({
...extension,
provider:
'provider' in extension &&
typeof extension.provider === 'string'
? extension.provider
: '',
})
}
} else {
extensionsMenu.push({
...extension,
})
}
}
setEngineActiveExtensions(engineMenu as any)
}
getAllSettings()
}, [])
const onSwitchChange = useCallback(
(name: string) => {
if (inActiveEngineProvider.includes(name)) {
setInActiveEngineProvider(
[...inActiveEngineProvider].filter((x) => x !== name)
)
} else {
setInActiveEngineProvider([...inActiveEngineProvider, name])
}
},
[inActiveEngineProvider, setInActiveEngineProvider]
)
return (
<ScrollArea className="h-full w-full">
<div className="flex w-full flex-col items-start justify-between gap-y-2 p-4 sm:flex-row">
<div className="w-full sm:w-[300px]">
<Input
prefixIcon={<SearchIcon size={16} />}
placeholder="Search"
value={searchText}
onChange={(e) => setSearchText(e.target.value)}
clearable={searchText.length > 0}
onClear={() => setSearchText('')}
/>
</div>
{/* <div>
<input type="file" hidden />
<Button>Install Engine</Button>
</div> */}
</div>
<div className="block w-full px-4">
<div className="mb-3 mt-4 pb-4">
<h6 className="text-xs text-[hsla(var(--text-secondary))]">
Local Engine
</h6>
{engines &&
Object.entries(engines)
// eslint-disable-next-line @typescript-eslint/no-unused-vars
.filter(([_, value]) => !(value as { type?: string }).type)
.map(([key]) => {
return <EngineItems engine={key as InferenceEngine} key={key} />
})}
</div>
</div>
{engineActiveExtensions.length !== 0 && (
<div className="mt-4 block w-full px-4">
<div className="mb-3 mt-4 pb-4">
<h6 className="text-xs text-[hsla(var(--text-secondary))]">
Remote Engine
</h6>
{engineActiveExtensions
.filter((x) => x.name.includes(searchText.toLowerCase().trim()))
.sort((a, b) => a.provider.localeCompare(b.provider))
.map((item, i) => {
return (
<div
key={i}
className="flex w-full flex-col items-start justify-between border-b border-[hsla(var(--app-border))] py-3 sm:flex-row"
>
<div className="w-full flex-shrink-0 space-y-1.5">
<div className="flex items-center justify-between gap-x-2">
<div className="flex items-center gap-x-2">
<h6 className="line-clamp-1 font-semibold">
{item.productName?.replace(
'Inference Engine',
''
) ?? formatExtensionsName(item.name)}
</h6>
<Badge variant="outline" theme="secondary">
v{item.version}
</Badge>
<p>{item.provider}</p>
</div>
<div className="flex items-center gap-x-2">
<Switch
checked={
!inActiveEngineProvider.includes(item.provider)
}
onChange={() => onSwitchChange(item.provider)}
/>
{!inActiveEngineProvider.includes(item.provider) && (
<SetupRemoteModel engine={item.provider} />
)}
</div>
</div>
{
<div
className="w-full font-medium leading-relaxed text-[hsla(var(--text-secondary))] sm:w-4/5"
dangerouslySetInnerHTML={{
__html: marked.parse(item.description ?? '', {
async: false,
}),
}}
/>
}
</div>
</div>
)
})}
</div>
</div>
)}
</ScrollArea>
)
}
export default Engines

9
web/screens/Settings/SettingDetail/index.tsx
View File

@ -1,8 +1,11 @@
import { InferenceEngine } from '@janhq/core'
import { useAtomValue } from 'jotai'
import Advanced from '@/screens/Settings/Advanced'
import AppearanceOptions from '@/screens/Settings/Appearance'
import ExtensionCatalog from '@/screens/Settings/CoreExtensions'
import Engines from '@/screens/Settings/Engines'
import EngineSettings from '@/screens/Settings/Engines/Settings'
import ExtensionSetting from '@/screens/Settings/ExtensionSetting'
import Hotkeys from '@/screens/Settings/Hotkeys'
import MyModels from '@/screens/Settings/MyModels'
@ -14,6 +17,9 @@ const SettingDetail = () => {
const selectedSetting = useAtomValue(selectedSettingAtom)
switch (selectedSetting) {
case 'Engines':
return <Engines />
case 'Extensions':
return <ExtensionCatalog />
@ -32,6 +38,9 @@ const SettingDetail = () => {
case 'My Models':
return <MyModels />
case InferenceEngine.cortex_llamacpp:
return <EngineSettings engine={selectedSetting} />
default:
return <ExtensionSetting />
}

2
web/screens/Settings/SettingLeftPanel/SettingItem/index.tsx
View File

@ -29,7 +29,7 @@ const SettingItem = ({ name, setting }: Props) => {
>
<span
className={twMerge(
'p-1.5 font-medium capitalize text-[hsla(var(--left-panel-menu))]',
'p-1.5 font-medium text-[hsla(var(--left-panel-menu))]',
isActive && 'relative z-10 text-[hsla(var(--left-panel-menu-active))]'
)}
>

38
web/screens/Settings/SettingLeftPanel/index.tsx
View File

@ -1,18 +1,28 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
import { memo, useEffect, useState } from 'react'
import { useAtomValue } from 'jotai'
import LeftPanelContainer from '@/containers/LeftPanelContainer'
import { useGetEngines } from '@/hooks/useEngineManagement'
import SettingItem from './SettingItem'
import { extensionManager } from '@/extension'
import { inActiveEngineProviderAtom } from '@/helpers/atoms/Extension.atom'
import {
inActiveEngineProviderAtom,
showSettingActiveLocalEngineAtom,
} from '@/helpers/atoms/Extension.atom'
import { janSettingScreenAtom } from '@/helpers/atoms/Setting.atom'
const SettingLeftPanel = () => {
const { engines } = useGetEngines()
const settingScreens = useAtomValue(janSettingScreenAtom)
const inActiveEngineProvider = useAtomValue(inActiveEngineProviderAtom)
const showSettingActiveLocalEngine = useAtomValue(
showSettingActiveLocalEngineAtom
)
const [extensionHasSettings, setExtensionHasSettings] = useState<
{ name?: string; setting: string }[]
@ -84,12 +94,36 @@ const SettingLeftPanel = () => {
/>
))}
{engines &&
Object.entries(engines)
.filter(([key]) => !showSettingActiveLocalEngine.includes(key))
.filter(([_, value]) => !(value as { type?: string }).type).length >
0 && (
<>
<div className="mb-1 mt-4 px-2">
<label className="text-xs font-medium text-[hsla(var(--text-secondary))]">
Local Engine
</label>
</div>
{engines &&
Object.entries(engines)
.filter(([_, value]) => !(value as { type?: string }).type)
.filter(
([key]) => !showSettingActiveLocalEngine.includes(key)
)
.map(([key]) => {
return <SettingItem key={key} name={key} setting={key} />
})}
</>
)}
{engineHasSettings.filter(
(x) => !inActiveEngineProvider.includes(x.provider)
).length > 0 && (
<div className="mb-1 mt-4 px-2">
<label className="text-xs font-medium text-[hsla(var(--text-secondary))]">
Model Providers
Remote Engine
</label>
</div>
)}

1
web/screens/Settings/index.tsx
View File

@ -17,6 +17,7 @@ export const SettingScreenList = [
'Keyboard Shortcuts',
'Privacy',
'Advanced Settings',
'Engines',
'Extensions',
] as const

15
yarn.lock
View File

@ -1798,6 +1798,7 @@ __metadata:
slate-dom: "npm:0.111.0"
slate-history: "npm:0.110.3"
slate-react: "npm:0.110.3"
swr: "npm:^2.2.5"
tailwind-merge: "npm:^2.0.0"
tailwindcss: "npm:3.3.5"
ts-jest: "npm:^29.2.5"
@ -18995,6 +18996,18 @@ __metadata:
languageName: node
linkType: hard
"swr@npm:^2.2.5":
version: 2.3.0
resolution: "swr@npm:2.3.0"
dependencies:
dequal: "npm:^2.0.3"
use-sync-external-store: "npm:^1.4.0"
peerDependencies:
react: ^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
checksum: 10c0/192497881013654bc82d2787b60ad0701113e8ae41c511dfa8d55bcf58582657a92a4cb2854d4ea2ceaa1055e67e58daf9bd98ada2786a3035ba12898da578f1
languageName: node
linkType: hard
"symbol-tree@npm:^3.2.4":
version: 3.2.4
resolution: "symbol-tree@npm:3.2.4"
@ -19972,7 +19985,7 @@ __metadata:
languageName: node
linkType: hard
"use-sync-external-store@npm:^1.2.0":
"use-sync-external-store@npm:^1.2.0, use-sync-external-store@npm:^1.4.0":
version: 1.4.0
resolution: "use-sync-external-store@npm:1.4.0"
peerDependencies: