diff --git a/core/src/types/setting/settingComponent.ts b/core/src/types/setting/settingComponent.ts index 2474f6bd4..7294b0bbe 100644 --- a/core/src/types/setting/settingComponent.ts +++ b/core/src/types/setting/settingComponent.ts @@ -3,7 +3,11 @@ export type SettingComponentProps = { title: string description: string controllerType: ControllerType - controllerProps: SliderComponentProps | CheckboxComponentProps | InputComponentProps + controllerProps: + | SliderComponentProps + | CheckboxComponentProps + | InputComponentProps + | DropdownComponentProps extensionName?: string requireModelReload?: boolean @@ -12,13 +16,26 @@ export type SettingComponentProps = { export type ConfigType = 'runtime' | 'setting' -export type ControllerType = 'slider' | 'checkbox' | 'input' | 'tag' +export type ControllerType = + | 'slider' + | 'checkbox' + | 'input' + | 'tag' + | 'dropdown' -export type InputType = 'password' | 'text' | 'email' | 'number' | 'tel' | 'url' +export type InputType = + | 'password' + | 'text' + | 'email' + | 'number' + | 'tel' + | 'url' + | 'dropdown' const InputActions = ['unobscure', 'copy'] as const export type InputActionsTuple = typeof InputActions export type InputAction = InputActionsTuple[number] +export type DropdownOption = { name: string; value: string } export type InputComponentProps = { placeholder: string @@ -38,3 +55,9 @@ export type SliderComponentProps = { export type CheckboxComponentProps = { value: boolean } + +export type DropdownComponentProps = { + value: string + type?: InputType + options?: DropdownOption[] +} diff --git a/docs/src/pages/index.mdx b/docs/src/pages/index.mdx index 43c0875d7..4a22a7174 100644 --- a/docs/src/pages/index.mdx +++ b/docs/src/pages/index.mdx @@ -20,7 +20,7 @@ import Home from "@/components/Home" export const getStaticProps = async() => { const resReleaseLatest = await fetch('https://api.github.com/repos/janhq/jan/releases/latest') - const resRelease = await fetch('https://api.github.com/repos/janhq/jan/releases') + const resRelease = await fetch('https://api.github.com/repos/janhq/jan/releases?per_page=500') const resRepo = await fetch('https://api.github.com/repos/janhq/jan') const repo = await resRepo.json() const latestRelease = await resReleaseLatest.json() diff --git a/electron/package.json b/electron/package.json index c058370a7..54e34f247 100644 --- a/electron/package.json +++ b/electron/package.json @@ -82,7 +82,9 @@ "copy:assets": "rimraf --glob \"./pre-install/*.tgz\" && cpx \"../pre-install/*.tgz\" \"./pre-install\"", "version-patch": "jq '.version' package.json | tr -d '\"' > .version.bak && jq --arg ver \"0.1.$(date +%s)\" '.version = $ver' package.json > package.tmp && mv package.tmp package.json", "version-restore": "jq --arg ver $(cat .version.bak) '.version = $ver' package.json > package.tmp && mv package.tmp package.json && rm .version.bak", - "dev": "yarn copy:assets && tsc -p . && yarn version-patch && electron . && yarn version-restore", + "dev:darwin:linux": "yarn copy:assets && tsc -p . && yarn version-patch && electron . && yarn version-restore", + "dev:windows": "yarn copy:assets && tsc -p . && electron .", + "dev": "run-script-os", "compile": "tsc -p .", "start": "electron .", "build": "yarn copy:assets && run-script-os", @@ -129,6 +131,7 @@ "electron-playwright-helpers": "^1.6.0", "eslint": "8.57.0", "eslint-plugin-react": "^7.34.0", + "jq": "^1.7.2", "rimraf": "^5.0.5", "run-script-os": "^1.1.6", "typescript": "^5.3.3", diff --git a/extensions/engine-management-extension/package.json b/extensions/engine-management-extension/package.json index 2a9da41ec..5ffe2d42a 100644 --- a/extensions/engine-management-extension/package.json +++ b/extensions/engine-management-extension/package.json @@ -10,7 +10,10 @@ "scripts": { "test": "jest", "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install" + "codesign:darwin": "../../.github/scripts/auto-sign.sh", + "codesign:win32:linux": "echo 'No codesigning required'", + "codesign": "run-script-os", + "build:publish": "rimraf *.tgz --glob || true && yarn build && yarn codesign && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -20,6 +23,7 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "rolldown": "^1.0.0-beta.1", + "run-script-os": "^1.1.6", "ts-loader": "^9.5.0", "typescript": "^5.3.3" }, diff --git a/extensions/inference-cortex-extension/resources/default_settings.json b/extensions/inference-cortex-extension/resources/default_settings.json index 6a0dcd4a0..a3a93f305 100644 --- a/extensions/inference-cortex-extension/resources/default_settings.json +++ b/extensions/inference-cortex-extension/resources/default_settings.json @@ -1,8 +1,8 @@ [ { "key": "cont_batching", - "title": "Continuous batching", - "description": "The number of parallel operations", + "title": "Continuous Batching", + "description": "Allows processing prompts in parallel with text generation, which usually improves performance.", "controllerType": "checkbox", "controllerProps": { "value": true @@ -10,28 +10,32 @@ }, { "key": "n_parallel", - "title": "Parallel operations", - "description": "The number of parallel operations", + "title": "Parallel Operations", + "description": "Number of prompts that can be processed simultaneously by the model.", "controllerType": "input", "controllerProps": { "value": "4", - "placeholder": "4" + "placeholder": "4", + "type": "number", + "textAlign": "right" } }, { "key": "cpu_threads", "title": "CPU Threads", - "description": "The number of CPU threads to use (when in CPU mode)", + "description": "Number of CPU cores used for model processing when running without GPU.", "controllerType": "input", "controllerProps": { "value": "", - "placeholder": "Number of CPU threads" + "placeholder": "Number of CPU threads", + "type": "number", + "textAlign": "right" } }, { "key": "flash_attn", - "title": "Flash Attention enabled", - "description": "To enable Flash Attention, default is true", + "title": "Flash Attention", + "description": "Optimizes memory usage and speeds up model inference using an efficient attention implementation.", "controllerType": "checkbox", "controllerProps": { "value": true @@ -40,8 +44,8 @@ { "key": "caching_enabled", - "title": "Caching enabled", - "description": "To enable prompt caching or not", + "title": "Caching", + "description": "Stores recent prompts and responses to improve speed when similar questions are asked.", "controllerType": "checkbox", "controllerProps": { "value": true @@ -50,17 +54,30 @@ { "key": "cache_type", "title": "KV Cache Type", - "description": "KV cache type: f16, q8_0, q4_0, default is f16 (change this could break the model).", - "controllerType": "input", + "description": "Controls memory usage and precision trade-off.", + "controllerType": "dropdown", "controllerProps": { - "placeholder": "f16", - "value": "f16" + "value": "f16", + "options": [ + { + "value": "q4_0", + "name": "q4_0" + }, + { + "value": "q8_0", + "name": "q8_0" + }, + { + "value": "f16", + "name": "f16" + } + ] } }, { "key": "use_mmap", - "title": "To enable mmap", - "description": "To enable mmap, default is true", + "title": "MMAP", + "description": "Loads model files more efficiently by mapping them to memory, reducing RAM usage.", "controllerType": "checkbox", "controllerProps": { "value": true diff --git a/extensions/inference-cortex-extension/rolldown.config.mjs b/extensions/inference-cortex-extension/rolldown.config.mjs index 4f8a4ac06..c20bf7ed0 100644 --- a/extensions/inference-cortex-extension/rolldown.config.mjs +++ b/extensions/inference-cortex-extension/rolldown.config.mjs @@ -111,7 +111,7 @@ export default defineConfig([ SETTINGS: JSON.stringify(defaultSettingJson), CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'), CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'), }, }, { diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 4ce35c83d..dda92fbcf 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -1,6 +1,7 @@ import path from 'path' -import { getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node' +import { appResourcePath, getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node' import { ProcessWatchdog } from './watchdog' +import { readdir, symlink } from 'fs/promises' // The HOST address to use for the Nitro subprocess const LOCAL_PORT = '39291' @@ -17,7 +18,13 @@ function run(systemInfo?: SystemInformation): Promise { let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? '' let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}` const binPath = path.join(__dirname, '..', 'bin') + await createEngineSymlinks(binPath) + const executablePath = path.join(binPath, binaryName) + const sharedPath = path.join( + appResourcePath(), + 'shared' + ) // Execute the binary log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`) @@ -46,6 +53,7 @@ function run(systemInfo?: SystemInformation): Promise { GGML_VK_VISIBLE_DEVICES: gpuVisibleDevices, }), }, + cwd: sharedPath, } ) watchdog.start() @@ -53,6 +61,23 @@ function run(systemInfo?: SystemInformation): Promise { }) } +/** + * Create symlinks for the engine shared libraries + * @param binPath + */ +async function createEngineSymlinks(binPath: string) { + const sharedPath = path.join(appResourcePath(), 'shared') + const sharedLibFiles = await readdir(sharedPath) + for (const sharedLibFile of sharedLibFiles) { + if (sharedLibFile.endsWith('.dll') || sharedLibFile.endsWith('.so')) { + const targetDllPath = path.join(sharedPath, sharedLibFile) + const symlinkDllPath = path.join(binPath, sharedLibFile) + await symlink(targetDllPath, symlinkDllPath).catch(console.error) + console.log(`Symlink created: ${targetDllPath} -> ${symlinkDllPath}`) + } + } +} + /** * Every module should have a dispose function * This will be called when the extension is unloaded and should clean up any resources diff --git a/package.json b/package.json index 174b39d96..743406122 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "rimraf": "^3.0.2", - "run-script-os": "^1.1.6", "wait-on": "^7.0.1" }, "version": "0.0.0", diff --git a/web/screens/Settings/SettingDetail/SettingDetailItem/SettingDetailDropdownItem/index.tsx b/web/screens/Settings/SettingDetail/SettingDetailItem/SettingDetailDropdownItem/index.tsx new file mode 100644 index 000000000..e4066998d --- /dev/null +++ b/web/screens/Settings/SettingDetail/SettingDetailItem/SettingDetailDropdownItem/index.tsx @@ -0,0 +1,50 @@ +import { DropdownComponentProps, SettingComponentProps } from '@janhq/core' +import { Select } from '@janhq/joi' +import { Marked, Renderer } from 'marked' + +type Props = { + settingProps: SettingComponentProps + onValueChanged?: (e: string) => void +} + +const marked: Marked = new Marked({ + renderer: { + link: (href, title, text) => { + return Renderer.prototype.link + ?.apply(this, [href, title, text]) + .replace( + ' = ({ + settingProps, + onValueChanged, +}) => { + const { value, options } = + settingProps.controllerProps as DropdownComponentProps + + const description = marked.parse(settingProps.description ?? '', { + async: false, + }) + + return ( +
+
+

{settingProps.title}

+ { +
+ } +
+ onValueChanged?.(e.target.value)} - className="!pr-20" + className={twMerge(obscure && '!pr-20')} suffixIcon={ - + obscure ? ( + + ) : undefined } />
diff --git a/web/screens/Settings/SettingDetail/SettingDetailItem/index.tsx b/web/screens/Settings/SettingDetail/SettingDetailItem/index.tsx index 7c44095c8..93636a28a 100644 --- a/web/screens/Settings/SettingDetail/SettingDetailItem/index.tsx +++ b/web/screens/Settings/SettingDetail/SettingDetailItem/index.tsx @@ -1,5 +1,6 @@ import { SettingComponentProps } from '@janhq/core' +import SettingDetailDropdownItem from './SettingDetailDropdownItem' import SettingDetailTextInputItem from './SettingDetailTextInputItem' import SettingDetailToggleItem from './SettingDetailToggleItem' @@ -36,13 +37,23 @@ const SettingDetailItem = ({ componentProps, onValueUpdated }: Props) => { ) } + case 'dropdown': { + return ( + onValueUpdated(data.key, value)} + /> + ) + } + default: return null } }) return ( -
+
{components.map((component, index) => (