From 7ac927ff020bb352b3a94e61dc76995c8edc0e92 Mon Sep 17 00:00:00 2001 From: Akarshan Date: Wed, 3 Sep 2025 12:53:20 +0530 Subject: [PATCH] feat: enhance llamacpp backend management and installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `src-tauri/resources/` to `.gitignore`. - Introduced utilities to read locally installed backends (`getLocalInstalledBackends`) and fetch remote supported backends (`fetchRemoteSupportedBackends`). - Refactored `listSupportedBackends` to merge remote and local entries with deduplication and proper sorting. - Exported `getBackendDir` and integrated it into the extension. - Added helper `parseBackendVersion` and new method `checkBackendForUpdates` to detect newer backend versions. - Implemented `installBackend` for manual backend archive installation, including platform‑specific binary path handling. - Updated command‑line argument logic for `--flash-attn` to respect version‑specific defaults. - Modified Tauri filesystem `decompress` command to remove overly strict path validation. --- .gitignore | 1 + extensions/llamacpp-extension/src/backend.ts | 123 +++++++++++++++---- extensions/llamacpp-extension/src/index.ts | 82 ++++++++++++- src-tauri/src/core/filesystem/commands.rs | 7 -- 4 files changed, 179 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 4b41f1c49..e087b3b66 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ docs/.next/ ## cargo target Cargo.lock +src-tauri/resources/ ## test test-data diff --git a/extensions/llamacpp-extension/src/backend.ts b/extensions/llamacpp-extension/src/backend.ts index 710dfeba8..cacaa56a8 100644 --- a/extensions/llamacpp-extension/src/backend.ts +++ b/extensions/llamacpp-extension/src/backend.ts @@ -1,9 +1,80 @@ import { getJanDataFolderPath, fs, joinPath, events } from '@janhq/core' import { invoke } from '@tauri-apps/api/core' import { getProxyConfig } from './util' -import { dirname } from '@tauri-apps/api/path' +import { dirname, basename } from '@tauri-apps/api/path' import { getSystemInfo } from '@janhq/tauri-plugin-hardware-api' +/* + * Reads currently installed backends in janDataFolderPath + * + */ +export async function getLocalInstalledBackends(): Promise< + { version: string; backend: string }[] +> { + const local: Array<{ version: string; backend: string }> = [] + const janDataFolderPath = await getJanDataFolderPath() + const backendsDir = await joinPath([ + janDataFolderPath, + 'llamacpp', + 'backends', + ]) + const versionDirs = await fs.readdirSync(backendsDir) + + // If the folder does not exist we are done. + if (!versionDirs) { + return local + } + for (const version of versionDirs) { + const versionPath = await joinPath([backendsDir, version]) + const versionName = await basename(versionPath) + const backendTypes = await fs.readdirSync(versionPath) + + // Verify that the backend is really installed + for (const backendType of backendTypes) { + const backendName = await basename(backendType) + if (await isBackendInstalled(backendType, versionName)) { + local.push({ version: versionName, backend: backendName }) + } + } + } + return local +} + +/* + * currently reads available backends in remote + * + */ +async function fetchRemoteSupportedBackends( + supportedBackends: string[] +): Promise<{ version: string; backend: string }[]> { + // Pull the latest releases from the repo + const { releases } = await _fetchGithubReleases('menloresearch', 'llama.cpp') + releases.sort((a, b) => b.tag_name.localeCompare(a.tag_name)) + releases.splice(10) // keep only the latest 10 releases + + // Walk the assets and keep only those that match a supported backend + const remote: { version: string; backend: string }[] = [] + + for (const release of releases) { + const version = release.tag_name + const prefix = `llama-${version}-bin-` + + for (const asset of release.assets) { + const name = asset.name + + if (!name.startsWith(prefix)) continue + + const backend = name.replace(prefix, '').replace('.tar.gz', '') + + if (supportedBackends.includes(backend)) { + remote.push({ version, backend }) + } + } + } + + return remote +} + // folder structure // /llamacpp/backends// @@ -76,31 +147,29 @@ export async function listSupportedBackends(): Promise< } else if (sysType === 'macos-aarch64' || sysType === 'macos-arm64') { supportedBackends.push('macos-arm64') } + // get latest backends from Github + const remoteBackendVersions = + await fetchRemoteSupportedBackends(supportedBackends) - const { releases } = await _fetchGithubReleases('menloresearch', 'llama.cpp') - releases.sort((a, b) => b.tag_name.localeCompare(a.tag_name)) - releases.splice(10) // keep only the latest 10 releases - - let backendVersions = [] - for (const release of releases) { - const version = release.tag_name - const prefix = `llama-${version}-bin-` - - // NOTE: there is checksum.yml. we can also download it to verify the download - for (const asset of release.assets) { - const name = asset.name - if (!name.startsWith(prefix)) { - continue - } - - const backend = name.replace(prefix, '').replace('.tar.gz', '') - if (supportedBackends.includes(backend)) { - backendVersions.push({ version, backend }) - } - } + // Get locally installed versions + const localBackendVersions = await getLocalInstalledBackends() + // Use a Map keyed by “${version}|${backend}” for O(1) deduplication. + const mergedMap = new Map() + for (const entry of remoteBackendVersions) { + mergedMap.set(`${entry.version}|${entry.backend}`, entry) + } + for (const entry of localBackendVersions) { + mergedMap.set(`${entry.version}|${entry.backend}`, entry) } - return backendVersions + const merged = Array.from(mergedMap.values()) + // Sort newest version first; if versions tie, sort by backend name + merged.sort((a, b) => { + const versionCmp = b.version.localeCompare(a.version) + return versionCmp !== 0 ? versionCmp : a.backend.localeCompare(b.backend) + }) + + return merged } export async function getBackendDir( @@ -299,21 +368,23 @@ async function _fetchGithubReleases( const githubUrl = `https://api.github.com/repos/${owner}/${repo}/releases` try { const response = await fetch(githubUrl) - if (!response.ok) throw new Error(`GitHub error: ${response.status} ${response.statusText}`) + if (!response.ok) + throw new Error(`GitHub error: ${response.status} ${response.statusText}`) const releases = await response.json() return { releases, source: 'github' } } catch (_err) { const cdnUrl = 'https://catalog.jan.ai/llama.cpp/releases/releases.json' const response = await fetch(cdnUrl) if (!response.ok) { - throw new Error(`Failed to fetch releases from both sources. CDN error: ${response.status} ${response.statusText}`) + throw new Error( + `Failed to fetch releases from both sources. CDN error: ${response.status} ${response.statusText}` + ) } const releases = await response.json() return { releases, source: 'cdn' } } } - async function _isCudaInstalled(version: string): Promise { const sysInfo = await getSystemInfo() const os_type = sysInfo.os_type diff --git a/extensions/llamacpp-extension/src/index.ts b/extensions/llamacpp-extension/src/index.ts index e706b58ae..3151d2446 100644 --- a/extensions/llamacpp-extension/src/index.ts +++ b/extensions/llamacpp-extension/src/index.ts @@ -31,6 +31,7 @@ import { downloadBackend, isBackendInstalled, getBackendExePath, + getBackendDir, } from './backend' import { invoke } from '@tauri-apps/api/core' import { getProxyConfig } from './util' @@ -712,6 +713,47 @@ export default class llamacpp_extension extends AIEngine { } } + private parseBackendVersion(v: string): number { + // Remove any leading non‑digit characters (e.g. the "b") + const numeric = v.replace(/^[^\d]*/, '') + const n = Number(numeric) + return Number.isNaN(n) ? 0 : n + } + + async checkBackendForUpdates(): Promise<{ + updateNeeded: boolean + newVersion: string + }> { + // Parse current backend configuration + const [currentVersion, currentBackend] = ( + this.config.version_backend || '' + ).split('/') + + if (!currentVersion || !currentBackend) { + logger.warn( + `Invalid current backend format: ${this.config.version_backend}` + ) + return { updateNeeded: false, newVersion: '0' } + } + + // Find the latest version for the currently selected backend type + const version_backends = await listSupportedBackends() + const targetBackendString = this.findLatestVersionForBackend( + version_backends, + currentBackend + ) + const [latestVersion] = targetBackendString.split('/') + if (this.parseBackendVersion(latestVersion) > this.parseBackendVersion(currentVersion)) { + logger.info(`New update available: ${latestVersion}`) + return { updateNeeded: true, newVersion: latestVersion } + } else { + logger.info( + `Already at latest version: ${currentVersion} = ${latestVersion}` + ) + return { updateNeeded: false, newVersion: '0' } + } + } + private async removeOldBackend( latestVersion: string, backendType: string @@ -1016,6 +1058,40 @@ export default class llamacpp_extension extends AIEngine { localStorage.setItem('cortex_models_migrated', 'true') } + /* + * Manually installs a supported backend archive + * + */ + async installBackend(path: string): Promise { + const platformName = IS_WINDOWS ? 'win' : 'linux' + const re = /^llama-(b\d+)-bin-(.+?)\.tar\.gz$/ + const archiveName = await basename(path) + + let binPath: string + + if ((await fs.existsSync(path)) && path.endsWith('tar.gz')) { + const match = re.exec(archiveName) + const [, version, backend] = match + if (!version && !backend) { + throw new Error(`Invalid backend archive name: ${archiveName}`) + } + const backendDir = await getBackendDir(backend, version) + await invoke('decompress', { path: path, outputDir: backendDir }) + if (platformName == 'win') + binPath = await joinPath([ + backendDir, + 'build', + 'bin', + 'llama-server.exe', + ]) + else + binPath = await joinPath([backendDir, 'build', 'bin', 'llama-server']) + if (!fs.existsSync(binPath)) { + throw new Error('Not a supported backend archive!') + } + } + } + override async import(modelId: string, opts: ImportOptions): Promise { const isValidModelId = (id: string) => { // only allow alphanumeric, underscore, hyphen, and dot characters in modelId @@ -1438,7 +1514,11 @@ export default class llamacpp_extension extends AIEngine { // Boolean flags if (!cfg.ctx_shift) args.push('--no-context-shift') - if (cfg.flash_attn) args.push('--flash-attn') + if (Number(version.replace(/^b/, '')) >= 6325) { + if (!cfg.flash_attn) args.push('--flash-attn', 'off') //default: auto = ON when supported + } else { + if (cfg.flash_attn) args.push('--flash-attn') + } if (cfg.cont_batching) args.push('--cont-batching') args.push('--no-mmap') if (cfg.mlock) args.push('--mlock') diff --git a/src-tauri/src/core/filesystem/commands.rs b/src-tauri/src/core/filesystem/commands.rs index c70943db4..1601f87aa 100644 --- a/src-tauri/src/core/filesystem/commands.rs +++ b/src-tauri/src/core/filesystem/commands.rs @@ -165,13 +165,6 @@ pub fn read_yaml(app: tauri::AppHandle, path: &str) -> Result Result<(), String> { let jan_data_folder = crate::core::app::commands::get_jan_data_folder_path(app.clone()); let path_buf = jan_utils::normalize_path(&jan_data_folder.join(path)); - if !path_buf.starts_with(&jan_data_folder) { - return Err(format!( - "Error: path {} is not under jan_data_folder {}", - path_buf.to_string_lossy(), - jan_data_folder.to_string_lossy(), - )); - } let output_dir_buf = jan_utils::normalize_path(&jan_data_folder.join(output_dir)); if !output_dir_buf.starts_with(&jan_data_folder) {