diff --git a/README.md b/README.md index 60d977a81..ca5f3284c 100644 --- a/README.md +++ b/README.md @@ -43,31 +43,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Stable (Recommended) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage diff --git a/core/src/extension.ts b/core/src/extension.ts index 3528f581c..3b3edc7b3 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -4,6 +4,7 @@ export enum ExtensionTypeEnum { Inference = 'inference', Model = 'model', SystemMonitoring = 'systemMonitoring', + HuggingFace = 'huggingFace', } export interface ExtensionType { diff --git a/core/src/extensions/huggingface.ts b/core/src/extensions/huggingface.ts new file mode 100644 index 000000000..16a1d9b8a --- /dev/null +++ b/core/src/extensions/huggingface.ts @@ -0,0 +1,30 @@ +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { HuggingFaceInterface, HuggingFaceRepoData, Quantization } from '../types/huggingface' +import { Model } from '../types/model' + +/** + * Hugging Face extension for converting HF models to GGUF. + */ +export abstract class HuggingFaceExtension extends BaseExtension implements HuggingFaceInterface { + interrupted = false + /** + * Hugging Face extension type. + */ + type(): ExtensionTypeEnum | undefined { + return ExtensionTypeEnum.HuggingFace + } + + abstract downloadModelFiles( + repoID: string, + repoData: HuggingFaceRepoData, + network?: { ignoreSSL?: boolean; proxy?: string } + ): Promise + abstract convert(repoID: string): Promise + abstract quantize(repoID: string, quantization: Quantization): Promise + abstract generateMetadata( + repoID: string, + repoData: HuggingFaceRepoData, + quantization: Quantization + ): Promise + abstract cancelConvert(repoID: string, repoData: HuggingFaceRepoData): Promise +} diff --git a/core/src/extensions/index.ts b/core/src/extensions/index.ts index 522334548..c6834482c 100644 --- a/core/src/extensions/index.ts +++ b/core/src/extensions/index.ts @@ -23,3 +23,8 @@ export { AssistantExtension } from './assistant' * Model extension for managing models. */ export { ModelExtension } from './model' + +/** + * Hugging Face extension for converting HF models to GGUF. + */ +export { HuggingFaceExtension } from './huggingface' diff --git a/core/src/types/huggingface/huggingfaceEntity.ts b/core/src/types/huggingface/huggingfaceEntity.ts new file mode 100644 index 000000000..c3c320354 --- /dev/null +++ b/core/src/types/huggingface/huggingfaceEntity.ts @@ -0,0 +1,34 @@ +export interface HuggingFaceRepoData { + id: string + author: string + tags: Array<'transformers' | 'pytorch' | 'safetensors' | string> + siblings: { + rfilename: string + }[] + createdAt: string // ISO 8601 timestamp +} + +/* eslint-disable @typescript-eslint/naming-convention */ +export enum Quantization { + Q3_K_S = 'Q3_K_S', + Q3_K_M = 'Q3_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values + Q3_K_L = 'Q3_K_L', + Q4_K_S = 'Q4_K_S', + Q4_K_M = 'Q4_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values + Q5_K_S = 'Q5_K_S', + Q5_K_M = 'Q5_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values + Q4_0 = 'Q4_0', + Q4_1 = 'Q4_1', + Q5_0 = 'Q5_0', + Q5_1 = 'Q5_1', + IQ2_XXS = 'IQ2_XXS', + IQ2_XS = 'IQ2_XS', + Q2_K = 'Q2_K', + Q2_K_S = 'Q2_K_S', + Q6_K = 'Q6_K', + Q8_0 = 'Q8_0', + F16 = 'F16', + F32 = 'F32', + COPY = 'COPY', +} +/* eslint-enable @typescript-eslint/naming-convention */ diff --git a/core/src/types/huggingface/huggingfaceInterface.ts b/core/src/types/huggingface/huggingfaceInterface.ts new file mode 100644 index 000000000..c99b2177d --- /dev/null +++ b/core/src/types/huggingface/huggingfaceInterface.ts @@ -0,0 +1,58 @@ +import { Model } from '../model' +import { HuggingFaceRepoData, Quantization } from './huggingfaceEntity' + +/** + * Hugging Face extension for converting HF models to GGUF. + * @extends BaseExtension + */ +export interface HuggingFaceInterface { + interrupted: boolean + /** + * Downloads a Hugging Face model. + * @param repoID - The repo ID of the model to convert. + * @param repoData - The repo data of the model to convert. + * @param network - Optional object to specify proxy/whether to ignore SSL certificates. + * @returns A promise that resolves when the download is complete. + */ + downloadModelFiles( + repoID: string, + repoData: HuggingFaceRepoData, + network?: { ignoreSSL?: boolean; proxy?: string } + ): Promise + + /** + * Converts a Hugging Face model to GGUF. + * @param repoID - The repo ID of the model to convert. + * @returns A promise that resolves when the conversion is complete. + */ + convert(repoID: string): Promise + + /** + * Quantizes a GGUF model. + * @param repoID - The repo ID of the model to quantize. + * @param quantization - The quantization to use. + * @returns A promise that resolves when the quantization is complete. + */ + quantize(repoID: string, quantization: Quantization): Promise + + /** + * Generates Jan model metadata from a Hugging Face model. + * @param repoID - The repo ID of the model to generate metadata for. + * @param repoData - The repo data of the model to generate metadata for. + * @param quantization - The quantization of the model. + * @returns A promise that resolves when the model metadata generation is complete. + */ + generateMetadata( + repoID: string, + repoData: HuggingFaceRepoData, + quantization: Quantization + ): Promise + + /** + * Cancels the convert of current Hugging Face model. + * @param repoID - The repository ID to cancel. + * @param repoData - The repository data to cancel. + * @returns {Promise} A promise that resolves when the download has been cancelled. + */ + cancelConvert(repoID: string, repoData: HuggingFaceRepoData): Promise +} diff --git a/core/src/types/huggingface/index.ts b/core/src/types/huggingface/index.ts new file mode 100644 index 000000000..c108c55e2 --- /dev/null +++ b/core/src/types/huggingface/index.ts @@ -0,0 +1,2 @@ +export * from './huggingfaceInterface' +export * from './huggingfaceEntity' diff --git a/core/src/types/index.ts b/core/src/types/index.ts index ee6f4ef08..295d054e7 100644 --- a/core/src/types/index.ts +++ b/core/src/types/index.ts @@ -6,4 +6,5 @@ export * from './inference' export * from './monitoring' export * from './file' export * from './config' +export * from './huggingface' export * from './miscellaneous' diff --git a/electron/package.json b/electron/package.json index a89803077..7cdb98360 100644 --- a/electron/package.json +++ b/electron/package.json @@ -15,12 +15,14 @@ "build/**/*.{js,map}", "pre-install", "models/**/*", - "docs/**/*" + "docs/**/*", + "scripts/**/*" ], "asarUnpack": [ "pre-install", "models", - "docs" + "docs", + "scripts" ], "publish": [ { diff --git a/extensions/huggingface-extension/.gitignore b/extensions/huggingface-extension/.gitignore new file mode 100644 index 000000000..bdf39cc7f --- /dev/null +++ b/extensions/huggingface-extension/.gitignore @@ -0,0 +1,3 @@ +bin +scripts/convert* +scripts/gguf-py diff --git a/extensions/huggingface-extension/.prettierrc b/extensions/huggingface-extension/.prettierrc new file mode 100644 index 000000000..46f1abcb0 --- /dev/null +++ b/extensions/huggingface-extension/.prettierrc @@ -0,0 +1,8 @@ +{ + "semi": false, + "singleQuote": true, + "quoteProps": "consistent", + "trailingComma": "es5", + "endOfLine": "auto", + "plugins": ["prettier-plugin-tailwindcss"] +} diff --git a/extensions/huggingface-extension/README.md b/extensions/huggingface-extension/README.md new file mode 100644 index 000000000..ae70eb4ec --- /dev/null +++ b/extensions/huggingface-extension/README.md @@ -0,0 +1,73 @@ +# Create a Jan Plugin using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 + +## Create Your Own Plugin + +To create your own plugin, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your plugin directory now + +## Update the Plugin Metadata + +The [`package.json`](package.json) file defines metadata about your plugin, such as +plugin name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your plugin. + +## Update the Plugin Code + +The [`src/`](./src/) directory is the heart of your plugin! This contains the +source code that will be run when your plugin extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your plugin code: + +- Most Jan Plugin Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { core } from "@janhq/core"; + + function onStart(): Promise { + return core.invokePluginFunc(MODULE_PATH, "run", 0); + } + ``` + + For more information about the Jan Plugin Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your plugin! diff --git a/extensions/huggingface-extension/bin/mac-arm64/quantize b/extensions/huggingface-extension/bin/mac-arm64/quantize new file mode 100755 index 000000000..f8a149b10 Binary files /dev/null and b/extensions/huggingface-extension/bin/mac-arm64/quantize differ diff --git a/extensions/huggingface-extension/download.bat b/extensions/huggingface-extension/download.bat new file mode 100644 index 000000000..de055cb80 --- /dev/null +++ b/extensions/huggingface-extension/download.bat @@ -0,0 +1,3 @@ +@echo off +set /p LLAMA_CPP_VERSION=<./scripts/version.txt +.\node_modules\.bin\download https://github.com/ggerganov/llama.cpp/archive/refs/tags/%LLAMA_CPP_VERSION%.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf .\scripts\llama.cpp.tar.gz "llama.cpp-%LLAMA_CPP_VERSION%/convert.py" "llama.cpp-%LLAMA_CPP_VERSION%/convert-hf-to-gguf.py" "llama.cpp-%LLAMA_CPP_VERSION%/gguf-py" && cpx "./llama.cpp-%LLAMA_CPP_VERSION%/**" "scripts" && rimraf "./scripts/llama.cpp.tar.gz" && rimraf "./llama.cpp-%LLAMA_CPP_VERSION%" \ No newline at end of file diff --git a/extensions/huggingface-extension/package.json b/extensions/huggingface-extension/package.json new file mode 100644 index 000000000..283b989ce --- /dev/null +++ b/extensions/huggingface-extension/package.json @@ -0,0 +1,54 @@ +{ + "name": "@janhq/huggingface-extension", + "version": "1.0.0", + "description": "Hugging Face extension for converting HF models to GGUF", + "main": "dist/index.js", + "node": "dist/node/index.cjs.js", + "author": "Jan ", + "license": "AGPL-3.0", + "scripts": { + "build": "tsc --module commonjs && rollup -c rollup.config.ts --configPlugin @rollup/plugin-typescript --bundleConfigAsCjs", + "download:llama": "run-script-os", + "download:llama:linux": "LLAMA_CPP_VERSION=$(cat ./scripts/version.txt) && download https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf ./scripts/llama.cpp.tar.gz --wildcards '*/convert.py' '*/convert-hf-to-gguf.py' '*/gguf-py' && cpx \"./llama.cpp-$LLAMA_CPP_VERSION/**\" \"scripts\" && rimraf \"./scripts/llama.cpp.tar.gz\" && rimraf \"./llama.cpp-$LLAMA_CPP_VERSION\"", + "download:llama:darwin": "LLAMA_CPP_VERSION=$(cat ./scripts/version.txt) && download https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf ./scripts/llama.cpp.tar.gz '*/convert.py' '*/convert-hf-to-gguf.py' '*/gguf-py' && cpx \"./llama.cpp-$LLAMA_CPP_VERSION/**\" \"scripts\" && rimraf \"./scripts/llama.cpp.tar.gz\" && rimraf \"./llama.cpp-$LLAMA_CPP_VERSION\"", + "download:llama:win32": "download.bat", + "build:publish": "rimraf *.tgz --glob && npm run build && npm run download:llama && cpx \"scripts/**\" \"dist/scripts\" && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/node/index.cjs.js" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-replace": "^5.0.5", + "@rollup/plugin-typescript": "^11.1.6", + "@types/node": "^20.11.16", + "cpx": "^1.5.0", + "download-cli": "^1.1.1", + "rimraf": "^5.0.5", + "rollup": "^4.9.6", + "rollup-plugin-sourcemaps": "^0.6.3", + "rollup-plugin-typescript2": "^0.36.0", + "run-script-os": "^1.1.6", + "typescript": "^5.3.3" + }, + "dependencies": { + "@janhq/core": "file:../../core", + "hyllama": "^0.1.2", + "python-shell": "^5.0.0", + "ts-loader": "^9.5.0" + }, + "bundledDependencies": [ + "python-shell" + ], + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ] +} diff --git a/extensions/huggingface-extension/rollup.config.ts b/extensions/huggingface-extension/rollup.config.ts new file mode 100644 index 000000000..7ae2c5781 --- /dev/null +++ b/extensions/huggingface-extension/rollup.config.ts @@ -0,0 +1,72 @@ +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' + +const packageJson = require('./package.json') + +export default [ + { + input: `src/index.ts`, + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [], + watch: { + include: 'src/**', + }, + plugins: [ + replace({ + EXTENSION_NAME: JSON.stringify(packageJson.name), + NODE_MODULE_PATH: JSON.stringify( + `${packageJson.name}/${packageJson.node}` + ), + }), + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Compile TypeScript files + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: ['.js', '.ts'], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, + { + input: `src/node/index.ts`, + output: [ + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, + ], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [], + watch: { + include: 'src/node/**', + }, + plugins: [ + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: ['.ts', '.js', '.json'], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, +] diff --git a/extensions/huggingface-extension/scripts/install_deps.py b/extensions/huggingface-extension/scripts/install_deps.py new file mode 100644 index 000000000..2dfabed07 --- /dev/null +++ b/extensions/huggingface-extension/scripts/install_deps.py @@ -0,0 +1,14 @@ +import subprocess +import sys + +deps = [ + 'numpy~=1.24.4', + 'sentencepiece~=0.1.98', + 'transformers>=4.35.2,<5.0.0', + 'gguf>=0.1.0', + 'protobuf>=4.21.0,<5.0.0', + 'torch~=2.1.1', + 'packaging>=20.0', + 'tiktoken~=0.5.0' +] +subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--upgrade', '--force-reinstall', *deps]) diff --git a/extensions/huggingface-extension/scripts/version.txt b/extensions/huggingface-extension/scripts/version.txt new file mode 100644 index 000000000..f743d6c4a --- /dev/null +++ b/extensions/huggingface-extension/scripts/version.txt @@ -0,0 +1 @@ +b2106 \ No newline at end of file diff --git a/extensions/huggingface-extension/src/@types/global.d.ts b/extensions/huggingface-extension/src/@types/global.d.ts new file mode 100644 index 000000000..495ecf00e --- /dev/null +++ b/extensions/huggingface-extension/src/@types/global.d.ts @@ -0,0 +1,2 @@ +declare const EXTENSION_NAME: string +declare const NODE_MODULE_PATH: string diff --git a/extensions/huggingface-extension/src/index.ts b/extensions/huggingface-extension/src/index.ts new file mode 100644 index 000000000..d8f755080 --- /dev/null +++ b/extensions/huggingface-extension/src/index.ts @@ -0,0 +1,396 @@ +import { + fs, + downloadFile, + abortDownload, + joinPath, + HuggingFaceExtension, + HuggingFaceRepoData, + executeOnMain, + Quantization, + Model, + InferenceEngine, + getJanDataFolderPath, + events, + DownloadEvent, + log, +} from '@janhq/core' +import { ggufMetadata } from 'hyllama' + +declare global { + interface Window { + electronAPI?: any + } +} + +/** + * A extension for models + */ +export default class JanHuggingFaceExtension extends HuggingFaceExtension { + private static readonly _safetensorsRegexs = [ + /model\.safetensors$/, + /model-[0-9]+-of-[0-9]+\.safetensors$/, + ] + private static readonly _pytorchRegexs = [ + /pytorch_model\.bin$/, + /consolidated\.[0-9]+\.pth$/, + /pytorch_model-[0-9]+-of-[0-9]+\.bin$/, + /.*\.pt$/, + ] + interrupted = false + + /** + * Called when the extension is loaded. + * @override + */ + onLoad() {} + + /** + * Called when the extension is unloaded. + * @override + */ + onUnload(): void {} + + private getFileList(repoData: HuggingFaceRepoData): string[] { + // SafeTensors first, if not, then PyTorch + const modelFiles = repoData.siblings + .map((file) => file.rfilename) + .filter((file) => + JanHuggingFaceExtension._safetensorsRegexs.some((regex) => + regex.test(file) + ) + ) + if (modelFiles.length === 0) { + repoData.siblings.forEach((file) => { + if ( + JanHuggingFaceExtension._pytorchRegexs.some((regex) => + regex.test(file.rfilename) + ) + ) { + modelFiles.push(file.rfilename) + } + }) + } + + const vocabFiles = [ + 'tokenizer.model', + 'vocab.json', + 'tokenizer.json', + ].filter((file) => + repoData.siblings.some((sibling) => sibling.rfilename === file) + ) + + const etcFiles = repoData.siblings + .map((file) => file.rfilename) + .filter( + (file) => + (file.endsWith('.json') && !vocabFiles.includes(file)) || + file.endsWith('.txt') || + file.endsWith('.py') || + file.endsWith('.tiktoken') + ) + + return [...modelFiles, ...vocabFiles, ...etcFiles] + } + + private async getModelDirPath(repoID: string): Promise { + const modelName = repoID.split('/').slice(1).join('/') + return joinPath([await getJanDataFolderPath(), 'models', modelName]) + } + private async getConvertedModelPath(repoID: string): Promise { + const modelName = repoID.split('/').slice(1).join('/') + const modelDirPath = await this.getModelDirPath(repoID) + return joinPath([modelDirPath, modelName + '.gguf']) + } + private async getQuantizedModelPath( + repoID: string, + quantization: Quantization + ): Promise { + const modelName = repoID.split('/').slice(1).join('/') + const modelDirPath = await this.getModelDirPath(repoID) + return joinPath([ + modelDirPath, + modelName + `-${quantization.toLowerCase()}.gguf`, + ]) + } + private getCtxLength(config: { + max_sequence_length?: number + max_position_embeddings?: number + n_ctx?: number + }): number { + if (config.max_sequence_length) return config.max_sequence_length + if (config.max_position_embeddings) return config.max_position_embeddings + if (config.n_ctx) return config.n_ctx + return 4096 + } + + /** + * Downloads a Hugging Face model. + * @param repoID - The repo ID of the model to convert. + * @param repoData - The repo data of the model to convert. + * @param network - Optional object to specify proxy/whether to ignore SSL certificates. + * @returns A promise that resolves when the download is complete. + */ + async downloadModelFiles( + repoID: string, + repoData: HuggingFaceRepoData, + network?: { ignoreSSL?: boolean; proxy?: string } + ): Promise { + if (this.interrupted) return + const modelDirPath = await this.getModelDirPath(repoID) + if (!(await fs.existsSync(modelDirPath))) await fs.mkdirSync(modelDirPath) + const files = this.getFileList(repoData) + const filePaths: string[] = [] + + for (const file of files) { + const filePath = file + const localPath = await joinPath([modelDirPath, filePath]) + const url = `https://huggingface.co/${repoID}/resolve/main/${filePath}` + + if (this.interrupted) return + if (!(await fs.existsSync(localPath))) { + downloadFile(url, localPath, network) + filePaths.push(filePath) + } + } + + await new Promise((resolve, reject) => { + if (filePaths.length === 0) resolve() + const onDownloadSuccess = async ({ fileName }: { fileName: string }) => { + if (filePaths.includes(fileName)) { + filePaths.splice(filePaths.indexOf(fileName), 1) + if (filePaths.length === 0) { + events.off(DownloadEvent.onFileDownloadSuccess, onDownloadSuccess) + events.off(DownloadEvent.onFileDownloadError, onDownloadError) + resolve() + } + } + } + + const onDownloadError = async ({ + fileName, + error, + }: { + fileName: string + error: Error + }) => { + if (filePaths.includes(fileName)) { + this.cancelConvert(repoID, repoData) + events.off(DownloadEvent.onFileDownloadSuccess, onDownloadSuccess) + events.off(DownloadEvent.onFileDownloadError, onDownloadError) + reject(error) + } + } + + events.on(DownloadEvent.onFileDownloadSuccess, onDownloadSuccess) + events.on(DownloadEvent.onFileDownloadError, onDownloadError) + }) + } + + /** + * Converts a Hugging Face model to GGUF. + * @param repoID - The repo ID of the model to convert. + * @returns A promise that resolves when the conversion is complete. + */ + async convert(repoID: string): Promise { + if (this.interrupted) return + const modelDirPath = await this.getModelDirPath(repoID) + const modelOutPath = await this.getConvertedModelPath(repoID) + if (!(await fs.existsSync(modelDirPath))) { + throw new Error('Model dir not found') + } + if (await fs.existsSync(modelOutPath)) return + + await executeOnMain(NODE_MODULE_PATH, 'installDeps') + if (this.interrupted) return + + try { + await executeOnMain( + NODE_MODULE_PATH, + 'convertHf', + modelDirPath, + modelOutPath + '.temp' + ) + } catch (err) { + log(`[Conversion]::Debug: Error using hf-to-gguf.py, trying convert.py`) + + let ctx = 4096 + try { + const config = await fs.readFileSync( + await joinPath([modelDirPath, 'config.json']), + 'utf8' + ) + const configParsed = JSON.parse(config) + ctx = this.getCtxLength(configParsed) + configParsed.max_sequence_length = ctx + await fs.writeFileSync( + await joinPath([modelDirPath, 'config.json']), + JSON.stringify(configParsed, null, 2) + ) + } catch (err) { + log(`${err}`) + // ignore missing config.json + } + + const bpe = await fs.existsSync( + await joinPath([modelDirPath, 'vocab.json']) + ) + + await executeOnMain( + NODE_MODULE_PATH, + 'convert', + modelDirPath, + modelOutPath + '.temp', + { + ctx, + bpe, + } + ) + } + await executeOnMain( + NODE_MODULE_PATH, + 'renameSync', + modelOutPath + '.temp', + modelOutPath + ) + + for (const file of await fs.readdirSync(modelDirPath)) { + if ( + modelOutPath.endsWith(file) || + (file.endsWith('config.json') && !file.endsWith('_config.json')) + ) + continue + await fs.unlinkSync(await joinPath([modelDirPath, file])) + } + } + + /** + * Quantizes a GGUF model. + * @param repoID - The repo ID of the model to quantize. + * @param quantization - The quantization to use. + * @returns A promise that resolves when the quantization is complete. + */ + async quantize(repoID: string, quantization: Quantization): Promise { + if (this.interrupted) return + const modelDirPath = await this.getModelDirPath(repoID) + const modelOutPath = await this.getQuantizedModelPath(repoID, quantization) + if (!(await fs.existsSync(modelDirPath))) { + throw new Error('Model dir not found') + } + if (await fs.existsSync(modelOutPath)) return + + await executeOnMain( + NODE_MODULE_PATH, + 'quantize', + await this.getConvertedModelPath(repoID), + modelOutPath + '.temp', + quantization + ) + await executeOnMain( + NODE_MODULE_PATH, + 'renameSync', + modelOutPath + '.temp', + modelOutPath + ) + + await fs.unlinkSync(await this.getConvertedModelPath(repoID)) + } + + /** + * Generates Jan model metadata from a Hugging Face model. + * @param repoID - The repo ID of the model to generate metadata for. + * @param repoData - The repo data of the model to generate metadata for. + * @param quantization - The quantization of the model. + * @returns A promise that resolves when the model metadata generation is complete. + */ + async generateMetadata( + repoID: string, + repoData: HuggingFaceRepoData, + quantization: Quantization + ): Promise { + const modelName = repoID.split('/').slice(1).join('/') + const filename = `${modelName}-${quantization.toLowerCase()}.gguf` + const modelDirPath = await this.getModelDirPath(repoID) + const modelPath = await this.getQuantizedModelPath(repoID, quantization) + const modelConfigPath = await joinPath([modelDirPath, 'model.json']) + if (!(await fs.existsSync(modelPath))) { + throw new Error('Model not found') + } + + const size = await executeOnMain(NODE_MODULE_PATH, 'getSize', modelPath) + let ctx = 4096 + try { + const config = await fs.readFileSync( + await joinPath([modelDirPath, 'config.json']), + 'utf8' + ) + ctx = this.getCtxLength(JSON.parse(config)) + fs.unlinkSync(await joinPath([modelDirPath, 'config.json'])) + } catch (err) { + // ignore missing config.json + } + // maybe later, currently it's gonna use too much memory + // const buffer = await fs.readFileSync(quantizedModelPath) + // const ggufData = ggufMetadata(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength)) + + const metadata: Model = { + object: 'model', + version: 1, + format: 'gguf', + sources: [ + { + url: `https://huggingface.co/${repoID}`, // i think this is just for download but not sure, + filename, + }, + ], + id: modelName, + name: modelName, + created: Date.now(), + description: `Auto converted from Hugging Face model: ${repoID}`, + settings: { + ctx_len: ctx, + prompt_template: '', + llama_model_path: modelName, + }, + parameters: { + temperature: 0.7, + top_p: 0.95, + stream: true, + max_tokens: 4096, + // stop: [''], seems like we dont really need this..? + frequency_penalty: 0, + presence_penalty: 0, + }, + metadata: { + author: repoData.author, + tags: repoData.tags, + size, + }, + engine: InferenceEngine.nitro, + } + + await fs.writeFileSync(modelConfigPath, JSON.stringify(metadata, null, 2)) + } + + /** + * Cancels the convert of current Hugging Face model. + * @param repoID - The repository ID to cancel. + * @param repoData - The repository data to cancel. + * @returns {Promise} A promise that resolves when the download has been cancelled. + */ + async cancelConvert( + repoID: string, + repoData: HuggingFaceRepoData + ): Promise { + this.interrupted = true + const modelDirPath = await this.getModelDirPath(repoID) + const files = this.getFileList(repoData) + for (const file of files) { + const filePath = file + const localPath = await joinPath([modelDirPath, filePath]) + await abortDownload(localPath) + } + // ;(await fs.existsSync(modelDirPath)) && (await fs.rmdirSync(modelDirPath)) + + executeOnMain(NODE_MODULE_PATH, 'killProcesses') + } +} diff --git a/extensions/huggingface-extension/src/node/index.ts b/extensions/huggingface-extension/src/node/index.ts new file mode 100644 index 000000000..cd36c1ab9 --- /dev/null +++ b/extensions/huggingface-extension/src/node/index.ts @@ -0,0 +1,187 @@ +import { PythonShell } from 'python-shell' +import { spawn, ChildProcess } from 'child_process' +import { resolve as presolve, join as pjoin } from 'path' +import type { Quantization } from '@janhq/core' +import { log } from '@janhq/core/node' +import { statSync } from 'fs' +export { renameSync } from 'fs' + +let pythonShell: PythonShell | undefined = undefined +let quantizeProcess: ChildProcess | undefined = undefined + +export const getSize = (path: string): number => statSync(path).size + +export const killProcesses = () => { + if (pythonShell) { + pythonShell.kill() + pythonShell = undefined + } + if (quantizeProcess) { + quantizeProcess.kill() + quantizeProcess = undefined + } +} + +export const getQuantizeExecutable = (): string => { + let binaryFolder = pjoin(__dirname, '..', 'bin') // Current directory by default + let binaryName = 'quantize' + /** + * The binary folder is different for each platform. + */ + if (process.platform === 'win32') { + binaryFolder = pjoin(binaryFolder, 'win') + binaryName = 'quantize.exe' + } else if (process.platform === 'darwin') { + /** + * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) + */ + if (process.arch === 'arm64') { + binaryFolder = pjoin(binaryFolder, 'mac-arm64') + } else { + binaryFolder = pjoin(binaryFolder, 'mac-x64') + } + } else { + binaryFolder = pjoin(binaryFolder, 'linux-cpu') + } + return pjoin(binaryFolder, binaryName) +} + +export const installDeps = (): Promise => { + return new Promise((resolve, reject) => { + const _pythonShell = new PythonShell( + presolve(__dirname, '..', 'scripts', 'install_deps.py') + ) + _pythonShell.on('message', (message) => { + log(`[Install Deps]::Debug: ${message}`) + }) + _pythonShell.on('stderr', (stderr) => { + log(`[Install Deps]::Error: ${stderr}`) + }) + _pythonShell.on('error', (err) => { + pythonShell = undefined + log(`[Install Deps]::Error: ${err}`) + reject(err) + }) + _pythonShell.on('close', () => { + const exitCode = _pythonShell.exitCode + pythonShell = undefined + log( + `[Install Deps]::Debug: Deps installation exited with code: ${exitCode}` + ) + exitCode === 0 ? resolve() : reject(exitCode) + }) + }) +} + +export const convertHf = async ( + modelDirPath: string, + outPath: string +): Promise => { + return await new Promise((resolve, reject) => { + const _pythonShell = new PythonShell( + presolve(__dirname, '..', 'scripts', 'convert-hf-to-gguf.py'), + { + args: [modelDirPath, '--outfile', outPath], + } + ) + pythonShell = _pythonShell + _pythonShell.on('message', (message) => { + log(`[Conversion]::Debug: ${message}`) + }) + _pythonShell.on('stderr', (stderr) => { + log(`[Conversion]::Error: ${stderr}`) + }) + _pythonShell.on('error', (err) => { + pythonShell = undefined + log(`[Conversion]::Error: ${err}`) + reject(err) + }) + _pythonShell.on('close', () => { + const exitCode = _pythonShell.exitCode + pythonShell = undefined + if (exitCode !== 0) { + log(`[Conversion]::Debug: Conversion exited with code: ${exitCode}`) + reject(exitCode) + } else { + resolve() + } + }) + }) +} + +export const convert = async ( + modelDirPath: string, + outPath: string, + { ctx, bpe }: { ctx?: number; bpe?: boolean } +): Promise => { + const args = [modelDirPath, '--outfile', outPath] + if (ctx) { + args.push('--ctx') + args.push(ctx.toString()) + } + if (bpe) { + args.push('--vocab-type') + args.push('bpe') + } + return await new Promise((resolve, reject) => { + const _pythonShell = new PythonShell( + presolve(__dirname, '..', 'scripts', 'convert.py'), + { + args, + } + ) + _pythonShell.on('message', (message) => { + log(`[Conversion]::Debug: ${message}`) + }) + _pythonShell.on('stderr', (stderr) => { + log(`[Conversion]::Error: ${stderr}`) + }) + _pythonShell.on('error', (err) => { + pythonShell = undefined + log(`[Conversion]::Error: ${err}`) + reject(err) + }) + _pythonShell.on('close', () => { + const exitCode = _pythonShell.exitCode + pythonShell = undefined + if (exitCode !== 0) { + log(`[Conversion]::Debug: Conversion exited with code: ${exitCode}`) + reject(exitCode) + } else { + resolve() + } + }) + }) +} + +export const quantize = async ( + modelPath: string, + outPath: string, + quantization: Quantization +): Promise => { + return await new Promise((resolve, reject) => { + const quantizeExecutable = getQuantizeExecutable() + const _quantizeProcess = spawn(quantizeExecutable, [ + modelPath, + outPath, + quantization, + ]) + quantizeProcess = _quantizeProcess + + _quantizeProcess.stdout?.on('data', (data) => { + log(`[Quantization]::Debug: ${data}`) + }) + _quantizeProcess.stderr?.on('data', (data) => { + log(`[Quantization]::Error: ${data}`) + }) + + _quantizeProcess.on('close', (code) => { + if (code !== 0) { + log(`[Quantization]::Debug: Quantization exited with code: ${code}`) + reject(code) + } else { + resolve() + } + }) + }) +} diff --git a/extensions/huggingface-extension/tsconfig.json b/extensions/huggingface-extension/tsconfig.json new file mode 100644 index 000000000..a42f31602 --- /dev/null +++ b/extensions/huggingface-extension/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "moduleResolution": "node", + "target": "es2020", + "module": "ES2020", + "lib": ["es2015", "es2016", "es2017", "dom"], + "strict": true, + "sourceMap": true, + "declaration": true, + "allowSyntheticDefaultImports": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declarationDir": "dist/types", + "outDir": "dist", + "importHelpers": true, + "typeRoots": ["node_modules/@types"], + "resolveJsonModule": true, + }, + "include": ["src"], +} diff --git a/extensions/inference-nitro-extension/bin/version.txt b/extensions/inference-nitro-extension/bin/version.txt index 0b9c01996..e4737652c 100644 --- a/extensions/inference-nitro-extension/bin/version.txt +++ b/extensions/inference-nitro-extension/bin/version.txt @@ -1 +1 @@ -0.3.12 +0.3.13 diff --git a/extensions/inference-nitro-extension/src/node/accelerator.ts b/extensions/inference-nitro-extension/src/node/accelerator.ts index 972f88681..bba4c1b03 100644 --- a/extensions/inference-nitro-extension/src/node/accelerator.ts +++ b/extensions/inference-nitro-extension/src/node/accelerator.ts @@ -23,10 +23,7 @@ const DEFALT_SETTINGS = { gpus_in_use: [], is_initial: true, // TODO: This needs to be set based on user toggle in settings - vulkan: { - enabled: true, - gpu_in_use: '1', - }, + vulkan: false } /** diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index 08baba0d5..f9a668507 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -67,7 +67,7 @@ export const executableNitroFile = (): NitroExecutableOptions => { if (gpuInfo['vulkan'] === true) { binaryFolder = path.join(__dirname, '..', 'bin') - binaryFolder = path.join(binaryFolder, 'win-vulkan') + binaryFolder = path.join(binaryFolder, 'linux-vulkan') vkVisibleDevices = gpuInfo['gpus_in_use'].toString() } } diff --git a/web/containers/ModalTroubleShoot/index.tsx b/web/containers/ModalTroubleShoot/index.tsx index 547398c4f..2438d6333 100644 --- a/web/containers/ModalTroubleShoot/index.tsx +++ b/web/containers/ModalTroubleShoot/index.tsx @@ -69,7 +69,7 @@ const ModalTroubleShooting: React.FC = () => { > Discord -   & send it to #🆘|get-help channel for further support. +  & send it to #🆘|get-help channel for further support.

diff --git a/web/helpers/atoms/HFConverter.atom.ts b/web/helpers/atoms/HFConverter.atom.ts new file mode 100644 index 000000000..717ab05a9 --- /dev/null +++ b/web/helpers/atoms/HFConverter.atom.ts @@ -0,0 +1,44 @@ +import { HuggingFaceRepoData } from '@janhq/core' +import { atom } from 'jotai' + +export const repoIDAtom = atom(null) +export const loadingAtom = atom(false) +export const fetchErrorAtom = atom(null) +export const conversionStatusAtom = atom< + | 'downloading' + | 'converting' + | 'quantizing' + | 'done' + | 'stopping' + | 'generating' + | null +>(null) +export const conversionErrorAtom = atom(null) +const _repoDataAtom = atom(null) +const _unsupportedAtom = atom(false) + +export const resetAtom = atom(null, (_get, set) => { + set(repoIDAtom, null) + set(loadingAtom, false) + set(fetchErrorAtom, null) + set(conversionStatusAtom, null) + set(conversionErrorAtom, null) + set(_repoDataAtom, null) + set(_unsupportedAtom, false) +}) + +export const repoDataAtom = atom( + (get) => get(_repoDataAtom), + (_get, set, repoData: HuggingFaceRepoData) => { + set(_repoDataAtom, repoData) + if ( + !repoData.tags.includes('transformers') || + (!repoData.tags.includes('pytorch') && + !repoData.tags.includes('safetensors')) + ) { + set(_unsupportedAtom, true) + } + } +) + +export const unsupportedAtom = atom((get) => get(_unsupportedAtom)) diff --git a/web/hooks/useConvertHuggingFaceModel.ts b/web/hooks/useConvertHuggingFaceModel.ts new file mode 100644 index 000000000..bbf33207b --- /dev/null +++ b/web/hooks/useConvertHuggingFaceModel.ts @@ -0,0 +1,81 @@ +import { useContext } from 'react' + +import { + ExtensionTypeEnum, + HuggingFaceExtension, + HuggingFaceRepoData, + Quantization, +} from '@janhq/core' + +import { useSetAtom } from 'jotai' + +import { FeatureToggleContext } from '@/context/FeatureToggle' + +import { extensionManager } from '@/extension/ExtensionManager' +import { + conversionStatusAtom, + conversionErrorAtom, +} from '@/helpers/atoms/HFConverter.atom' + +export const useConvertHuggingFaceModel = () => { + const { ignoreSSL, proxy } = useContext(FeatureToggleContext) + const setConversionStatus = useSetAtom(conversionStatusAtom) + const setConversionError = useSetAtom(conversionErrorAtom) + + const convertHuggingFaceModel = async ( + repoID: string, + repoData: HuggingFaceRepoData, + quantization: Quantization + ) => { + const extension = await extensionManager.get( + ExtensionTypeEnum.HuggingFace + ) + try { + if (extension) { + extension.interrupted = false + } + setConversionStatus('downloading') + await extension?.downloadModelFiles(repoID, repoData, { + ignoreSSL, + proxy, + }) + if (extension?.interrupted) return + setConversionStatus('converting') + await extension?.convert(repoID) + if (extension?.interrupted) return + setConversionStatus('quantizing') + await extension?.quantize(repoID, quantization) + if (extension?.interrupted) return + setConversionStatus('generating') + await extension?.generateMetadata(repoID, repoData, quantization) + setConversionStatus('done') + } catch (err) { + if (extension?.interrupted) return + extension?.cancelConvert(repoID, repoData) + if (typeof err === 'number') { + setConversionError(new Error(`exit code: ${err}`)) + } else { + setConversionError(err as Error) + } + console.error(err) + } + } + + const cancelConvertHuggingFaceModel = async ( + repoID: string, + repoData: HuggingFaceRepoData + ) => { + const extension = await extensionManager.get( + ExtensionTypeEnum.HuggingFace + ) + + setConversionStatus('stopping') + await extension?.cancelConvert(repoID, repoData) + setConversionStatus(null) + } + + return { + convertHuggingFaceModel, + cancelConvertHuggingFaceModel, + } +} diff --git a/web/hooks/useGetHFRepoData.ts b/web/hooks/useGetHFRepoData.ts new file mode 100644 index 000000000..45f979fbd --- /dev/null +++ b/web/hooks/useGetHFRepoData.ts @@ -0,0 +1,29 @@ +import { useAtomValue, useSetAtom } from 'jotai' + +import { + repoDataAtom, + repoIDAtom, + loadingAtom, + fetchErrorAtom, +} from '@/helpers/atoms/HFConverter.atom' + +export const useGetHFRepoData = () => { + const repoID = useAtomValue(repoIDAtom) + const setRepoData = useSetAtom(repoDataAtom) + const setLoading = useSetAtom(loadingAtom) + const setFetchError = useSetAtom(fetchErrorAtom) + + const getRepoData = async () => { + setLoading(true) + try { + const res = await fetch(`https://huggingface.co/api/models/${repoID}`) + const data = await res.json() + setRepoData(data) + } catch (err) { + setFetchError(err as Error) + } + setLoading(false) + } + + return getRepoData +} diff --git a/web/screens/Chat/SimpleTextMessage/index.tsx b/web/screens/Chat/SimpleTextMessage/index.tsx index f064132b8..c7f6db274 100644 --- a/web/screens/Chat/SimpleTextMessage/index.tsx +++ b/web/screens/Chat/SimpleTextMessage/index.tsx @@ -207,7 +207,7 @@ const SimpleTextMessage: React.FC = (props) => { {messages[messages.length - 1]?.id === props.id && (props.status === MessageStatus.Pending || tokenSpeed > 0) && (

- Token Speed: {Number(tokenSpeed).toFixed(2)}/s + Token Speed: {Number(tokenSpeed).toFixed(2)}t/s

)} diff --git a/web/screens/ExploreModels/HuggingFaceConvertingErrorModal/index.tsx b/web/screens/ExploreModels/HuggingFaceConvertingErrorModal/index.tsx new file mode 100644 index 000000000..2bd0fde3c --- /dev/null +++ b/web/screens/ExploreModels/HuggingFaceConvertingErrorModal/index.tsx @@ -0,0 +1,30 @@ +import { useAtomValue } from 'jotai' + +import { + conversionErrorAtom, + conversionStatusAtom, + repoDataAtom, +} from '@/helpers/atoms/HFConverter.atom' + +export const HuggingFaceConvertingErrorModal = () => { + // This component only loads when repoData is not null + const repoData = useAtomValue(repoDataAtom)! + // This component only loads when conversionStatus is not null + const conversionStatus = useAtomValue(conversionStatusAtom)! + // This component only loads when conversionError is not null + const conversionError = useAtomValue(conversionErrorAtom)! + + return ( + <> +
+

Hugging Face Converter

+
+
+

+ An error occured while {conversionStatus} model {repoData.id}. +

+

Please close this modal and try again.

+
+ + ) +} diff --git a/web/screens/ExploreModels/HuggingFaceConvertingModal/index.tsx b/web/screens/ExploreModels/HuggingFaceConvertingModal/index.tsx new file mode 100644 index 000000000..175722dda --- /dev/null +++ b/web/screens/ExploreModels/HuggingFaceConvertingModal/index.tsx @@ -0,0 +1,73 @@ +import { useEffect, useState } from 'react' + +import { Button } from '@janhq/uikit' +import { useAtomValue } from 'jotai' + +import { useConvertHuggingFaceModel } from '@/hooks/useConvertHuggingFaceModel' + +import { + conversionStatusAtom, + repoDataAtom, +} from '@/helpers/atoms/HFConverter.atom' + +export const HuggingFaceConvertingModal = () => { + // This component only loads when repoData is not null + const repoData = useAtomValue(repoDataAtom)! + // This component only loads when conversionStatus is not null + const conversionStatus = useAtomValue(conversionStatusAtom)! + const [status, setStatus] = useState('') + const { cancelConvertHuggingFaceModel } = useConvertHuggingFaceModel() + + useEffect(() => { + switch (conversionStatus) { + case 'downloading': + setStatus('Downloading files...') + break + case 'converting': + setStatus('Converting...') + break + case 'quantizing': + setStatus('Quantizing...') + break + case 'stopping': + setStatus('Stopping...') + break + case 'generating': + setStatus('Generating metadata...') + break + } + }, [conversionStatus]) + + const onStopClick = () => { + cancelConvertHuggingFaceModel(repoData.id, repoData) + } + + return ( + <> +
+

Hugging Face Converter

+
+ {conversionStatus === 'done' ? ( +
+

Done!

+

Now you can use the model on Jan as usual. Have fun!

+
+ ) : ( + <> +
+

{status}

+
+ + + )} + + ) +} diff --git a/web/screens/ExploreModels/HuggingFaceModal/index.tsx b/web/screens/ExploreModels/HuggingFaceModal/index.tsx new file mode 100644 index 000000000..9051e15e6 --- /dev/null +++ b/web/screens/ExploreModels/HuggingFaceModal/index.tsx @@ -0,0 +1,70 @@ +import { CommandModal, Modal, ModalContent } from '@janhq/uikit' +import { useAtomValue, useSetAtom } from 'jotai' + +import { HuggingFaceConvertingErrorModal } from '../HuggingFaceConvertingErrorModal' +import { HuggingFaceConvertingModal } from '../HuggingFaceConvertingModal' +import { HuggingFaceRepoDataLoadedModal } from '../HuggingFaceRepoDataLoadedModal' +import { HuggingFaceSearchErrorModal } from '../HuggingFaceSearchErrorModal' +import { HuggingFaceSearchModal } from '../HuggingFaceSearchModal' + +import { + repoDataAtom, + fetchErrorAtom, + resetAtom, + conversionStatusAtom, + conversionErrorAtom, +} from '@/helpers/atoms/HFConverter.atom' + +const HuggingFaceModal = ({ + ...props +}: Omit[0], 'children'>) => { + const repoData = useAtomValue(repoDataAtom) + const fetchError = useAtomValue(fetchErrorAtom) + const conversionStatus = useAtomValue(conversionStatusAtom) + const conversionError = useAtomValue(conversionErrorAtom) + const setReset = useSetAtom(resetAtom) + + return ( + { + if (open === false) { + if ( + !repoData || + ['done', 'stopping'].includes(conversionStatus ?? '') || + conversionError + ) { + setReset() + } + } + if (props.onOpenChange) { + props.onOpenChange(open) + } + }} + > + +
+
+ {repoData ? ( + conversionStatus ? ( + conversionError ? ( + + ) : ( + + ) + ) : ( + + ) + ) : fetchError ? ( + + ) : ( + + )} +
+
+ + + ) +} + +export { HuggingFaceModal } diff --git a/web/screens/ExploreModels/HuggingFaceRepoDataLoadedModal/index.tsx b/web/screens/ExploreModels/HuggingFaceRepoDataLoadedModal/index.tsx new file mode 100644 index 000000000..c4e9131bc --- /dev/null +++ b/web/screens/ExploreModels/HuggingFaceRepoDataLoadedModal/index.tsx @@ -0,0 +1,100 @@ +import { useState } from 'react' + +import { Quantization } from '@janhq/core' +import { + Button, + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectPortal, + SelectTrigger, + SelectValue, +} from '@janhq/uikit' +import { useAtomValue } from 'jotai' + +import { twMerge } from 'tailwind-merge' + +import { useConvertHuggingFaceModel } from '@/hooks/useConvertHuggingFaceModel' + +import { + loadingAtom, + repoDataAtom, + unsupportedAtom, +} from '@/helpers/atoms/HFConverter.atom' + +export const HuggingFaceRepoDataLoadedModal = () => { + const loading = useAtomValue(loadingAtom) + // This component only loads when repoData is not null + const repoData = useAtomValue(repoDataAtom)! + const unsupported = useAtomValue(unsupportedAtom) + const [quantization, setQuantization] = useState( + Quantization.Q4_K_M + ) + const { convertHuggingFaceModel } = useConvertHuggingFaceModel() + + const onValueSelected = (value: Quantization) => { + setQuantization(value) + } + const onConvertClick = () => { + convertHuggingFaceModel(repoData.id, repoData, quantization) + } + + return ( + <> +
+

Hugging Face Converter

+

Found the repository!

+
+
+

{repoData.id}

+

+ {unsupported + ? '❌ This model is not supported!' + : '✅ This model is supported!'} +

+ {repoData.tags.includes('gguf') ? ( +

...But you can import it manually!

+ ) : null} +
+ + + + ) +} diff --git a/web/screens/ExploreModels/HuggingFaceSearchErrorModal/index.tsx b/web/screens/ExploreModels/HuggingFaceSearchErrorModal/index.tsx new file mode 100644 index 000000000..31c7d48d4 --- /dev/null +++ b/web/screens/ExploreModels/HuggingFaceSearchErrorModal/index.tsx @@ -0,0 +1,32 @@ +import { Button } from '@janhq/uikit' +import { useAtomValue } from 'jotai' + +import { useGetHFRepoData } from '@/hooks/useGetHFRepoData' + +import { fetchErrorAtom, loadingAtom } from '@/helpers/atoms/HFConverter.atom' + +export const HuggingFaceSearchErrorModal = () => { + // This component only loads when fetchError is not null + const fetchError = useAtomValue(fetchErrorAtom)! + const loading = useAtomValue(loadingAtom) + + const getRepoData = useGetHFRepoData() + + return ( + <> +
+

Error!

+

Fetch error

+
+

{fetchError.message}

+ + + ) +} diff --git a/web/screens/ExploreModels/HuggingFaceSearchModal/index.tsx b/web/screens/ExploreModels/HuggingFaceSearchModal/index.tsx new file mode 100644 index 000000000..a81df29fa --- /dev/null +++ b/web/screens/ExploreModels/HuggingFaceSearchModal/index.tsx @@ -0,0 +1,45 @@ +import { Button, Input } from '@janhq/uikit' +import { useSetAtom, useAtomValue } from 'jotai' + +import { useGetHFRepoData } from '@/hooks/useGetHFRepoData' + +import { repoIDAtom, loadingAtom } from '@/helpers/atoms/HFConverter.atom' + +export const HuggingFaceSearchModal = () => { + const setRepoID = useSetAtom(repoIDAtom) + const loading = useAtomValue(loadingAtom) + + const getRepoData = useGetHFRepoData() + + const onKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + e.preventDefault() + getRepoData() + } + } + + return ( + <> +
+

Hugging Face Convertor

+

Type the repository id below

+
+ { + setRepoID(e.target.value) + }} + onKeyDown={onKeyDown} + /> + + + ) +} diff --git a/web/screens/ExploreModels/index.tsx b/web/screens/ExploreModels/index.tsx index 7002c60b7..96e1356ac 100644 --- a/web/screens/ExploreModels/index.tsx +++ b/web/screens/ExploreModels/index.tsx @@ -16,6 +16,7 @@ import { useAtomValue } from 'jotai' import { SearchIcon } from 'lucide-react' import ExploreModelList from './ExploreModelList' +import { HuggingFaceModal } from './HuggingFaceModal' import { configuredModelsAtom, @@ -28,6 +29,7 @@ const ExploreModelsScreen = () => { const [searchValue, setsearchValue] = useState('') const [sortSelected, setSortSelected] = useState('All Models') const sortMenu = ['All Models', 'Recommended', 'Downloaded'] + const [showHuggingFaceModal, setShowHuggingFaceModal] = useState(false) const filteredModels = configuredModels.filter((x) => { if (sortSelected === 'Downloaded') { @@ -49,6 +51,10 @@ const ExploreModelsScreen = () => { openExternalUrl('https://jan.ai/guides/using-models/import-manually/') }, []) + const onHuggingFaceConverterClick = () => { + setShowHuggingFaceModal(true) + } + return (
{ >
+
{ How to manually import models

+
+

or

+

+ Convert from Hugging Face +

+