feat: add a simple way to convert Hugging Face model to GGUF (#1972)
* chore: add react developer tools to electron * feat: add small convert modal * feat: separate modals and add hugging face extension * feat: fully implement hugging face converter * fix: forgot to uncomment this... * fix: typo * feat: try hf-to-gguf script first and then use convert.py HF-to-GGUF has support for some unusual models maybe using convert.py first would be better but we can change the usage order later * fix: pre-install directory changed * fix: sometimes exit code is undefined * chore: download additional files for qwen * fix: event handling changed * chore: add one more necessary package * feat: download gguf-py from llama.cpp * fix: cannot interpret wildcards on GNU tar Co-authored-by: hiento09 <136591877+hiento09@users.noreply.github.com> --------- Co-authored-by: hiento09 <136591877+hiento09@users.noreply.github.com>
This commit is contained in:
Helloyunho
GitHub
parent
2b676fee42
commit
e86cd7e661
@ -4,6 +4,7 @@ export enum ExtensionTypeEnum {
|
||||
Inference = 'inference',
|
||||
Model = 'model',
|
||||
SystemMonitoring = 'systemMonitoring',
|
||||
HuggingFace = 'huggingFace',
|
||||
}
|
||||
|
||||
export interface ExtensionType {
|
||||
|
||||
30
core/src/extensions/huggingface.ts
Normal file
30
core/src/extensions/huggingface.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import { BaseExtension, ExtensionTypeEnum } from '../extension'
|
||||
import { HuggingFaceInterface, HuggingFaceRepoData, Quantization } from '../types/huggingface'
|
||||
import { Model } from '../types/model'
|
||||
|
||||
/**
|
||||
* Hugging Face extension for converting HF models to GGUF.
|
||||
*/
|
||||
export abstract class HuggingFaceExtension extends BaseExtension implements HuggingFaceInterface {
|
||||
interrupted = false
|
||||
/**
|
||||
* Hugging Face extension type.
|
||||
*/
|
||||
type(): ExtensionTypeEnum | undefined {
|
||||
return ExtensionTypeEnum.HuggingFace
|
||||
}
|
||||
|
||||
abstract downloadModelFiles(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
network?: { ignoreSSL?: boolean; proxy?: string }
|
||||
): Promise<void>
|
||||
abstract convert(repoID: string): Promise<void>
|
||||
abstract quantize(repoID: string, quantization: Quantization): Promise<void>
|
||||
abstract generateMetadata(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
quantization: Quantization
|
||||
): Promise<void>
|
||||
abstract cancelConvert(repoID: string, repoData: HuggingFaceRepoData): Promise<void>
|
||||
}
|
||||
@ -23,3 +23,8 @@ export { AssistantExtension } from './assistant'
|
||||
* Model extension for managing models.
|
||||
*/
|
||||
export { ModelExtension } from './model'
|
||||
|
||||
/**
|
||||
* Hugging Face extension for converting HF models to GGUF.
|
||||
*/
|
||||
export { HuggingFaceExtension } from './huggingface'
|
||||
|
||||
34
core/src/types/huggingface/huggingfaceEntity.ts
Normal file
34
core/src/types/huggingface/huggingfaceEntity.ts
Normal file
@ -0,0 +1,34 @@
|
||||
export interface HuggingFaceRepoData {
|
||||
id: string
|
||||
author: string
|
||||
tags: Array<'transformers' | 'pytorch' | 'safetensors' | string>
|
||||
siblings: {
|
||||
rfilename: string
|
||||
}[]
|
||||
createdAt: string // ISO 8601 timestamp
|
||||
}
|
||||
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
export enum Quantization {
|
||||
Q3_K_S = 'Q3_K_S',
|
||||
Q3_K_M = 'Q3_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values
|
||||
Q3_K_L = 'Q3_K_L',
|
||||
Q4_K_S = 'Q4_K_S',
|
||||
Q4_K_M = 'Q4_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values
|
||||
Q5_K_S = 'Q5_K_S',
|
||||
Q5_K_M = 'Q5_K_M', // eslint-disable-line @typescript-eslint/no-duplicate-enum-values
|
||||
Q4_0 = 'Q4_0',
|
||||
Q4_1 = 'Q4_1',
|
||||
Q5_0 = 'Q5_0',
|
||||
Q5_1 = 'Q5_1',
|
||||
IQ2_XXS = 'IQ2_XXS',
|
||||
IQ2_XS = 'IQ2_XS',
|
||||
Q2_K = 'Q2_K',
|
||||
Q2_K_S = 'Q2_K_S',
|
||||
Q6_K = 'Q6_K',
|
||||
Q8_0 = 'Q8_0',
|
||||
F16 = 'F16',
|
||||
F32 = 'F32',
|
||||
COPY = 'COPY',
|
||||
}
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
58
core/src/types/huggingface/huggingfaceInterface.ts
Normal file
58
core/src/types/huggingface/huggingfaceInterface.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import { Model } from '../model'
|
||||
import { HuggingFaceRepoData, Quantization } from './huggingfaceEntity'
|
||||
|
||||
/**
|
||||
* Hugging Face extension for converting HF models to GGUF.
|
||||
* @extends BaseExtension
|
||||
*/
|
||||
export interface HuggingFaceInterface {
|
||||
interrupted: boolean
|
||||
/**
|
||||
* Downloads a Hugging Face model.
|
||||
* @param repoID - The repo ID of the model to convert.
|
||||
* @param repoData - The repo data of the model to convert.
|
||||
* @param network - Optional object to specify proxy/whether to ignore SSL certificates.
|
||||
* @returns A promise that resolves when the download is complete.
|
||||
*/
|
||||
downloadModelFiles(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
network?: { ignoreSSL?: boolean; proxy?: string }
|
||||
): Promise<void>
|
||||
|
||||
/**
|
||||
* Converts a Hugging Face model to GGUF.
|
||||
* @param repoID - The repo ID of the model to convert.
|
||||
* @returns A promise that resolves when the conversion is complete.
|
||||
*/
|
||||
convert(repoID: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Quantizes a GGUF model.
|
||||
* @param repoID - The repo ID of the model to quantize.
|
||||
* @param quantization - The quantization to use.
|
||||
* @returns A promise that resolves when the quantization is complete.
|
||||
*/
|
||||
quantize(repoID: string, quantization: Quantization): Promise<void>
|
||||
|
||||
/**
|
||||
* Generates Jan model metadata from a Hugging Face model.
|
||||
* @param repoID - The repo ID of the model to generate metadata for.
|
||||
* @param repoData - The repo data of the model to generate metadata for.
|
||||
* @param quantization - The quantization of the model.
|
||||
* @returns A promise that resolves when the model metadata generation is complete.
|
||||
*/
|
||||
generateMetadata(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
quantization: Quantization
|
||||
): Promise<void>
|
||||
|
||||
/**
|
||||
* Cancels the convert of current Hugging Face model.
|
||||
* @param repoID - The repository ID to cancel.
|
||||
* @param repoData - The repository data to cancel.
|
||||
* @returns {Promise<void>} A promise that resolves when the download has been cancelled.
|
||||
*/
|
||||
cancelConvert(repoID: string, repoData: HuggingFaceRepoData): Promise<void>
|
||||
}
|
||||
2
core/src/types/huggingface/index.ts
Normal file
2
core/src/types/huggingface/index.ts
Normal file
@ -0,0 +1,2 @@
|
||||
export * from './huggingfaceInterface'
|
||||
export * from './huggingfaceEntity'
|
||||
@ -6,4 +6,5 @@ export * from './inference'
|
||||
export * from './monitoring'
|
||||
export * from './file'
|
||||
export * from './config'
|
||||
export * from './huggingface'
|
||||
export * from './miscellaneous'
|
||||
|
||||
@ -15,12 +15,14 @@
|
||||
"build/**/*.{js,map}",
|
||||
"pre-install",
|
||||
"models/**/*",
|
||||
"docs/**/*"
|
||||
"docs/**/*",
|
||||
"scripts/**/*"
|
||||
],
|
||||
"asarUnpack": [
|
||||
"pre-install",
|
||||
"models",
|
||||
"docs"
|
||||
"docs",
|
||||
"scripts"
|
||||
],
|
||||
"publish": [
|
||||
{
|
||||
|
||||
3
extensions/huggingface-extension/.gitignore
vendored
Normal file
3
extensions/huggingface-extension/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
bin
|
||||
scripts/convert*
|
||||
scripts/gguf-py
|
||||
8
extensions/huggingface-extension/.prettierrc
Normal file
8
extensions/huggingface-extension/.prettierrc
Normal file
@ -0,0 +1,8 @@
|
||||
{
|
||||
"semi": false,
|
||||
"singleQuote": true,
|
||||
"quoteProps": "consistent",
|
||||
"trailingComma": "es5",
|
||||
"endOfLine": "auto",
|
||||
"plugins": ["prettier-plugin-tailwindcss"]
|
||||
}
|
||||
73
extensions/huggingface-extension/README.md
Normal file
73
extensions/huggingface-extension/README.md
Normal file
@ -0,0 +1,73 @@
|
||||
# Create a Jan Plugin using Typescript
|
||||
|
||||
Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀
|
||||
|
||||
## Create Your Own Plugin
|
||||
|
||||
To create your own plugin, you can use this repository as a template! Just follow the below instructions:
|
||||
|
||||
1. Click the Use this template button at the top of the repository
|
||||
2. Select Create a new repository
|
||||
3. Select an owner and name for your new repository
|
||||
4. Click Create repository
|
||||
5. Clone your new repository
|
||||
|
||||
## Initial Setup
|
||||
|
||||
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin.
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> You'll need to have a reasonably modern version of
|
||||
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
|
||||
> [`nodenv`](https://github.com/nodenv/nodenv) or
|
||||
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
|
||||
> root of your repository to install the version specified in
|
||||
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
|
||||
|
||||
1. :hammer_and_wrench: Install the dependencies
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
1. :building_construction: Package the TypeScript for distribution
|
||||
|
||||
```bash
|
||||
npm run bundle
|
||||
```
|
||||
|
||||
1. :white_check_mark: Check your artifact
|
||||
|
||||
There will be a tgz file in your plugin directory now
|
||||
|
||||
## Update the Plugin Metadata
|
||||
|
||||
The [`package.json`](package.json) file defines metadata about your plugin, such as
|
||||
plugin name, main entry, description and version.
|
||||
|
||||
When you copy this repository, update `package.json` with the name, description for your plugin.
|
||||
|
||||
## Update the Plugin Code
|
||||
|
||||
The [`src/`](./src/) directory is the heart of your plugin! This contains the
|
||||
source code that will be run when your plugin extension functions are invoked. You can replace the
|
||||
contents of this directory with your own code.
|
||||
|
||||
There are a few things to keep in mind when writing your plugin code:
|
||||
|
||||
- Most Jan Plugin Extension functions are processed asynchronously.
|
||||
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
|
||||
|
||||
```typescript
|
||||
import { core } from "@janhq/core";
|
||||
|
||||
function onStart(): Promise<any> {
|
||||
return core.invokePluginFunc(MODULE_PATH, "run", 0);
|
||||
}
|
||||
```
|
||||
|
||||
For more information about the Jan Plugin Core module, see the
|
||||
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
|
||||
|
||||
So, what are you waiting for? Go ahead and start customizing your plugin!
|
||||
BIN
extensions/huggingface-extension/bin/mac-arm64/quantize
Executable file
BIN
extensions/huggingface-extension/bin/mac-arm64/quantize
Executable file
Binary file not shown.
3
extensions/huggingface-extension/download.bat
Normal file
3
extensions/huggingface-extension/download.bat
Normal file
@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
set /p LLAMA_CPP_VERSION=<./scripts/version.txt
|
||||
.\node_modules\.bin\download https://github.com/ggerganov/llama.cpp/archive/refs/tags/%LLAMA_CPP_VERSION%.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf .\scripts\llama.cpp.tar.gz "llama.cpp-%LLAMA_CPP_VERSION%/convert.py" "llama.cpp-%LLAMA_CPP_VERSION%/convert-hf-to-gguf.py" "llama.cpp-%LLAMA_CPP_VERSION%/gguf-py" && cpx "./llama.cpp-%LLAMA_CPP_VERSION%/**" "scripts" && rimraf "./scripts/llama.cpp.tar.gz" && rimraf "./llama.cpp-%LLAMA_CPP_VERSION%"
|
||||
54
extensions/huggingface-extension/package.json
Normal file
54
extensions/huggingface-extension/package.json
Normal file
@ -0,0 +1,54 @@
|
||||
{
|
||||
"name": "@janhq/huggingface-extension",
|
||||
"version": "1.0.0",
|
||||
"description": "Hugging Face extension for converting HF models to GGUF",
|
||||
"main": "dist/index.js",
|
||||
"node": "dist/node/index.cjs.js",
|
||||
"author": "Jan <service@jan.ai>",
|
||||
"license": "AGPL-3.0",
|
||||
"scripts": {
|
||||
"build": "tsc --module commonjs && rollup -c rollup.config.ts --configPlugin @rollup/plugin-typescript --bundleConfigAsCjs",
|
||||
"download:llama": "run-script-os",
|
||||
"download:llama:linux": "LLAMA_CPP_VERSION=$(cat ./scripts/version.txt) && download https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf ./scripts/llama.cpp.tar.gz --wildcards '*/convert.py' '*/convert-hf-to-gguf.py' '*/gguf-py' && cpx \"./llama.cpp-$LLAMA_CPP_VERSION/**\" \"scripts\" && rimraf \"./scripts/llama.cpp.tar.gz\" && rimraf \"./llama.cpp-$LLAMA_CPP_VERSION\"",
|
||||
"download:llama:darwin": "LLAMA_CPP_VERSION=$(cat ./scripts/version.txt) && download https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz -o . --filename ./scripts/llama.cpp.tar.gz && tar -xzf ./scripts/llama.cpp.tar.gz '*/convert.py' '*/convert-hf-to-gguf.py' '*/gguf-py' && cpx \"./llama.cpp-$LLAMA_CPP_VERSION/**\" \"scripts\" && rimraf \"./scripts/llama.cpp.tar.gz\" && rimraf \"./llama.cpp-$LLAMA_CPP_VERSION\"",
|
||||
"download:llama:win32": "download.bat",
|
||||
"build:publish": "rimraf *.tgz --glob && npm run build && npm run download:llama && cpx \"scripts/**\" \"dist/scripts\" && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install"
|
||||
},
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
"./main": "./dist/node/index.cjs.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-commonjs": "^25.0.7",
|
||||
"@rollup/plugin-json": "^6.1.0",
|
||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
||||
"@rollup/plugin-replace": "^5.0.5",
|
||||
"@rollup/plugin-typescript": "^11.1.6",
|
||||
"@types/node": "^20.11.16",
|
||||
"cpx": "^1.5.0",
|
||||
"download-cli": "^1.1.1",
|
||||
"rimraf": "^5.0.5",
|
||||
"rollup": "^4.9.6",
|
||||
"rollup-plugin-sourcemaps": "^0.6.3",
|
||||
"rollup-plugin-typescript2": "^0.36.0",
|
||||
"run-script-os": "^1.1.6",
|
||||
"typescript": "^5.3.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@janhq/core": "file:../../core",
|
||||
"hyllama": "^0.1.2",
|
||||
"python-shell": "^5.0.0",
|
||||
"ts-loader": "^9.5.0"
|
||||
},
|
||||
"bundledDependencies": [
|
||||
"python-shell"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"files": [
|
||||
"dist/*",
|
||||
"package.json",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
72
extensions/huggingface-extension/rollup.config.ts
Normal file
72
extensions/huggingface-extension/rollup.config.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import resolve from '@rollup/plugin-node-resolve'
|
||||
import commonjs from '@rollup/plugin-commonjs'
|
||||
import sourceMaps from 'rollup-plugin-sourcemaps'
|
||||
import typescript from 'rollup-plugin-typescript2'
|
||||
import json from '@rollup/plugin-json'
|
||||
import replace from '@rollup/plugin-replace'
|
||||
|
||||
const packageJson = require('./package.json')
|
||||
|
||||
export default [
|
||||
{
|
||||
input: `src/index.ts`,
|
||||
output: [{ file: packageJson.main, format: 'es', sourcemap: true }],
|
||||
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
|
||||
external: [],
|
||||
watch: {
|
||||
include: 'src/**',
|
||||
},
|
||||
plugins: [
|
||||
replace({
|
||||
EXTENSION_NAME: JSON.stringify(packageJson.name),
|
||||
NODE_MODULE_PATH: JSON.stringify(
|
||||
`${packageJson.name}/${packageJson.node}`
|
||||
),
|
||||
}),
|
||||
// Allow json resolution
|
||||
json(),
|
||||
// Compile TypeScript files
|
||||
typescript({ useTsconfigDeclarationDir: true }),
|
||||
// Compile TypeScript files
|
||||
// Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
|
||||
commonjs(),
|
||||
// Allow node_modules resolution, so you can use 'external' to control
|
||||
// which external modules to include in the bundle
|
||||
// https://github.com/rollup/rollup-plugin-node-resolve#usage
|
||||
resolve({
|
||||
extensions: ['.js', '.ts'],
|
||||
}),
|
||||
|
||||
// Resolve source maps to the original source
|
||||
sourceMaps(),
|
||||
],
|
||||
},
|
||||
{
|
||||
input: `src/node/index.ts`,
|
||||
output: [
|
||||
{ file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
|
||||
],
|
||||
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
|
||||
external: [],
|
||||
watch: {
|
||||
include: 'src/node/**',
|
||||
},
|
||||
plugins: [
|
||||
// Allow json resolution
|
||||
json(),
|
||||
// Compile TypeScript files
|
||||
typescript({ useTsconfigDeclarationDir: true }),
|
||||
// Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
|
||||
commonjs(),
|
||||
// Allow node_modules resolution, so you can use 'external' to control
|
||||
// which external modules to include in the bundle
|
||||
// https://github.com/rollup/rollup-plugin-node-resolve#usage
|
||||
resolve({
|
||||
extensions: ['.ts', '.js', '.json'],
|
||||
}),
|
||||
|
||||
// Resolve source maps to the original source
|
||||
sourceMaps(),
|
||||
],
|
||||
},
|
||||
]
|
||||
14
extensions/huggingface-extension/scripts/install_deps.py
Normal file
14
extensions/huggingface-extension/scripts/install_deps.py
Normal file
@ -0,0 +1,14 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
deps = [
|
||||
'numpy~=1.24.4',
|
||||
'sentencepiece~=0.1.98',
|
||||
'transformers>=4.35.2,<5.0.0',
|
||||
'gguf>=0.1.0',
|
||||
'protobuf>=4.21.0,<5.0.0',
|
||||
'torch~=2.1.1',
|
||||
'packaging>=20.0',
|
||||
'tiktoken~=0.5.0'
|
||||
]
|
||||
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--upgrade', '--force-reinstall', *deps])
|
||||
1
extensions/huggingface-extension/scripts/version.txt
Normal file
1
extensions/huggingface-extension/scripts/version.txt
Normal file
@ -0,0 +1 @@
|
||||
b2106
|
||||
2
extensions/huggingface-extension/src/@types/global.d.ts
vendored
Normal file
2
extensions/huggingface-extension/src/@types/global.d.ts
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
declare const EXTENSION_NAME: string
|
||||
declare const NODE_MODULE_PATH: string
|
||||
396
extensions/huggingface-extension/src/index.ts
Normal file
396
extensions/huggingface-extension/src/index.ts
Normal file
@ -0,0 +1,396 @@
|
||||
import {
|
||||
fs,
|
||||
downloadFile,
|
||||
abortDownload,
|
||||
joinPath,
|
||||
HuggingFaceExtension,
|
||||
HuggingFaceRepoData,
|
||||
executeOnMain,
|
||||
Quantization,
|
||||
Model,
|
||||
InferenceEngine,
|
||||
getJanDataFolderPath,
|
||||
events,
|
||||
DownloadEvent,
|
||||
log,
|
||||
} from '@janhq/core'
|
||||
import { ggufMetadata } from 'hyllama'
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
electronAPI?: any
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A extension for models
|
||||
*/
|
||||
export default class JanHuggingFaceExtension extends HuggingFaceExtension {
|
||||
private static readonly _safetensorsRegexs = [
|
||||
/model\.safetensors$/,
|
||||
/model-[0-9]+-of-[0-9]+\.safetensors$/,
|
||||
]
|
||||
private static readonly _pytorchRegexs = [
|
||||
/pytorch_model\.bin$/,
|
||||
/consolidated\.[0-9]+\.pth$/,
|
||||
/pytorch_model-[0-9]+-of-[0-9]+\.bin$/,
|
||||
/.*\.pt$/,
|
||||
]
|
||||
interrupted = false
|
||||
|
||||
/**
|
||||
* Called when the extension is loaded.
|
||||
* @override
|
||||
*/
|
||||
onLoad() {}
|
||||
|
||||
/**
|
||||
* Called when the extension is unloaded.
|
||||
* @override
|
||||
*/
|
||||
onUnload(): void {}
|
||||
|
||||
private getFileList(repoData: HuggingFaceRepoData): string[] {
|
||||
// SafeTensors first, if not, then PyTorch
|
||||
const modelFiles = repoData.siblings
|
||||
.map((file) => file.rfilename)
|
||||
.filter((file) =>
|
||||
JanHuggingFaceExtension._safetensorsRegexs.some((regex) =>
|
||||
regex.test(file)
|
||||
)
|
||||
)
|
||||
if (modelFiles.length === 0) {
|
||||
repoData.siblings.forEach((file) => {
|
||||
if (
|
||||
JanHuggingFaceExtension._pytorchRegexs.some((regex) =>
|
||||
regex.test(file.rfilename)
|
||||
)
|
||||
) {
|
||||
modelFiles.push(file.rfilename)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const vocabFiles = [
|
||||
'tokenizer.model',
|
||||
'vocab.json',
|
||||
'tokenizer.json',
|
||||
].filter((file) =>
|
||||
repoData.siblings.some((sibling) => sibling.rfilename === file)
|
||||
)
|
||||
|
||||
const etcFiles = repoData.siblings
|
||||
.map((file) => file.rfilename)
|
||||
.filter(
|
||||
(file) =>
|
||||
(file.endsWith('.json') && !vocabFiles.includes(file)) ||
|
||||
file.endsWith('.txt') ||
|
||||
file.endsWith('.py') ||
|
||||
file.endsWith('.tiktoken')
|
||||
)
|
||||
|
||||
return [...modelFiles, ...vocabFiles, ...etcFiles]
|
||||
}
|
||||
|
||||
private async getModelDirPath(repoID: string): Promise<string> {
|
||||
const modelName = repoID.split('/').slice(1).join('/')
|
||||
return joinPath([await getJanDataFolderPath(), 'models', modelName])
|
||||
}
|
||||
private async getConvertedModelPath(repoID: string): Promise<string> {
|
||||
const modelName = repoID.split('/').slice(1).join('/')
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
return joinPath([modelDirPath, modelName + '.gguf'])
|
||||
}
|
||||
private async getQuantizedModelPath(
|
||||
repoID: string,
|
||||
quantization: Quantization
|
||||
): Promise<string> {
|
||||
const modelName = repoID.split('/').slice(1).join('/')
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
return joinPath([
|
||||
modelDirPath,
|
||||
modelName + `-${quantization.toLowerCase()}.gguf`,
|
||||
])
|
||||
}
|
||||
private getCtxLength(config: {
|
||||
max_sequence_length?: number
|
||||
max_position_embeddings?: number
|
||||
n_ctx?: number
|
||||
}): number {
|
||||
if (config.max_sequence_length) return config.max_sequence_length
|
||||
if (config.max_position_embeddings) return config.max_position_embeddings
|
||||
if (config.n_ctx) return config.n_ctx
|
||||
return 4096
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads a Hugging Face model.
|
||||
* @param repoID - The repo ID of the model to convert.
|
||||
* @param repoData - The repo data of the model to convert.
|
||||
* @param network - Optional object to specify proxy/whether to ignore SSL certificates.
|
||||
* @returns A promise that resolves when the download is complete.
|
||||
*/
|
||||
async downloadModelFiles(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
network?: { ignoreSSL?: boolean; proxy?: string }
|
||||
): Promise<void> {
|
||||
if (this.interrupted) return
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
if (!(await fs.existsSync(modelDirPath))) await fs.mkdirSync(modelDirPath)
|
||||
const files = this.getFileList(repoData)
|
||||
const filePaths: string[] = []
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = file
|
||||
const localPath = await joinPath([modelDirPath, filePath])
|
||||
const url = `https://huggingface.co/${repoID}/resolve/main/${filePath}`
|
||||
|
||||
if (this.interrupted) return
|
||||
if (!(await fs.existsSync(localPath))) {
|
||||
downloadFile(url, localPath, network)
|
||||
filePaths.push(filePath)
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
if (filePaths.length === 0) resolve()
|
||||
const onDownloadSuccess = async ({ fileName }: { fileName: string }) => {
|
||||
if (filePaths.includes(fileName)) {
|
||||
filePaths.splice(filePaths.indexOf(fileName), 1)
|
||||
if (filePaths.length === 0) {
|
||||
events.off(DownloadEvent.onFileDownloadSuccess, onDownloadSuccess)
|
||||
events.off(DownloadEvent.onFileDownloadError, onDownloadError)
|
||||
resolve()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const onDownloadError = async ({
|
||||
fileName,
|
||||
error,
|
||||
}: {
|
||||
fileName: string
|
||||
error: Error
|
||||
}) => {
|
||||
if (filePaths.includes(fileName)) {
|
||||
this.cancelConvert(repoID, repoData)
|
||||
events.off(DownloadEvent.onFileDownloadSuccess, onDownloadSuccess)
|
||||
events.off(DownloadEvent.onFileDownloadError, onDownloadError)
|
||||
reject(error)
|
||||
}
|
||||
}
|
||||
|
||||
events.on(DownloadEvent.onFileDownloadSuccess, onDownloadSuccess)
|
||||
events.on(DownloadEvent.onFileDownloadError, onDownloadError)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Hugging Face model to GGUF.
|
||||
* @param repoID - The repo ID of the model to convert.
|
||||
* @returns A promise that resolves when the conversion is complete.
|
||||
*/
|
||||
async convert(repoID: string): Promise<void> {
|
||||
if (this.interrupted) return
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
const modelOutPath = await this.getConvertedModelPath(repoID)
|
||||
if (!(await fs.existsSync(modelDirPath))) {
|
||||
throw new Error('Model dir not found')
|
||||
}
|
||||
if (await fs.existsSync(modelOutPath)) return
|
||||
|
||||
await executeOnMain(NODE_MODULE_PATH, 'installDeps')
|
||||
if (this.interrupted) return
|
||||
|
||||
try {
|
||||
await executeOnMain(
|
||||
NODE_MODULE_PATH,
|
||||
'convertHf',
|
||||
modelDirPath,
|
||||
modelOutPath + '.temp'
|
||||
)
|
||||
} catch (err) {
|
||||
log(`[Conversion]::Debug: Error using hf-to-gguf.py, trying convert.py`)
|
||||
|
||||
let ctx = 4096
|
||||
try {
|
||||
const config = await fs.readFileSync(
|
||||
await joinPath([modelDirPath, 'config.json']),
|
||||
'utf8'
|
||||
)
|
||||
const configParsed = JSON.parse(config)
|
||||
ctx = this.getCtxLength(configParsed)
|
||||
configParsed.max_sequence_length = ctx
|
||||
await fs.writeFileSync(
|
||||
await joinPath([modelDirPath, 'config.json']),
|
||||
JSON.stringify(configParsed, null, 2)
|
||||
)
|
||||
} catch (err) {
|
||||
log(`${err}`)
|
||||
// ignore missing config.json
|
||||
}
|
||||
|
||||
const bpe = await fs.existsSync(
|
||||
await joinPath([modelDirPath, 'vocab.json'])
|
||||
)
|
||||
|
||||
await executeOnMain(
|
||||
NODE_MODULE_PATH,
|
||||
'convert',
|
||||
modelDirPath,
|
||||
modelOutPath + '.temp',
|
||||
{
|
||||
ctx,
|
||||
bpe,
|
||||
}
|
||||
)
|
||||
}
|
||||
await executeOnMain(
|
||||
NODE_MODULE_PATH,
|
||||
'renameSync',
|
||||
modelOutPath + '.temp',
|
||||
modelOutPath
|
||||
)
|
||||
|
||||
for (const file of await fs.readdirSync(modelDirPath)) {
|
||||
if (
|
||||
modelOutPath.endsWith(file) ||
|
||||
(file.endsWith('config.json') && !file.endsWith('_config.json'))
|
||||
)
|
||||
continue
|
||||
await fs.unlinkSync(await joinPath([modelDirPath, file]))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Quantizes a GGUF model.
|
||||
* @param repoID - The repo ID of the model to quantize.
|
||||
* @param quantization - The quantization to use.
|
||||
* @returns A promise that resolves when the quantization is complete.
|
||||
*/
|
||||
async quantize(repoID: string, quantization: Quantization): Promise<void> {
|
||||
if (this.interrupted) return
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
const modelOutPath = await this.getQuantizedModelPath(repoID, quantization)
|
||||
if (!(await fs.existsSync(modelDirPath))) {
|
||||
throw new Error('Model dir not found')
|
||||
}
|
||||
if (await fs.existsSync(modelOutPath)) return
|
||||
|
||||
await executeOnMain(
|
||||
NODE_MODULE_PATH,
|
||||
'quantize',
|
||||
await this.getConvertedModelPath(repoID),
|
||||
modelOutPath + '.temp',
|
||||
quantization
|
||||
)
|
||||
await executeOnMain(
|
||||
NODE_MODULE_PATH,
|
||||
'renameSync',
|
||||
modelOutPath + '.temp',
|
||||
modelOutPath
|
||||
)
|
||||
|
||||
await fs.unlinkSync(await this.getConvertedModelPath(repoID))
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates Jan model metadata from a Hugging Face model.
|
||||
* @param repoID - The repo ID of the model to generate metadata for.
|
||||
* @param repoData - The repo data of the model to generate metadata for.
|
||||
* @param quantization - The quantization of the model.
|
||||
* @returns A promise that resolves when the model metadata generation is complete.
|
||||
*/
|
||||
async generateMetadata(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
quantization: Quantization
|
||||
): Promise<void> {
|
||||
const modelName = repoID.split('/').slice(1).join('/')
|
||||
const filename = `${modelName}-${quantization.toLowerCase()}.gguf`
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
const modelPath = await this.getQuantizedModelPath(repoID, quantization)
|
||||
const modelConfigPath = await joinPath([modelDirPath, 'model.json'])
|
||||
if (!(await fs.existsSync(modelPath))) {
|
||||
throw new Error('Model not found')
|
||||
}
|
||||
|
||||
const size = await executeOnMain(NODE_MODULE_PATH, 'getSize', modelPath)
|
||||
let ctx = 4096
|
||||
try {
|
||||
const config = await fs.readFileSync(
|
||||
await joinPath([modelDirPath, 'config.json']),
|
||||
'utf8'
|
||||
)
|
||||
ctx = this.getCtxLength(JSON.parse(config))
|
||||
fs.unlinkSync(await joinPath([modelDirPath, 'config.json']))
|
||||
} catch (err) {
|
||||
// ignore missing config.json
|
||||
}
|
||||
// maybe later, currently it's gonna use too much memory
|
||||
// const buffer = await fs.readFileSync(quantizedModelPath)
|
||||
// const ggufData = ggufMetadata(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength))
|
||||
|
||||
const metadata: Model = {
|
||||
object: 'model',
|
||||
version: 1,
|
||||
format: 'gguf',
|
||||
sources: [
|
||||
{
|
||||
url: `https://huggingface.co/${repoID}`, // i think this is just for download but not sure,
|
||||
filename,
|
||||
},
|
||||
],
|
||||
id: modelName,
|
||||
name: modelName,
|
||||
created: Date.now(),
|
||||
description: `Auto converted from Hugging Face model: ${repoID}`,
|
||||
settings: {
|
||||
ctx_len: ctx,
|
||||
prompt_template: '',
|
||||
llama_model_path: modelName,
|
||||
},
|
||||
parameters: {
|
||||
temperature: 0.7,
|
||||
top_p: 0.95,
|
||||
stream: true,
|
||||
max_tokens: 4096,
|
||||
// stop: [''], seems like we dont really need this..?
|
||||
frequency_penalty: 0,
|
||||
presence_penalty: 0,
|
||||
},
|
||||
metadata: {
|
||||
author: repoData.author,
|
||||
tags: repoData.tags,
|
||||
size,
|
||||
},
|
||||
engine: InferenceEngine.nitro,
|
||||
}
|
||||
|
||||
await fs.writeFileSync(modelConfigPath, JSON.stringify(metadata, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels the convert of current Hugging Face model.
|
||||
* @param repoID - The repository ID to cancel.
|
||||
* @param repoData - The repository data to cancel.
|
||||
* @returns {Promise<void>} A promise that resolves when the download has been cancelled.
|
||||
*/
|
||||
async cancelConvert(
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData
|
||||
): Promise<void> {
|
||||
this.interrupted = true
|
||||
const modelDirPath = await this.getModelDirPath(repoID)
|
||||
const files = this.getFileList(repoData)
|
||||
for (const file of files) {
|
||||
const filePath = file
|
||||
const localPath = await joinPath([modelDirPath, filePath])
|
||||
await abortDownload(localPath)
|
||||
}
|
||||
// ;(await fs.existsSync(modelDirPath)) && (await fs.rmdirSync(modelDirPath))
|
||||
|
||||
executeOnMain(NODE_MODULE_PATH, 'killProcesses')
|
||||
}
|
||||
}
|
||||
187
extensions/huggingface-extension/src/node/index.ts
Normal file
187
extensions/huggingface-extension/src/node/index.ts
Normal file
@ -0,0 +1,187 @@
|
||||
import { PythonShell } from 'python-shell'
|
||||
import { spawn, ChildProcess } from 'child_process'
|
||||
import { resolve as presolve, join as pjoin } from 'path'
|
||||
import type { Quantization } from '@janhq/core'
|
||||
import { log } from '@janhq/core/node'
|
||||
import { statSync } from 'fs'
|
||||
export { renameSync } from 'fs'
|
||||
|
||||
let pythonShell: PythonShell | undefined = undefined
|
||||
let quantizeProcess: ChildProcess | undefined = undefined
|
||||
|
||||
export const getSize = (path: string): number => statSync(path).size
|
||||
|
||||
export const killProcesses = () => {
|
||||
if (pythonShell) {
|
||||
pythonShell.kill()
|
||||
pythonShell = undefined
|
||||
}
|
||||
if (quantizeProcess) {
|
||||
quantizeProcess.kill()
|
||||
quantizeProcess = undefined
|
||||
}
|
||||
}
|
||||
|
||||
export const getQuantizeExecutable = (): string => {
|
||||
let binaryFolder = pjoin(__dirname, '..', 'bin') // Current directory by default
|
||||
let binaryName = 'quantize'
|
||||
/**
|
||||
* The binary folder is different for each platform.
|
||||
*/
|
||||
if (process.platform === 'win32') {
|
||||
binaryFolder = pjoin(binaryFolder, 'win')
|
||||
binaryName = 'quantize.exe'
|
||||
} else if (process.platform === 'darwin') {
|
||||
/**
|
||||
* For MacOS: mac-arm64 (Silicon), mac-x64 (InteL)
|
||||
*/
|
||||
if (process.arch === 'arm64') {
|
||||
binaryFolder = pjoin(binaryFolder, 'mac-arm64')
|
||||
} else {
|
||||
binaryFolder = pjoin(binaryFolder, 'mac-x64')
|
||||
}
|
||||
} else {
|
||||
binaryFolder = pjoin(binaryFolder, 'linux-cpu')
|
||||
}
|
||||
return pjoin(binaryFolder, binaryName)
|
||||
}
|
||||
|
||||
export const installDeps = (): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const _pythonShell = new PythonShell(
|
||||
presolve(__dirname, '..', 'scripts', 'install_deps.py')
|
||||
)
|
||||
_pythonShell.on('message', (message) => {
|
||||
log(`[Install Deps]::Debug: ${message}`)
|
||||
})
|
||||
_pythonShell.on('stderr', (stderr) => {
|
||||
log(`[Install Deps]::Error: ${stderr}`)
|
||||
})
|
||||
_pythonShell.on('error', (err) => {
|
||||
pythonShell = undefined
|
||||
log(`[Install Deps]::Error: ${err}`)
|
||||
reject(err)
|
||||
})
|
||||
_pythonShell.on('close', () => {
|
||||
const exitCode = _pythonShell.exitCode
|
||||
pythonShell = undefined
|
||||
log(
|
||||
`[Install Deps]::Debug: Deps installation exited with code: ${exitCode}`
|
||||
)
|
||||
exitCode === 0 ? resolve() : reject(exitCode)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export const convertHf = async (
|
||||
modelDirPath: string,
|
||||
outPath: string
|
||||
): Promise<void> => {
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
const _pythonShell = new PythonShell(
|
||||
presolve(__dirname, '..', 'scripts', 'convert-hf-to-gguf.py'),
|
||||
{
|
||||
args: [modelDirPath, '--outfile', outPath],
|
||||
}
|
||||
)
|
||||
pythonShell = _pythonShell
|
||||
_pythonShell.on('message', (message) => {
|
||||
log(`[Conversion]::Debug: ${message}`)
|
||||
})
|
||||
_pythonShell.on('stderr', (stderr) => {
|
||||
log(`[Conversion]::Error: ${stderr}`)
|
||||
})
|
||||
_pythonShell.on('error', (err) => {
|
||||
pythonShell = undefined
|
||||
log(`[Conversion]::Error: ${err}`)
|
||||
reject(err)
|
||||
})
|
||||
_pythonShell.on('close', () => {
|
||||
const exitCode = _pythonShell.exitCode
|
||||
pythonShell = undefined
|
||||
if (exitCode !== 0) {
|
||||
log(`[Conversion]::Debug: Conversion exited with code: ${exitCode}`)
|
||||
reject(exitCode)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export const convert = async (
|
||||
modelDirPath: string,
|
||||
outPath: string,
|
||||
{ ctx, bpe }: { ctx?: number; bpe?: boolean }
|
||||
): Promise<void> => {
|
||||
const args = [modelDirPath, '--outfile', outPath]
|
||||
if (ctx) {
|
||||
args.push('--ctx')
|
||||
args.push(ctx.toString())
|
||||
}
|
||||
if (bpe) {
|
||||
args.push('--vocab-type')
|
||||
args.push('bpe')
|
||||
}
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
const _pythonShell = new PythonShell(
|
||||
presolve(__dirname, '..', 'scripts', 'convert.py'),
|
||||
{
|
||||
args,
|
||||
}
|
||||
)
|
||||
_pythonShell.on('message', (message) => {
|
||||
log(`[Conversion]::Debug: ${message}`)
|
||||
})
|
||||
_pythonShell.on('stderr', (stderr) => {
|
||||
log(`[Conversion]::Error: ${stderr}`)
|
||||
})
|
||||
_pythonShell.on('error', (err) => {
|
||||
pythonShell = undefined
|
||||
log(`[Conversion]::Error: ${err}`)
|
||||
reject(err)
|
||||
})
|
||||
_pythonShell.on('close', () => {
|
||||
const exitCode = _pythonShell.exitCode
|
||||
pythonShell = undefined
|
||||
if (exitCode !== 0) {
|
||||
log(`[Conversion]::Debug: Conversion exited with code: ${exitCode}`)
|
||||
reject(exitCode)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export const quantize = async (
|
||||
modelPath: string,
|
||||
outPath: string,
|
||||
quantization: Quantization
|
||||
): Promise<void> => {
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
const quantizeExecutable = getQuantizeExecutable()
|
||||
const _quantizeProcess = spawn(quantizeExecutable, [
|
||||
modelPath,
|
||||
outPath,
|
||||
quantization,
|
||||
])
|
||||
quantizeProcess = _quantizeProcess
|
||||
|
||||
_quantizeProcess.stdout?.on('data', (data) => {
|
||||
log(`[Quantization]::Debug: ${data}`)
|
||||
})
|
||||
_quantizeProcess.stderr?.on('data', (data) => {
|
||||
log(`[Quantization]::Error: ${data}`)
|
||||
})
|
||||
|
||||
_quantizeProcess.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
log(`[Quantization]::Debug: Quantization exited with code: ${code}`)
|
||||
reject(code)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
20
extensions/huggingface-extension/tsconfig.json
Normal file
20
extensions/huggingface-extension/tsconfig.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"moduleResolution": "node",
|
||||
"target": "es2020",
|
||||
"module": "ES2020",
|
||||
"lib": ["es2015", "es2016", "es2017", "dom"],
|
||||
"strict": true,
|
||||
"sourceMap": true,
|
||||
"declaration": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"experimentalDecorators": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"declarationDir": "dist/types",
|
||||
"outDir": "dist",
|
||||
"importHelpers": true,
|
||||
"typeRoots": ["node_modules/@types"],
|
||||
"resolveJsonModule": true,
|
||||
},
|
||||
"include": ["src"],
|
||||
}
|
||||
44
web/helpers/atoms/HFConverter.atom.ts
Normal file
44
web/helpers/atoms/HFConverter.atom.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import { HuggingFaceRepoData } from '@janhq/core'
|
||||
import { atom } from 'jotai'
|
||||
|
||||
export const repoIDAtom = atom<string | null>(null)
|
||||
export const loadingAtom = atom<boolean>(false)
|
||||
export const fetchErrorAtom = atom<Error | null>(null)
|
||||
export const conversionStatusAtom = atom<
|
||||
| 'downloading'
|
||||
| 'converting'
|
||||
| 'quantizing'
|
||||
| 'done'
|
||||
| 'stopping'
|
||||
| 'generating'
|
||||
| null
|
||||
>(null)
|
||||
export const conversionErrorAtom = atom<Error | null>(null)
|
||||
const _repoDataAtom = atom<HuggingFaceRepoData | null>(null)
|
||||
const _unsupportedAtom = atom<boolean>(false)
|
||||
|
||||
export const resetAtom = atom(null, (_get, set) => {
|
||||
set(repoIDAtom, null)
|
||||
set(loadingAtom, false)
|
||||
set(fetchErrorAtom, null)
|
||||
set(conversionStatusAtom, null)
|
||||
set(conversionErrorAtom, null)
|
||||
set(_repoDataAtom, null)
|
||||
set(_unsupportedAtom, false)
|
||||
})
|
||||
|
||||
export const repoDataAtom = atom(
|
||||
(get) => get(_repoDataAtom),
|
||||
(_get, set, repoData: HuggingFaceRepoData) => {
|
||||
set(_repoDataAtom, repoData)
|
||||
if (
|
||||
!repoData.tags.includes('transformers') ||
|
||||
(!repoData.tags.includes('pytorch') &&
|
||||
!repoData.tags.includes('safetensors'))
|
||||
) {
|
||||
set(_unsupportedAtom, true)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
export const unsupportedAtom = atom((get) => get(_unsupportedAtom))
|
||||
81
web/hooks/useConvertHuggingFaceModel.ts
Normal file
81
web/hooks/useConvertHuggingFaceModel.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import { useContext } from 'react'
|
||||
|
||||
import {
|
||||
ExtensionTypeEnum,
|
||||
HuggingFaceExtension,
|
||||
HuggingFaceRepoData,
|
||||
Quantization,
|
||||
} from '@janhq/core'
|
||||
|
||||
import { useSetAtom } from 'jotai'
|
||||
|
||||
import { FeatureToggleContext } from '@/context/FeatureToggle'
|
||||
|
||||
import { extensionManager } from '@/extension/ExtensionManager'
|
||||
import {
|
||||
conversionStatusAtom,
|
||||
conversionErrorAtom,
|
||||
} from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const useConvertHuggingFaceModel = () => {
|
||||
const { ignoreSSL, proxy } = useContext(FeatureToggleContext)
|
||||
const setConversionStatus = useSetAtom(conversionStatusAtom)
|
||||
const setConversionError = useSetAtom(conversionErrorAtom)
|
||||
|
||||
const convertHuggingFaceModel = async (
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData,
|
||||
quantization: Quantization
|
||||
) => {
|
||||
const extension = await extensionManager.get<HuggingFaceExtension>(
|
||||
ExtensionTypeEnum.HuggingFace
|
||||
)
|
||||
try {
|
||||
if (extension) {
|
||||
extension.interrupted = false
|
||||
}
|
||||
setConversionStatus('downloading')
|
||||
await extension?.downloadModelFiles(repoID, repoData, {
|
||||
ignoreSSL,
|
||||
proxy,
|
||||
})
|
||||
if (extension?.interrupted) return
|
||||
setConversionStatus('converting')
|
||||
await extension?.convert(repoID)
|
||||
if (extension?.interrupted) return
|
||||
setConversionStatus('quantizing')
|
||||
await extension?.quantize(repoID, quantization)
|
||||
if (extension?.interrupted) return
|
||||
setConversionStatus('generating')
|
||||
await extension?.generateMetadata(repoID, repoData, quantization)
|
||||
setConversionStatus('done')
|
||||
} catch (err) {
|
||||
if (extension?.interrupted) return
|
||||
extension?.cancelConvert(repoID, repoData)
|
||||
if (typeof err === 'number') {
|
||||
setConversionError(new Error(`exit code: ${err}`))
|
||||
} else {
|
||||
setConversionError(err as Error)
|
||||
}
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
const cancelConvertHuggingFaceModel = async (
|
||||
repoID: string,
|
||||
repoData: HuggingFaceRepoData
|
||||
) => {
|
||||
const extension = await extensionManager.get<HuggingFaceExtension>(
|
||||
ExtensionTypeEnum.HuggingFace
|
||||
)
|
||||
|
||||
setConversionStatus('stopping')
|
||||
await extension?.cancelConvert(repoID, repoData)
|
||||
setConversionStatus(null)
|
||||
}
|
||||
|
||||
return {
|
||||
convertHuggingFaceModel,
|
||||
cancelConvertHuggingFaceModel,
|
||||
}
|
||||
}
|
||||
29
web/hooks/useGetHFRepoData.ts
Normal file
29
web/hooks/useGetHFRepoData.ts
Normal file
@ -0,0 +1,29 @@
|
||||
import { useAtomValue, useSetAtom } from 'jotai'
|
||||
|
||||
import {
|
||||
repoDataAtom,
|
||||
repoIDAtom,
|
||||
loadingAtom,
|
||||
fetchErrorAtom,
|
||||
} from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const useGetHFRepoData = () => {
|
||||
const repoID = useAtomValue(repoIDAtom)
|
||||
const setRepoData = useSetAtom(repoDataAtom)
|
||||
const setLoading = useSetAtom(loadingAtom)
|
||||
const setFetchError = useSetAtom(fetchErrorAtom)
|
||||
|
||||
const getRepoData = async () => {
|
||||
setLoading(true)
|
||||
try {
|
||||
const res = await fetch(`https://huggingface.co/api/models/${repoID}`)
|
||||
const data = await res.json()
|
||||
setRepoData(data)
|
||||
} catch (err) {
|
||||
setFetchError(err as Error)
|
||||
}
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
return getRepoData
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
import { useAtomValue } from 'jotai'
|
||||
|
||||
import {
|
||||
conversionErrorAtom,
|
||||
conversionStatusAtom,
|
||||
repoDataAtom,
|
||||
} from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const HuggingFaceConvertingErrorModal = () => {
|
||||
// This component only loads when repoData is not null
|
||||
const repoData = useAtomValue(repoDataAtom)!
|
||||
// This component only loads when conversionStatus is not null
|
||||
const conversionStatus = useAtomValue(conversionStatusAtom)!
|
||||
// This component only loads when conversionError is not null
|
||||
const conversionError = useAtomValue(conversionErrorAtom)!
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="text-2xl font-bold">Hugging Face Converter</p>
|
||||
</div>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="text-center">
|
||||
An error occured while {conversionStatus} model {repoData.id}.
|
||||
</p>
|
||||
<p>Please close this modal and try again.</p>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
|
||||
import { Button } from '@janhq/uikit'
|
||||
import { useAtomValue } from 'jotai'
|
||||
|
||||
import { useConvertHuggingFaceModel } from '@/hooks/useConvertHuggingFaceModel'
|
||||
|
||||
import {
|
||||
conversionStatusAtom,
|
||||
repoDataAtom,
|
||||
} from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const HuggingFaceConvertingModal = () => {
|
||||
// This component only loads when repoData is not null
|
||||
const repoData = useAtomValue(repoDataAtom)!
|
||||
// This component only loads when conversionStatus is not null
|
||||
const conversionStatus = useAtomValue(conversionStatusAtom)!
|
||||
const [status, setStatus] = useState('')
|
||||
const { cancelConvertHuggingFaceModel } = useConvertHuggingFaceModel()
|
||||
|
||||
useEffect(() => {
|
||||
switch (conversionStatus) {
|
||||
case 'downloading':
|
||||
setStatus('Downloading files...')
|
||||
break
|
||||
case 'converting':
|
||||
setStatus('Converting...')
|
||||
break
|
||||
case 'quantizing':
|
||||
setStatus('Quantizing...')
|
||||
break
|
||||
case 'stopping':
|
||||
setStatus('Stopping...')
|
||||
break
|
||||
case 'generating':
|
||||
setStatus('Generating metadata...')
|
||||
break
|
||||
}
|
||||
}, [conversionStatus])
|
||||
|
||||
const onStopClick = () => {
|
||||
cancelConvertHuggingFaceModel(repoData.id, repoData)
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="text-2xl font-bold">Hugging Face Converter</p>
|
||||
</div>
|
||||
{conversionStatus === 'done' ? (
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p>Done!</p>
|
||||
<p>Now you can use the model on Jan as usual. Have fun!</p>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p>{status}</p>
|
||||
</div>
|
||||
<Button
|
||||
onClick={onStopClick}
|
||||
className="w-full"
|
||||
loading={conversionStatus === 'stopping'}
|
||||
disabled={conversionStatus === 'stopping'}
|
||||
themes="danger"
|
||||
>
|
||||
{conversionStatus === 'stopping' ? 'Stopping...' : 'Stop'}
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
70
web/screens/ExploreModels/HuggingFaceModal/index.tsx
Normal file
70
web/screens/ExploreModels/HuggingFaceModal/index.tsx
Normal file
@ -0,0 +1,70 @@
|
||||
import { CommandModal, Modal, ModalContent } from '@janhq/uikit'
|
||||
import { useAtomValue, useSetAtom } from 'jotai'
|
||||
|
||||
import { HuggingFaceConvertingErrorModal } from '../HuggingFaceConvertingErrorModal'
|
||||
import { HuggingFaceConvertingModal } from '../HuggingFaceConvertingModal'
|
||||
import { HuggingFaceRepoDataLoadedModal } from '../HuggingFaceRepoDataLoadedModal'
|
||||
import { HuggingFaceSearchErrorModal } from '../HuggingFaceSearchErrorModal'
|
||||
import { HuggingFaceSearchModal } from '../HuggingFaceSearchModal'
|
||||
|
||||
import {
|
||||
repoDataAtom,
|
||||
fetchErrorAtom,
|
||||
resetAtom,
|
||||
conversionStatusAtom,
|
||||
conversionErrorAtom,
|
||||
} from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
const HuggingFaceModal = ({
|
||||
...props
|
||||
}: Omit<Parameters<typeof CommandModal>[0], 'children'>) => {
|
||||
const repoData = useAtomValue(repoDataAtom)
|
||||
const fetchError = useAtomValue(fetchErrorAtom)
|
||||
const conversionStatus = useAtomValue(conversionStatusAtom)
|
||||
const conversionError = useAtomValue(conversionErrorAtom)
|
||||
const setReset = useSetAtom(resetAtom)
|
||||
|
||||
return (
|
||||
<Modal
|
||||
{...props}
|
||||
onOpenChange={(open) => {
|
||||
if (open === false) {
|
||||
if (
|
||||
!repoData ||
|
||||
['done', 'stopping'].includes(conversionStatus ?? '') ||
|
||||
conversionError
|
||||
) {
|
||||
setReset()
|
||||
}
|
||||
}
|
||||
if (props.onOpenChange) {
|
||||
props.onOpenChange(open)
|
||||
}
|
||||
}}
|
||||
>
|
||||
<ModalContent>
|
||||
<div className="px-2 py-3">
|
||||
<div className="flex w-full flex-col items-center justify-center gap-4 p-4">
|
||||
{repoData ? (
|
||||
conversionStatus ? (
|
||||
conversionError ? (
|
||||
<HuggingFaceConvertingErrorModal />
|
||||
) : (
|
||||
<HuggingFaceConvertingModal />
|
||||
)
|
||||
) : (
|
||||
<HuggingFaceRepoDataLoadedModal />
|
||||
)
|
||||
) : fetchError ? (
|
||||
<HuggingFaceSearchErrorModal />
|
||||
) : (
|
||||
<HuggingFaceSearchModal />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
export { HuggingFaceModal }
|
||||
@ -0,0 +1,100 @@
|
||||
import { useState } from 'react'
|
||||
|
||||
import { Quantization } from '@janhq/core'
|
||||
import {
|
||||
Button,
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectGroup,
|
||||
SelectItem,
|
||||
SelectPortal,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@janhq/uikit'
|
||||
import { useAtomValue } from 'jotai'
|
||||
|
||||
import { twMerge } from 'tailwind-merge'
|
||||
|
||||
import { useConvertHuggingFaceModel } from '@/hooks/useConvertHuggingFaceModel'
|
||||
|
||||
import {
|
||||
loadingAtom,
|
||||
repoDataAtom,
|
||||
unsupportedAtom,
|
||||
} from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const HuggingFaceRepoDataLoadedModal = () => {
|
||||
const loading = useAtomValue(loadingAtom)
|
||||
// This component only loads when repoData is not null
|
||||
const repoData = useAtomValue(repoDataAtom)!
|
||||
const unsupported = useAtomValue(unsupportedAtom)
|
||||
const [quantization, setQuantization] = useState<Quantization>(
|
||||
Quantization.Q4_K_M
|
||||
)
|
||||
const { convertHuggingFaceModel } = useConvertHuggingFaceModel()
|
||||
|
||||
const onValueSelected = (value: Quantization) => {
|
||||
setQuantization(value)
|
||||
}
|
||||
const onConvertClick = () => {
|
||||
convertHuggingFaceModel(repoData.id, repoData, quantization)
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="text-2xl font-bold">Hugging Face Converter</p>
|
||||
<p className="text-gray-500">Found the repository!</p>
|
||||
</div>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="font-bold">{repoData.id}</p>
|
||||
<p>
|
||||
{unsupported
|
||||
? '❌ This model is not supported!'
|
||||
: '✅ This model is supported!'}
|
||||
</p>
|
||||
{repoData.tags.includes('gguf') ? (
|
||||
<p>...But you can import it manually!</p>
|
||||
) : null}
|
||||
</div>
|
||||
<Select
|
||||
value={quantization}
|
||||
onValueChange={onValueSelected}
|
||||
disabled={unsupported}
|
||||
>
|
||||
<SelectTrigger className="relative w-full">
|
||||
<SelectValue placeholder="Quantization">
|
||||
<span className={twMerge('relative z-20')}>{quantization}</span>
|
||||
</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectPortal>
|
||||
<SelectContent className="right-2 block w-full min-w-[450px] pr-0">
|
||||
<div className="border-b border-border" />
|
||||
<SelectGroup>
|
||||
{Object.values(Quantization).map((x, i) => (
|
||||
<SelectItem
|
||||
key={i}
|
||||
value={x}
|
||||
className={twMerge(x === quantization && 'bg-secondary')}
|
||||
>
|
||||
<div className="flex w-full justify-between">
|
||||
<span className="line-clamp-1 block">{x}</span>
|
||||
</div>
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</SelectPortal>
|
||||
</Select>
|
||||
<Button
|
||||
onClick={onConvertClick}
|
||||
className="w-full"
|
||||
loading={loading}
|
||||
disabled={unsupported}
|
||||
themes={loading ? 'ghost' : 'primary'}
|
||||
>
|
||||
{loading ? '' : 'Convert'}
|
||||
</Button>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
import { Button } from '@janhq/uikit'
|
||||
import { useAtomValue } from 'jotai'
|
||||
|
||||
import { useGetHFRepoData } from '@/hooks/useGetHFRepoData'
|
||||
|
||||
import { fetchErrorAtom, loadingAtom } from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const HuggingFaceSearchErrorModal = () => {
|
||||
// This component only loads when fetchError is not null
|
||||
const fetchError = useAtomValue(fetchErrorAtom)!
|
||||
const loading = useAtomValue(loadingAtom)
|
||||
|
||||
const getRepoData = useGetHFRepoData()
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="text-2xl font-bold">Error!</p>
|
||||
<p className="text-gray-500">Fetch error</p>
|
||||
</div>
|
||||
<p>{fetchError.message}</p>
|
||||
<Button
|
||||
onClick={getRepoData}
|
||||
className="w-full"
|
||||
loading={loading}
|
||||
themes={loading ? 'ghost' : 'danger'}
|
||||
>
|
||||
{loading ? '' : 'Try Again'}
|
||||
</Button>
|
||||
</>
|
||||
)
|
||||
}
|
||||
45
web/screens/ExploreModels/HuggingFaceSearchModal/index.tsx
Normal file
45
web/screens/ExploreModels/HuggingFaceSearchModal/index.tsx
Normal file
@ -0,0 +1,45 @@
|
||||
import { Button, Input } from '@janhq/uikit'
|
||||
import { useSetAtom, useAtomValue } from 'jotai'
|
||||
|
||||
import { useGetHFRepoData } from '@/hooks/useGetHFRepoData'
|
||||
|
||||
import { repoIDAtom, loadingAtom } from '@/helpers/atoms/HFConverter.atom'
|
||||
|
||||
export const HuggingFaceSearchModal = () => {
|
||||
const setRepoID = useSetAtom(repoIDAtom)
|
||||
const loading = useAtomValue(loadingAtom)
|
||||
|
||||
const getRepoData = useGetHFRepoData()
|
||||
|
||||
const onKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault()
|
||||
getRepoData()
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col items-center justify-center gap-1">
|
||||
<p className="text-2xl font-bold">Hugging Face Convertor</p>
|
||||
<p className="text-gray-500">Type the repository id below</p>
|
||||
</div>
|
||||
<Input
|
||||
placeholder="e.g. username/repo-name"
|
||||
className="bg-white dark:bg-background"
|
||||
onChange={(e) => {
|
||||
setRepoID(e.target.value)
|
||||
}}
|
||||
onKeyDown={onKeyDown}
|
||||
/>
|
||||
<Button
|
||||
onClick={getRepoData}
|
||||
className="w-full"
|
||||
loading={loading}
|
||||
themes={loading ? 'ghost' : 'primary'}
|
||||
>
|
||||
{loading ? '' : 'OK'}
|
||||
</Button>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@ -16,6 +16,7 @@ import { useAtomValue } from 'jotai'
|
||||
import { SearchIcon } from 'lucide-react'
|
||||
|
||||
import ExploreModelList from './ExploreModelList'
|
||||
import { HuggingFaceModal } from './HuggingFaceModal'
|
||||
|
||||
import {
|
||||
configuredModelsAtom,
|
||||
@ -28,6 +29,7 @@ const ExploreModelsScreen = () => {
|
||||
const [searchValue, setsearchValue] = useState('')
|
||||
const [sortSelected, setSortSelected] = useState('All Models')
|
||||
const sortMenu = ['All Models', 'Recommended', 'Downloaded']
|
||||
const [showHuggingFaceModal, setShowHuggingFaceModal] = useState(false)
|
||||
|
||||
const filteredModels = configuredModels.filter((x) => {
|
||||
if (sortSelected === 'Downloaded') {
|
||||
@ -49,6 +51,10 @@ const ExploreModelsScreen = () => {
|
||||
openExternalUrl('https://jan.ai/guides/using-models/import-manually/')
|
||||
}, [])
|
||||
|
||||
const onHuggingFaceConverterClick = () => {
|
||||
setShowHuggingFaceModal(true)
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex h-full w-full overflow-y-auto bg-background"
|
||||
@ -56,6 +62,10 @@ const ExploreModelsScreen = () => {
|
||||
>
|
||||
<div className="h-full w-full p-4">
|
||||
<div className="h-full">
|
||||
<HuggingFaceModal
|
||||
open={showHuggingFaceModal}
|
||||
onOpenChange={setShowHuggingFaceModal}
|
||||
/>
|
||||
<ScrollArea>
|
||||
<div className="relative">
|
||||
<img
|
||||
@ -85,6 +95,15 @@ const ExploreModelsScreen = () => {
|
||||
How to manually import models
|
||||
</p>
|
||||
</div>
|
||||
<div className="text-center">
|
||||
<p className="text-white">or</p>
|
||||
<p
|
||||
onClick={onHuggingFaceConverterClick}
|
||||
className="cursor-pointer font-semibold text-white underline"
|
||||
>
|
||||
Convert from Hugging Face
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="mx-auto w-4/5 py-6">
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user