From f44f291bd870462b13a54ed0b3d99dc162ec91d6 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 17 Oct 2024 15:21:00 +0700 Subject: [PATCH] chore: download progress finished should reload model list --- extensions/model-extension/src/cortex.ts | 52 +++++- extensions/model-extension/src/index.ts | 161 +----------------- extensions/model-extension/src/model-json.ts | 132 ++++++++++++++ web/containers/Providers/EventListener.tsx | 2 + .../Settings/MyModels/MyModelList/index.tsx | 2 +- 5 files changed, 194 insertions(+), 155 deletions(-) create mode 100644 extensions/model-extension/src/model-json.ts diff --git a/extensions/model-extension/src/cortex.ts b/extensions/model-extension/src/cortex.ts index 685bf3b9f..4945e4756 100644 --- a/extensions/model-extension/src/cortex.ts +++ b/extensions/model-extension/src/cortex.ts @@ -1,9 +1,9 @@ import PQueue from 'p-queue' import ky from 'ky' import { - DownloadEvent, events, Model, + ModelEvent, ModelRuntimeParams, ModelSettingParams, } from '@janhq/core' @@ -39,6 +39,11 @@ export class CortexAPI implements ICortexAPI { this.subscribeToEvents() } + /** + * Fetches a model detail from cortex.cpp + * @param model + * @returns + */ getModel(model: string): Promise { return this.queue.add(() => ky @@ -48,6 +53,11 @@ export class CortexAPI implements ICortexAPI { ) } + /** + * Fetches models list from cortex.cpp + * @param model + * @returns + */ getModels(): Promise { return this.queue .add(() => ky.get(`${API_URL}/models`).json()) @@ -56,6 +66,11 @@ export class CortexAPI implements ICortexAPI { ) } + /** + * Pulls a model from HuggingFace via cortex.cpp + * @param model + * @returns + */ pullModel(model: string): Promise { return this.queue.add(() => ky @@ -68,6 +83,11 @@ export class CortexAPI implements ICortexAPI { ) } + /** + * Imports a model from a local path via cortex.cpp + * @param model + * @returns + */ importModel(model: string, modelPath: string): Promise { return this.queue.add(() => ky @@ -78,12 +98,22 @@ export class CortexAPI implements ICortexAPI { ) } + /** + * Deletes a model from cortex.cpp + * @param model + * @returns + */ deleteModel(model: string): Promise { return this.queue.add(() => ky.delete(`${API_URL}/models/${model}`).json().then() ) } + /** + * Update a model in cortex.cpp + * @param model + * @returns + */ updateModel(model: object): Promise { return this.queue.add(() => ky @@ -92,6 +122,12 @@ export class CortexAPI implements ICortexAPI { .then() ) } + + /** + * Cancel model pull in cortex.cpp + * @param model + * @returns + */ cancelModelPull(model: string): Promise { return this.queue.add(() => ky @@ -101,6 +137,10 @@ export class CortexAPI implements ICortexAPI { ) } + /** + * Do health check on cortex.cpp + * @returns + */ healthz(): Promise { return ky .get(`${API_URL}/healthz`, { @@ -112,6 +152,9 @@ export class CortexAPI implements ICortexAPI { .then(() => {}) } + /** + * Subscribe to cortex.cpp websocket events + */ subscribeToEvents() { this.queue.add( () => @@ -140,12 +183,19 @@ export class CortexAPI implements ICortexAPI { total: total, }, }) + // Update models list from Hub + events.emit(ModelEvent.OnModelsUpdate, {}) }) resolve() }) ) } + /** + * TRansform model to the expected format (e.g. parameters, settings, metadata) + * @param model + * @returns + */ private transformModel(model: any) { model.parameters = setParameters(model) model.settings = setParameters(model) diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index b879e0bb9..c154c3754 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -2,21 +2,14 @@ import { ModelExtension, Model, InferenceEngine, - fs, joinPath, dirName, } from '@janhq/core' import { CortexAPI } from './cortex' +import { scanModelsFolder } from './model-json' declare const SETTINGS: Array -/** - * TODO: Set env for HF access token? or via API request? - */ -enum Settings { - huggingFaceAccessToken = 'hugging-face-access-token', -} - /** * Extension enum */ @@ -28,7 +21,6 @@ enum ExtensionEnum { * A extension for models */ export default class JanModelExtension extends ModelExtension { - private static readonly _homeDir = 'file://models' cortexAPI: CortexAPI = new CortexAPI() /** @@ -59,7 +51,7 @@ export default class JanModelExtension extends ModelExtension { /** * Sending POST to /models/pull/{id} endpoint to pull the model */ - return this.cortexAPI?.pullModel(model) + return this.cortexAPI.pullModel(model) } /** @@ -72,7 +64,7 @@ export default class JanModelExtension extends ModelExtension { /** * Sending DELETE to /models/pull/{id} endpoint to cancel a model pull */ - this.cortexAPI?.cancelModelPull(model) + this.cortexAPI.cancelModelPull(model) } /** @@ -81,7 +73,7 @@ export default class JanModelExtension extends ModelExtension { * @returns A Promise that resolves when the model is deleted. */ async deleteModel(model: string): Promise { - return this.cortexAPI?.deleteModel(model) + return this.cortexAPI.deleteModel(model) } /** @@ -99,7 +91,7 @@ export default class JanModelExtension extends ModelExtension { // Updated from an older version than 0.5.5 // Scan through the models folder and import them (Legacy flow) // Return models immediately - return this.scanModelsFolder().then((models) => { + return scanModelsFolder().then((models) => { return models ?? [] }) } @@ -123,7 +115,7 @@ export default class JanModelExtension extends ModelExtension { (e) => e.engine === InferenceEngine.nitro ) - await this.cortexAPI?.getModels().then((models) => { + await this.cortexAPI.getModels().then((models) => { const existingIds = models.map((e) => e.id) toImportModels = toImportModels.filter( (e: Model) => !existingIds.includes(e.id) @@ -161,7 +153,7 @@ export default class JanModelExtension extends ModelExtension { * just return models from cortex.cpp */ return ( - this.cortexAPI?.getModels().then((models) => { + this.cortexAPI.getModels().then((models) => { return models }) ?? Promise.resolve([]) ) @@ -183,143 +175,6 @@ export default class JanModelExtension extends ModelExtension { * @param optionType */ async importModel(model: string, modelPath: string): Promise { - return this.cortexAPI?.importModel(model, modelPath) + return this.cortexAPI.importModel(model, modelPath) } - - //// LEGACY MODEL FOLDER //// - /** - * Scan through models folder and return downloaded models - * @returns - */ - private async scanModelsFolder(): Promise { - try { - if (!(await fs.existsSync(JanModelExtension._homeDir))) { - console.debug('Model folder not found') - return [] - } - - const files: string[] = await fs.readdirSync(JanModelExtension._homeDir) - - const allDirectories: string[] = [] - - for (const modelFolder of files) { - const fullModelFolderPath = await joinPath([ - JanModelExtension._homeDir, - modelFolder, - ]) - if (!(await fs.fileStat(fullModelFolderPath)).isDirectory) continue - allDirectories.push(modelFolder) - } - - const readJsonPromises = allDirectories.map(async (dirName) => { - // filter out directories that don't match the selector - // read model.json - const folderFullPath = await joinPath([ - JanModelExtension._homeDir, - dirName, - ]) - - const jsonPath = await this.getModelJsonPath(folderFullPath) - - if (await fs.existsSync(jsonPath)) { - // if we have the model.json file, read it - let model = await fs.readFileSync(jsonPath, 'utf-8') - - model = typeof model === 'object' ? model : JSON.parse(model) - - // This to ensure backward compatibility with `model.json` with `source_url` - if (model['source_url'] != null) { - model['sources'] = [ - { - filename: model.id, - url: model['source_url'], - }, - ] - } - model.file_path = jsonPath - model.file_name = 'model.json' - - // Check model file exist - // model binaries (sources) are absolute path & exist (symlinked) - const existFiles = await Promise.all( - model.sources.map( - (source) => - // Supposed to be a local file url - !source.url.startsWith(`http://`) && - !source.url.startsWith(`https://`) - ) - ) - if (existFiles.every((exist) => exist)) return true - - const result = await fs - .readdirSync(await joinPath([JanModelExtension._homeDir, dirName])) - .then((files: string[]) => { - // Model binary exists in the directory - // Model binary name can match model ID or be a .gguf file and not be an incompleted model file - return ( - files.includes(dirName) || // Legacy model GGUF without extension - files.filter((file) => { - return ( - file.toLowerCase().endsWith('.gguf') || // GGUF - file.toLowerCase().endsWith('.engine') // Tensort-LLM - ) - })?.length > 0 // TODO: find better way (can use basename to check the file name with source url) - ) - }) - - if (result) return model - else return undefined - } - }) - const results = await Promise.allSettled(readJsonPromises) - const modelData = results - .map((result) => { - if (result.status === 'fulfilled' && result.value) { - try { - const model = - typeof result.value === 'object' - ? result.value - : JSON.parse(result.value) - return model as Model - } catch { - console.debug(`Unable to parse model metadata: ${result.value}`) - } - } - return undefined - }) - .filter((e) => !!e) - - return modelData - } catch (err) { - console.error(err) - return [] - } - } - - /** - * Retrieve the model.json path from a folder - * @param folderFullPath - * @returns - */ - private async getModelJsonPath( - folderFullPath: string - ): Promise { - // try to find model.json recursively inside each folder - if (!(await fs.existsSync(folderFullPath))) return undefined - const files: string[] = await fs.readdirSync(folderFullPath) - if (files.length === 0) return undefined - if (files.includes('model.json')) { - return joinPath([folderFullPath, 'model.json']) - } - // continue recursive - for (const file of files) { - const path = await joinPath([folderFullPath, file]) - const fileStats = await fs.fileStat(path) - if (fileStats.isDirectory) { - const result = await this.getModelJsonPath(path) - if (result) return result - } - } - } - //// END LEGACY MODEL FOLDER //// } diff --git a/extensions/model-extension/src/model-json.ts b/extensions/model-extension/src/model-json.ts new file mode 100644 index 000000000..af6f95b36 --- /dev/null +++ b/extensions/model-extension/src/model-json.ts @@ -0,0 +1,132 @@ +import { Model, fs, joinPath } from '@janhq/core' +//// LEGACY MODEL FOLDER //// +/** + * Scan through models folder and return downloaded models + * @returns + */ +export const scanModelsFolder = async (): Promise => { + const _homeDir = 'file://models' + try { + if (!(await fs.existsSync(_homeDir))) { + console.debug('Model folder not found') + return [] + } + + const files: string[] = await fs.readdirSync(_homeDir) + + const allDirectories: string[] = [] + + for (const modelFolder of files) { + const fullModelFolderPath = await joinPath([_homeDir, modelFolder]) + if (!(await fs.fileStat(fullModelFolderPath)).isDirectory) continue + allDirectories.push(modelFolder) + } + + const readJsonPromises = allDirectories.map(async (dirName) => { + // filter out directories that don't match the selector + // read model.json + const folderFullPath = await joinPath([_homeDir, dirName]) + + const jsonPath = await getModelJsonPath(folderFullPath) + + if (await fs.existsSync(jsonPath)) { + // if we have the model.json file, read it + let model = await fs.readFileSync(jsonPath, 'utf-8') + + model = typeof model === 'object' ? model : JSON.parse(model) + + // This to ensure backward compatibility with `model.json` with `source_url` + if (model['source_url'] != null) { + model['sources'] = [ + { + filename: model.id, + url: model['source_url'], + }, + ] + } + model.file_path = jsonPath + model.file_name = 'model.json' + + // Check model file exist + // model binaries (sources) are absolute path & exist (symlinked) + const existFiles = await Promise.all( + model.sources.map( + (source) => + // Supposed to be a local file url + !source.url.startsWith(`http://`) && + !source.url.startsWith(`https://`) + ) + ) + if (existFiles.every((exist) => exist)) return true + + const result = await fs + .readdirSync(await joinPath([_homeDir, dirName])) + .then((files: string[]) => { + // Model binary exists in the directory + // Model binary name can match model ID or be a .gguf file and not be an incompleted model file + return ( + files.includes(dirName) || // Legacy model GGUF without extension + files.filter((file) => { + return ( + file.toLowerCase().endsWith('.gguf') || // GGUF + file.toLowerCase().endsWith('.engine') // Tensort-LLM + ) + })?.length > 0 // TODO: find better way (can use basename to check the file name with source url) + ) + }) + + if (result) return model + else return undefined + } + }) + const results = await Promise.allSettled(readJsonPromises) + const modelData = results + .map((result) => { + if (result.status === 'fulfilled' && result.value) { + try { + const model = + typeof result.value === 'object' + ? result.value + : JSON.parse(result.value) + return model as Model + } catch { + console.debug(`Unable to parse model metadata: ${result.value}`) + } + } + return undefined + }) + .filter((e) => !!e) + + return modelData + } catch (err) { + console.error(err) + return [] + } +} + +/** + * Retrieve the model.json path from a folder + * @param folderFullPath + * @returns + */ +export const getModelJsonPath = async ( + folderFullPath: string +): Promise => { + // try to find model.json recursively inside each folder + if (!(await fs.existsSync(folderFullPath))) return undefined + const files: string[] = await fs.readdirSync(folderFullPath) + if (files.length === 0) return undefined + if (files.includes('model.json')) { + return joinPath([folderFullPath, 'model.json']) + } + // continue recursive + for (const file of files) { + const path = await joinPath([folderFullPath, file]) + const fileStats = await fs.fileStat(path) + if (fileStats.isDirectory) { + const result = await getModelJsonPath(path) + if (result) return result + } + } +} +//// END LEGACY MODEL FOLDER //// diff --git a/web/containers/Providers/EventListener.tsx b/web/containers/Providers/EventListener.tsx index 608160555..1832256e2 100644 --- a/web/containers/Providers/EventListener.tsx +++ b/web/containers/Providers/EventListener.tsx @@ -111,6 +111,7 @@ const EventListenerWrapper = ({ children }: PropsWithChildren) => { events.off(DownloadEvent.onFileDownloadUpdate, onFileDownloadUpdate) events.off(DownloadEvent.onFileDownloadError, onFileDownloadError) events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) events.off(DownloadEvent.onFileUnzipSuccess, onFileUnzipSuccess) } }, [ @@ -118,6 +119,7 @@ const EventListenerWrapper = ({ children }: PropsWithChildren) => { onFileDownloadError, onFileDownloadSuccess, onFileUnzipSuccess, + onFileDownloadStopped, ]) return ( diff --git a/web/screens/Settings/MyModels/MyModelList/index.tsx b/web/screens/Settings/MyModels/MyModelList/index.tsx index 6661ed068..756520107 100644 --- a/web/screens/Settings/MyModels/MyModelList/index.tsx +++ b/web/screens/Settings/MyModels/MyModelList/index.tsx @@ -78,7 +78,7 @@ const MyModelList = ({ model }: Props) => {
- {toGibibytes(model.metadata?.size)} + {model.metadata?.size ? toGibibytes(model.metadata?.size) : '-'}