Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: download progress finished should reload model list

Browse Source
This commit is contained in:
Louis 2024-10-17 15:21:00 +07:00
parent f3aa40bc0b
commit f44f291bd8
No known key found for this signature in database
GPG Key ID: 44FA9F4D33C37DE2
5 changed files with 194 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
extensions/model-extension/src/cortex.ts
View File

@ -1,9 +1,9 @@
import PQueue from 'p-queue'
import ky from 'ky'
import {
DownloadEvent,
events,
Model,
ModelEvent,
ModelRuntimeParams,
ModelSettingParams,
} from '@janhq/core'
@ -39,6 +39,11 @@ export class CortexAPI implements ICortexAPI {
this.subscribeToEvents()
}
/**
* Fetches a model detail from cortex.cpp
* @param model
* @returns
*/
getModel(model: string): Promise<any> {
return this.queue.add(() =>
ky
@ -48,6 +53,11 @@ export class CortexAPI implements ICortexAPI {
)
}
/**
* Fetches models list from cortex.cpp
* @param model
* @returns
*/
getModels(): Promise<Model[]> {
return this.queue
.add(() => ky.get(`${API_URL}/models`).json<ModelList>())
@ -56,6 +66,11 @@ export class CortexAPI implements ICortexAPI {
)
}
/**
* Pulls a model from HuggingFace via cortex.cpp
* @param model
* @returns
*/
pullModel(model: string): Promise<void> {
return this.queue.add(() =>
ky
@ -68,6 +83,11 @@ export class CortexAPI implements ICortexAPI {
)
}
/**
* Imports a model from a local path via cortex.cpp
* @param model
* @returns
*/
importModel(model: string, modelPath: string): Promise<void> {
return this.queue.add(() =>
ky
@ -78,12 +98,22 @@ export class CortexAPI implements ICortexAPI {
)
}
/**
* Deletes a model from cortex.cpp
* @param model
* @returns
*/
deleteModel(model: string): Promise<void> {
return this.queue.add(() =>
ky.delete(`${API_URL}/models/${model}`).json().then()
)
}
/**
* Update a model in cortex.cpp
* @param model
* @returns
*/
updateModel(model: object): Promise<void> {
return this.queue.add(() =>
ky
@ -92,6 +122,12 @@ export class CortexAPI implements ICortexAPI {
.then()
)
}
/**
* Cancel model pull in cortex.cpp
* @param model
* @returns
*/
cancelModelPull(model: string): Promise<void> {
return this.queue.add(() =>
ky
@ -101,6 +137,10 @@ export class CortexAPI implements ICortexAPI {
)
}
/**
* Do health check on cortex.cpp
* @returns
*/
healthz(): Promise<void> {
return ky
.get(`${API_URL}/healthz`, {
@ -112,6 +152,9 @@ export class CortexAPI implements ICortexAPI {
.then(() => {})
}
/**
* Subscribe to cortex.cpp websocket events
*/
subscribeToEvents() {
this.queue.add(
() =>
@ -140,12 +183,19 @@ export class CortexAPI implements ICortexAPI {
total: total,
},
})
// Update models list from Hub
events.emit(ModelEvent.OnModelsUpdate, {})
})
resolve()
})
)
}
/**
* TRansform model to the expected format (e.g. parameters, settings, metadata)
* @param model
* @returns
*/
private transformModel(model: any) {
model.parameters = setParameters<ModelRuntimeParams>(model)
model.settings = setParameters<ModelSettingParams>(model)

161
extensions/model-extension/src/index.ts
View File

@ -2,21 +2,14 @@ import {
ModelExtension,
Model,
InferenceEngine,
fs,
joinPath,
dirName,
} from '@janhq/core'
import { CortexAPI } from './cortex'
import { scanModelsFolder } from './model-json'
declare const SETTINGS: Array<any>
/**
* TODO: Set env for HF access token? or via API request?
*/
enum Settings {
huggingFaceAccessToken = 'hugging-face-access-token',
}
/**
* Extension enum
*/
@ -28,7 +21,6 @@ enum ExtensionEnum {
* A extension for models
*/
export default class JanModelExtension extends ModelExtension {
private static readonly _homeDir = 'file://models'
cortexAPI: CortexAPI = new CortexAPI()
/**
@ -59,7 +51,7 @@ export default class JanModelExtension extends ModelExtension {
/**
* Sending POST to /models/pull/{id} endpoint to pull the model
*/
return this.cortexAPI?.pullModel(model)
return this.cortexAPI.pullModel(model)
}
/**
@ -72,7 +64,7 @@ export default class JanModelExtension extends ModelExtension {
/**
* Sending DELETE to /models/pull/{id} endpoint to cancel a model pull
*/
this.cortexAPI?.cancelModelPull(model)
this.cortexAPI.cancelModelPull(model)
}
/**
@ -81,7 +73,7 @@ export default class JanModelExtension extends ModelExtension {
* @returns A Promise that resolves when the model is deleted.
*/
async deleteModel(model: string): Promise<void> {
return this.cortexAPI?.deleteModel(model)
return this.cortexAPI.deleteModel(model)
}
/**
@ -99,7 +91,7 @@ export default class JanModelExtension extends ModelExtension {
// Updated from an older version than 0.5.5
// Scan through the models folder and import them (Legacy flow)
// Return models immediately
return this.scanModelsFolder().then((models) => {
return scanModelsFolder().then((models) => {
return models ?? []
})
}
@ -123,7 +115,7 @@ export default class JanModelExtension extends ModelExtension {
(e) => e.engine === InferenceEngine.nitro
)
await this.cortexAPI?.getModels().then((models) => {
await this.cortexAPI.getModels().then((models) => {
const existingIds = models.map((e) => e.id)
toImportModels = toImportModels.filter(
(e: Model) => !existingIds.includes(e.id)
@ -161,7 +153,7 @@ export default class JanModelExtension extends ModelExtension {
* just return models from cortex.cpp
*/
return (
this.cortexAPI?.getModels().then((models) => {
this.cortexAPI.getModels().then((models) => {
return models
}) ?? Promise.resolve([])
)
@ -183,143 +175,6 @@ export default class JanModelExtension extends ModelExtension {
* @param optionType
*/
async importModel(model: string, modelPath: string): Promise<void> {
return this.cortexAPI?.importModel(model, modelPath)
return this.cortexAPI.importModel(model, modelPath)
}
//// LEGACY MODEL FOLDER ////
/**
* Scan through models folder and return downloaded models
* @returns
*/
private async scanModelsFolder(): Promise<Model[]> {
try {
if (!(await fs.existsSync(JanModelExtension._homeDir))) {
console.debug('Model folder not found')
return []
}
const files: string[] = await fs.readdirSync(JanModelExtension._homeDir)
const allDirectories: string[] = []
for (const modelFolder of files) {
const fullModelFolderPath = await joinPath([
JanModelExtension._homeDir,
modelFolder,
])
if (!(await fs.fileStat(fullModelFolderPath)).isDirectory) continue
allDirectories.push(modelFolder)
}
const readJsonPromises = allDirectories.map(async (dirName) => {
// filter out directories that don't match the selector
// read model.json
const folderFullPath = await joinPath([
JanModelExtension._homeDir,
dirName,
])
const jsonPath = await this.getModelJsonPath(folderFullPath)
if (await fs.existsSync(jsonPath)) {
// if we have the model.json file, read it
let model = await fs.readFileSync(jsonPath, 'utf-8')
model = typeof model === 'object' ? model : JSON.parse(model)
// This to ensure backward compatibility with `model.json` with `source_url`
if (model['source_url'] != null) {
model['sources'] = [
{
filename: model.id,
url: model['source_url'],
},
]
}
model.file_path = jsonPath
model.file_name = 'model.json'
// Check model file exist
// model binaries (sources) are absolute path & exist (symlinked)
const existFiles = await Promise.all(
model.sources.map(
(source) =>
// Supposed to be a local file url
!source.url.startsWith(`http://`) &&
!source.url.startsWith(`https://`)
)
)
if (existFiles.every((exist) => exist)) return true
const result = await fs
.readdirSync(await joinPath([JanModelExtension._homeDir, dirName]))
.then((files: string[]) => {
// Model binary exists in the directory
// Model binary name can match model ID or be a .gguf file and not be an incompleted model file
return (
files.includes(dirName) || // Legacy model GGUF without extension
files.filter((file) => {
return (
file.toLowerCase().endsWith('.gguf') || // GGUF
file.toLowerCase().endsWith('.engine') // Tensort-LLM
)
})?.length > 0 // TODO: find better way (can use basename to check the file name with source url)
)
})
if (result) return model
else return undefined
}
})
const results = await Promise.allSettled(readJsonPromises)
const modelData = results
.map((result) => {
if (result.status === 'fulfilled' && result.value) {
try {
const model =
typeof result.value === 'object'
? result.value
: JSON.parse(result.value)
return model as Model
} catch {
console.debug(`Unable to parse model metadata: ${result.value}`)
}
}
return undefined
})
.filter((e) => !!e)
return modelData
} catch (err) {
console.error(err)
return []
}
}
/**
* Retrieve the model.json path from a folder
* @param folderFullPath
* @returns
*/
private async getModelJsonPath(
folderFullPath: string
): Promise<string | undefined> {
// try to find model.json recursively inside each folder
if (!(await fs.existsSync(folderFullPath))) return undefined
const files: string[] = await fs.readdirSync(folderFullPath)
if (files.length === 0) return undefined
if (files.includes('model.json')) {
return joinPath([folderFullPath, 'model.json'])
}
// continue recursive
for (const file of files) {
const path = await joinPath([folderFullPath, file])
const fileStats = await fs.fileStat(path)
if (fileStats.isDirectory) {
const result = await this.getModelJsonPath(path)
if (result) return result
}
}
}
//// END LEGACY MODEL FOLDER ////
}

132
extensions/model-extension/src/model-json.ts Normal file
View File

@ -0,0 +1,132 @@
import { Model, fs, joinPath } from '@janhq/core'
//// LEGACY MODEL FOLDER ////
/**
* Scan through models folder and return downloaded models
* @returns
*/
export const scanModelsFolder = async (): Promise<Model[]> => {
const _homeDir = 'file://models'
try {
if (!(await fs.existsSync(_homeDir))) {
console.debug('Model folder not found')
return []
}
const files: string[] = await fs.readdirSync(_homeDir)
const allDirectories: string[] = []
for (const modelFolder of files) {
const fullModelFolderPath = await joinPath([_homeDir, modelFolder])
if (!(await fs.fileStat(fullModelFolderPath)).isDirectory) continue
allDirectories.push(modelFolder)
}
const readJsonPromises = allDirectories.map(async (dirName) => {
// filter out directories that don't match the selector
// read model.json
const folderFullPath = await joinPath([_homeDir, dirName])
const jsonPath = await getModelJsonPath(folderFullPath)
if (await fs.existsSync(jsonPath)) {
// if we have the model.json file, read it
let model = await fs.readFileSync(jsonPath, 'utf-8')
model = typeof model === 'object' ? model : JSON.parse(model)
// This to ensure backward compatibility with `model.json` with `source_url`
if (model['source_url'] != null) {
model['sources'] = [
{
filename: model.id,
url: model['source_url'],
},
]
}
model.file_path = jsonPath
model.file_name = 'model.json'
// Check model file exist
// model binaries (sources) are absolute path & exist (symlinked)
const existFiles = await Promise.all(
model.sources.map(
(source) =>
// Supposed to be a local file url
!source.url.startsWith(`http://`) &&
!source.url.startsWith(`https://`)
)
)
if (existFiles.every((exist) => exist)) return true
const result = await fs
.readdirSync(await joinPath([_homeDir, dirName]))
.then((files: string[]) => {
// Model binary exists in the directory
// Model binary name can match model ID or be a .gguf file and not be an incompleted model file
return (
files.includes(dirName) || // Legacy model GGUF without extension
files.filter((file) => {
return (
file.toLowerCase().endsWith('.gguf') || // GGUF
file.toLowerCase().endsWith('.engine') // Tensort-LLM
)
})?.length > 0 // TODO: find better way (can use basename to check the file name with source url)
)
})
if (result) return model
else return undefined
}
})
const results = await Promise.allSettled(readJsonPromises)
const modelData = results
.map((result) => {
if (result.status === 'fulfilled' && result.value) {
try {
const model =
typeof result.value === 'object'
? result.value
: JSON.parse(result.value)
return model as Model
} catch {
console.debug(`Unable to parse model metadata: ${result.value}`)
}
}
return undefined
})
.filter((e) => !!e)
return modelData
} catch (err) {
console.error(err)
return []
}
}
/**
* Retrieve the model.json path from a folder
* @param folderFullPath
* @returns
*/
export const getModelJsonPath = async (
folderFullPath: string
): Promise<string | undefined> => {
// try to find model.json recursively inside each folder
if (!(await fs.existsSync(folderFullPath))) return undefined
const files: string[] = await fs.readdirSync(folderFullPath)
if (files.length === 0) return undefined
if (files.includes('model.json')) {
return joinPath([folderFullPath, 'model.json'])
}
// continue recursive
for (const file of files) {
const path = await joinPath([folderFullPath, file])
const fileStats = await fs.fileStat(path)
if (fileStats.isDirectory) {
const result = await getModelJsonPath(path)
if (result) return result
}
}
}
//// END LEGACY MODEL FOLDER ////

2
web/containers/Providers/EventListener.tsx
View File

@ -111,6 +111,7 @@ const EventListenerWrapper = ({ children }: PropsWithChildren) => {
events.off(DownloadEvent.onFileDownloadUpdate, onFileDownloadUpdate)
events.off(DownloadEvent.onFileDownloadError, onFileDownloadError)
events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess)
events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess)
events.off(DownloadEvent.onFileUnzipSuccess, onFileUnzipSuccess)
}
}, [
@ -118,6 +119,7 @@ const EventListenerWrapper = ({ children }: PropsWithChildren) => {
onFileDownloadError,
onFileDownloadSuccess,
onFileUnzipSuccess,
onFileDownloadStopped,
])
return (

2
web/screens/Settings/MyModels/MyModelList/index.tsx
View File

@ -78,7 +78,7 @@ const MyModelList = ({ model }: Props) => {
<div className="flex gap-x-4">
<div className="md:min-w-[90px] md:max-w-[90px]">
<Badge theme="secondary" className="sm:mr-8">
{toGibibytes(model.metadata?.size)}
{model.metadata?.size ? toGibibytes(model.metadata?.size) : '-'}
</Badge>
</div>