import PQueue from 'p-queue' import ky from 'ky' import { extractModelLoadParams, Model, ModelSource } from '@janhq/core' import { extractInferenceParams } from '@janhq/core' /** * cortex.cpp Model APIs interface */ interface ICortexAPI { getModel(model: string): Promise getModels(): Promise pullModel(model: string, id?: string, name?: string): Promise importModel( path: string, modelPath: string, name?: string, option?: string ): Promise deleteModel(model: string): Promise updateModel(model: object): Promise cancelModelPull(model: string): Promise configs(body: { [key: string]: any }): Promise getSources(): Promise addSource(source: string): Promise deleteSource(source: string): Promise } type Data = { data: any[] } export class CortexAPI implements ICortexAPI { queue = new PQueue({ concurrency: 1 }) constructor() { this.queue.add(() => this.healthz()) } /** * Fetches a model detail from cortex.cpp * @param model * @returns */ getModel(model: string): Promise { return this.queue.add(() => ky .get(`${API_URL}/v1/models/${model}`) .json() .then((e) => this.transformModel(e)) ) } /** * Fetches models list from cortex.cpp * @param model * @returns */ getModels(): Promise { return this.queue .add(() => ky.get(`${API_URL}/v1/models?limit=-1`).json()) .then((e) => typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : [] ) } /** * Pulls a model from HuggingFace via cortex.cpp * @param model * @returns */ pullModel(model: string, id?: string, name?: string): Promise { return this.queue.add(() => ky .post(`${API_URL}/v1/models/pull`, { json: { model, id, name } }) .json() .catch(async (e) => { throw (await e.response?.json()) ?? e }) .then() ) } /** * Imports a model from a local path via cortex.cpp * @param model * @returns */ importModel( model: string, modelPath: string, name?: string, option?: string ): Promise { return this.queue.add(() => ky .post(`${API_URL}/v1/models/import`, { json: { model, modelPath, name, option }, }) .json() .catch((e) => console.debug(e)) // Ignore error .then() ) } /** * Deletes a model from cortex.cpp * @param model * @returns */ deleteModel(model: string): Promise { return this.queue.add(() => ky.delete(`${API_URL}/v1/models/${model}`).json().then() ) } /** * Update a model in cortex.cpp * @param model * @returns */ updateModel(model: Partial): Promise { return this.queue.add(() => ky .patch(`${API_URL}/v1/models/${model.id}`, { json: { ...model } }) .json() .then() ) } /** * Cancel model pull in cortex.cpp * @param model * @returns */ cancelModelPull(model: string): Promise { return this.queue.add(() => ky .delete(`${API_URL}/v1/models/pull`, { json: { taskId: model } }) .json() .then() ) } /** * Check model status * @param model */ async getModelStatus(model: string): Promise { return this.queue .add(() => ky.get(`${API_URL}/v1/models/status/${model}`)) .then((e) => true) .catch(() => false) } // BEGIN - Model Sources /** * Get model sources * @param model */ async getSources(): Promise { return this.queue .add(() => ky.get(`${API_URL}/v1/models/sources`).json()) .then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : [])) .catch(() => []) } /** * Add a model source * @param model */ async addSource(source: string): Promise { return this.queue.add(() => ky.post(`${API_URL}/v1/models/sources`, { json: { source, }, }) ) } /** * Delete a model source * @param model */ async deleteSource(source: string): Promise { return this.queue.add(() => ky.delete(`${API_URL}/v1/models/sources`, { json: { source, }, }) ) } // END - Model Sources /** * Do health check on cortex.cpp * @returns */ healthz(): Promise { return ky .get(`${API_URL}/healthz`, { retry: { limit: 20, delay: () => 500, methods: ['get'], }, }) .then(() => {}) } /** * Configure model pull options * @param body */ configs(body: { [key: string]: any }): Promise { return this.queue.add(() => ky.patch(`${API_URL}/v1/configs`, { json: body }).then(() => {}) ) } /** * TRansform model to the expected format (e.g. parameters, settings, metadata) * @param model * @returns */ private transformModel(model: any) { model.parameters = { ...extractInferenceParams(model), ...model.parameters, ...model.inference_params, } model.settings = { ...extractModelLoadParams(model), ...model.settings, } model.metadata = model.metadata ?? { tags: [], size: model.size ?? model.metadata?.size ?? 0, } return model as Model } }