diff --git a/.github/ISSUE_TEMPLATE/epic-request.md b/.github/ISSUE_TEMPLATE/epic-request.md index c52fbc6c1..51941c412 100644 --- a/.github/ISSUE_TEMPLATE/epic-request.md +++ b/.github/ISSUE_TEMPLATE/epic-request.md @@ -7,22 +7,19 @@ assignees: '' --- -**Motivation** +## Motivation - -**Specs & Designs** +## Specs - -**In Scope** +## Designs +[Figma](link) + +## Tasklist +- [ ] + +## Not in Scope - -**Not in Scope** -- - -**Tasklist** -> Note: All issues need to share the same `milestone` as this epic -- - -**Related Milestones** -- Past -- Future +## Appendix diff --git a/README.md b/README.md index 1443fe245..f5761903a 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nighlty Build) - + Github action artifactory diff --git a/core/src/api/index.ts b/core/src/api/index.ts index 64e781096..4f2b45f5f 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -10,6 +10,7 @@ export enum AppRoute { openAppDirectory = 'openAppDirectory', openFileExplore = 'openFileExplorer', relaunch = 'relaunch', + joinPath = 'joinPath' } export enum AppEvent { diff --git a/core/src/core.ts b/core/src/core.ts index f268233b7..0f20feb1e 100644 --- a/core/src/core.ts +++ b/core/src/core.ts @@ -44,6 +44,13 @@ const getUserSpace = (): Promise => global.core.api?.getUserSpace() const openFileExplorer: (path: string) => Promise = (path) => global.core.api?.openFileExplorer(path) +/** + * Joins multiple paths together. + * @param paths - The paths to join. + * @returns {Promise} A promise that resolves with the joined path. + */ +const joinPath: (paths: string[]) => Promise = (paths) => global.core.api?.joinPath(paths) + const getResourcePath: () => Promise = () => global.core.api?.getResourcePath() /** @@ -66,4 +73,5 @@ export { getUserSpace, openFileExplorer, getResourcePath, + joinPath, } diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts index 1dd12c89b..23d27935e 100644 --- a/core/src/types/model/modelEntity.ts +++ b/core/src/types/model/modelEntity.ts @@ -67,13 +67,6 @@ export type Model = { */ description: string - /** - * The model state. - * Default: "to_download" - * Enum: "to_download" "downloading" "ready" "running" - */ - state?: ModelState - /** * The model settings. */ @@ -101,15 +94,6 @@ export type ModelMetadata = { cover?: string } -/** - * The Model transition states. - */ -export enum ModelState { - Downloading = 'downloading', - Ready = 'ready', - Running = 'running', -} - /** * The available model settings. */ diff --git a/docs/docs/guides/04-using-models/02-import-manually.mdx b/docs/docs/guides/04-using-models/02-import-manually.mdx index fecb3fa4d..6fc7e04a3 100644 --- a/docs/docs/guides/04-using-models/02-import-manually.mdx +++ b/docs/docs/guides/04-using-models/02-import-manually.mdx @@ -1,24 +1,42 @@ --- title: Import Models Manually +slug: /guides/using-models/import-manually +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + import-models-manually, + ] --- +:::caution +This is currently under development. +::: + {/* Imports */} import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; Jan is compatible with all GGUF models. -If you don't see the model you want in the Hub, or if you have a custom model, you can add it to Jan. +If you can not find the model you want in the Hub or have a custom model you want to use, you can import it manually. -In this guide we will use our latest model, [Trinity](https://huggingface.co/janhq/trinity-v1-GGUF), as an example. +In this guide, we will show you how to import a GGUF model from [HuggingFace](https://huggingface.co/), using our lastest model, [Trinity](https://huggingface.co/janhq/trinity-v1-GGUF), as an example. > We are fast shipping a UI to make this easier, but it's a bit manual for now. Apologies. -## 1. Create a model folder +## Steps to Manually Import a Model -Navigate to the `~/jan/models` folder on your computer. +### 1. Create a Model Folder -In `App Settings`, go to `Advanced`, then `Open App Directory`. +Navigate to the `~/jan/models` folder. You can find this folder by going to `App Settings` > `Advanced` > `Open App Directory`. @@ -70,11 +88,11 @@ In the `models` folder, create a folder with the name of the model. -## 2. Create a model JSON +### 2. Create a Model JSON -Jan follows a folder-based, [standard model template](/specs/models) called a `model.json` to persist the model configurations on your local filesystem. +Jan follows a folder-based, [standard model template](/docs/engineering/models) called a `model.json` to persist the model configurations on your local filesystem. -This means you can easily & transparently reconfigure your models and export and share your preferences. +This means that you can easily reconfigure your models, export them, and share your preferences transparently. @@ -89,7 +107,7 @@ This means you can easily & transparently reconfigure your models and export and ```sh cd trinity-v1-7b - touch model.json + echo {} > model.json ``` @@ -103,45 +121,53 @@ This means you can easily & transparently reconfigure your models and export and -Copy the following configurations into the `model.json`. +Edit `model.json` and include the following configurations: -1. Make sure the `id` property is the same as the folder name you created. -2. Make sure the `source_url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the directl links in `Files and versions` tab. -3. Ensure you are using the correct `prompt_template`. This is usually provided in the HuggingFace model's description page. +- Ensure the filename must be `model.json`. +- Ensure the `id` property matches the folder name you created. +- Ensure the GGUF filename should match the `id` property exactly. +- Ensure the `source_url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in `Files and versions` tab. +- Ensure you are using the correct `prompt_template`. This is usually provided in the HuggingFace model's description page. +- Ensure the `state` property is set to `ready`. ```js { + // highlight-start "source_url": "https://huggingface.co/janhq/trinity-v1-GGUF/resolve/main/trinity-v1.Q4_K_M.gguf", "id": "trinity-v1-7b", + // highlight-end "object": "model", - "name": "Trinity 7B Q4", + "name": "Trinity-v1 7B Q4", "version": "1.0", "description": "Trinity is an experimental model merge of GreenNodeLM & LeoScorpius using the Slerp method. Recommended for daily assistance purposes.", "format": "gguf", "settings": { - "ctx_len": 2048, - "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant" + "ctx_len": 4096, + // highlight-next-line + "prompt_template": "{system_message}\n### Instruction:\n{prompt}\n### Response:" }, "parameters": { - "max_tokens": 2048 + "max_tokens": 4096 }, "metadata": { "author": "Jan", - "tags": ["7B", "Merged", "Featured"], + "tags": ["7B", "Merged"], "size": 4370000000 }, + // highlight-next-line + "state": "ready", "engine": "nitro" } ``` -## 3. Download your model +### 3. Download the Model -Restart the Jan application and look for your model in the Hub. +Restart Jan and navigate to the Hub. Locate your model and click the `Download` button to download the model binary. -Click the green `download` button to download your actual model binary. This pulls from the `source_url` you provided above. +![image](assets/download-model.png) -![image](https://hackmd.io/_uploads/HJLAqvwI6.png) +Your model is now ready to use in Jan. -There you go! You are ready to use your model. +## Assistance and Support -If you have any questions or want to request for more preconfigured GGUF models, please message us in [Discord](https://discord.gg/Dt7MxDyNNZ). +If you have questions or are looking for more preconfigured GGUF models, please feel free to join our [Discord community](https://discord.gg/Dt7MxDyNNZ) for support, updates, and discussions. diff --git a/docs/docs/guides/04-using-models/assets/download-model.png b/docs/docs/guides/04-using-models/assets/download-model.png new file mode 100644 index 000000000..b2fddaee9 Binary files /dev/null and b/docs/docs/guides/04-using-models/assets/download-model.png differ diff --git a/electron/handlers/app.ts b/electron/handlers/app.ts index fc2d0ee59..ff88cd8f1 100644 --- a/electron/handlers/app.ts +++ b/electron/handlers/app.ts @@ -48,6 +48,13 @@ export function handleAppIPCs() { shell.openPath(url) }) + /** + * Joins multiple paths together, respect to the current OS. + */ + ipcMain.handle(AppRoute.joinPath, async (_event, paths: string[]) => + join(...paths) + ) + /** * Relaunches the app in production - reload window in development. * @param _event - The IPC event object. diff --git a/electron/handlers/download.ts b/electron/handlers/download.ts index 6e64d23e2..145174ac2 100644 --- a/electron/handlers/download.ts +++ b/electron/handlers/download.ts @@ -3,7 +3,7 @@ import { DownloadManager } from './../managers/download' import { resolve, join } from 'path' import { WindowManager } from './../managers/window' import request from 'request' -import { createWriteStream } from 'fs' +import { createWriteStream, renameSync } from 'fs' import { DownloadEvent, DownloadRoute } from '@janhq/core' const progress = require('request-progress') @@ -48,6 +48,8 @@ export function handleDownloaderIPCs() { const userDataPath = join(app.getPath('home'), 'jan') const destination = resolve(userDataPath, fileName) const rq = request(url) + // downloading file to a temp file first + const downloadingTempFile = `${destination}.download` progress(rq, {}) .on('progress', function (state: any) { @@ -70,6 +72,9 @@ export function handleDownloaderIPCs() { }) .on('end', function () { if (DownloadManager.instance.networkRequests[fileName]) { + // Finished downloading, rename temp file to actual file + renameSync(downloadingTempFile, destination) + WindowManager?.instance.currentWindow?.webContents.send( DownloadEvent.onFileDownloadSuccess, { @@ -87,7 +92,7 @@ export function handleDownloaderIPCs() { ) } }) - .pipe(createWriteStream(destination)) + .pipe(createWriteStream(downloadingTempFile)) DownloadManager.instance.setRequest(fileName, rq) }) diff --git a/extensions/conversational-extension/src/index.ts b/extensions/conversational-extension/src/index.ts index 0fdf0b2d4..8aae791e8 100644 --- a/extensions/conversational-extension/src/index.ts +++ b/extensions/conversational-extension/src/index.ts @@ -1,7 +1,6 @@ -import { ExtensionType, fs } from '@janhq/core' +import { ExtensionType, fs, joinPath } from '@janhq/core' import { ConversationalExtension } from '@janhq/core' import { Thread, ThreadMessage } from '@janhq/core' -import { join } from 'path' /** * JSONConversationalExtension is a ConversationalExtension implementation that provides @@ -69,14 +68,14 @@ export default class JSONConversationalExtension */ async saveThread(thread: Thread): Promise { try { - const threadDirPath = join( + const threadDirPath = await joinPath([ JSONConversationalExtension._homeDir, - thread.id - ) - const threadJsonPath = join( + thread.id, + ]) + const threadJsonPath = await joinPath([ threadDirPath, - JSONConversationalExtension._threadInfoFileName - ) + JSONConversationalExtension._threadInfoFileName, + ]) await fs.mkdir(threadDirPath) await fs.writeFile(threadJsonPath, JSON.stringify(thread, null, 2)) Promise.resolve() @@ -89,20 +88,22 @@ export default class JSONConversationalExtension * Delete a thread with the specified ID. * @param threadId The ID of the thread to delete. */ - deleteThread(threadId: string): Promise { - return fs.rmdir(join(JSONConversationalExtension._homeDir, `${threadId}`)) + async deleteThread(threadId: string): Promise { + return fs.rmdir( + await joinPath([JSONConversationalExtension._homeDir, `${threadId}`]) + ) } async addNewMessage(message: ThreadMessage): Promise { try { - const threadDirPath = join( + const threadDirPath = await joinPath([ JSONConversationalExtension._homeDir, - message.thread_id - ) - const threadMessagePath = join( + message.thread_id, + ]) + const threadMessagePath = await joinPath([ threadDirPath, - JSONConversationalExtension._threadMessagesFileName - ) + JSONConversationalExtension._threadMessagesFileName, + ]) await fs.mkdir(threadDirPath) await fs.appendFile(threadMessagePath, JSON.stringify(message) + '\n') Promise.resolve() @@ -116,11 +117,14 @@ export default class JSONConversationalExtension messages: ThreadMessage[] ): Promise { try { - const threadDirPath = join(JSONConversationalExtension._homeDir, threadId) - const threadMessagePath = join( + const threadDirPath = await joinPath([ + JSONConversationalExtension._homeDir, + threadId, + ]) + const threadMessagePath = await joinPath([ threadDirPath, - JSONConversationalExtension._threadMessagesFileName - ) + JSONConversationalExtension._threadMessagesFileName, + ]) await fs.mkdir(threadDirPath) await fs.writeFile( threadMessagePath, @@ -140,11 +144,11 @@ export default class JSONConversationalExtension */ private async readThread(threadDirName: string): Promise { return fs.readFile( - join( + await joinPath([ JSONConversationalExtension._homeDir, threadDirName, - JSONConversationalExtension._threadInfoFileName - ) + JSONConversationalExtension._threadInfoFileName, + ]) ) } @@ -159,10 +163,10 @@ export default class JSONConversationalExtension const threadDirs: string[] = [] for (let i = 0; i < fileInsideThread.length; i++) { - const path = join( + const path = await joinPath([ JSONConversationalExtension._homeDir, - fileInsideThread[i] - ) + fileInsideThread[i], + ]) const isDirectory = await fs.isDirectory(path) if (!isDirectory) { console.debug(`Ignore ${path} because it is not a directory`) @@ -184,7 +188,10 @@ export default class JSONConversationalExtension async getAllMessages(threadId: string): Promise { try { - const threadDirPath = join(JSONConversationalExtension._homeDir, threadId) + const threadDirPath = await joinPath([ + JSONConversationalExtension._homeDir, + threadId, + ]) const isDir = await fs.isDirectory(threadDirPath) if (!isDir) { throw Error(`${threadDirPath} is not directory`) @@ -197,10 +204,10 @@ export default class JSONConversationalExtension throw Error(`${threadDirPath} not contains message file`) } - const messageFilePath = join( + const messageFilePath = await joinPath([ threadDirPath, - JSONConversationalExtension._threadMessagesFileName - ) + JSONConversationalExtension._threadMessagesFileName, + ]) const result = await fs.readLineByLine(messageFilePath) diff --git a/extensions/inference-nitro-extension/bin/linux-start.sh b/extensions/inference-nitro-extension/bin/linux-start.sh index 3cf9c8013..199589ad2 100644 --- a/extensions/inference-nitro-extension/bin/linux-start.sh +++ b/extensions/inference-nitro-extension/bin/linux-start.sh @@ -28,11 +28,12 @@ export CUDA_VISIBLE_DEVICES=$selectedGpuId # Attempt to run nitro_linux_amd64_cuda cd linux-cuda -if ./nitro "$@"; then +./nitro "$@" > output.log 2>&1 || ( + echo "Check output log" && + if grep -q "CUDA error" output.log; then + echo "CUDA error detected, attempting to run nitro_linux_amd64..." + cd ../linux-cpu && ./nitro "$@" + exit $? + fi exit $? -else - echo "nitro_linux_amd64_cuda encountered an error, attempting to run nitro_linux_amd64..." - cd ../linux-cpu - ./nitro "$@" - exit $? -fi \ No newline at end of file +) diff --git a/extensions/inference-nitro-extension/bin/win-start.bat b/extensions/inference-nitro-extension/bin/win-start.bat index e18c97a8e..250e0d218 100644 --- a/extensions/inference-nitro-extension/bin/win-start.bat +++ b/extensions/inference-nitro-extension/bin/win-start.bat @@ -31,9 +31,10 @@ set CUDA_VISIBLE_DEVICES=!gpuId! rem Attempt to run nitro_windows_amd64_cuda.exe cd win-cuda -nitro.exe %* -if %errorlevel% neq 0 goto RunCpuVersion -goto End + +nitro.exe %* > output.log +type output.log | findstr /C:"CUDA error" >nul +if %errorlevel% equ 0 ( goto :RunCpuVersion ) else ( goto :End ) :RunCpuVersion rem Run nitro_windows_amd64.exe... diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 946d526dd..d19f3853c 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -111,7 +111,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension { return; } const userSpacePath = await getUserSpace(); - const modelFullPath = join(userSpacePath, "models", model.id, model.id); + const modelFullPath = join(userSpacePath, "models", model.id); const nitroInitResult = await executeOnMain(MODULE, "initModel", { modelFullPath: modelFullPath, diff --git a/extensions/inference-nitro-extension/src/module.ts b/extensions/inference-nitro-extension/src/module.ts index bca0b6fcc..37b9e5b3b 100644 --- a/extensions/inference-nitro-extension/src/module.ts +++ b/extensions/inference-nitro-extension/src/module.ts @@ -13,10 +13,11 @@ const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/ const NITRO_HTTP_UNLOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/unloadModel`; const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; +const SUPPORTED_MODEL_FORMAT = ".gguf"; // The subprocess instance for Nitro let subprocess = undefined; -let currentModelFile = undefined; +let currentModelFile: string = undefined; let currentSettings = undefined; /** @@ -37,6 +38,17 @@ function stopModel(): Promise { */ async function initModel(wrapper: any): Promise { currentModelFile = wrapper.modelFullPath; + const files: string[] = fs.readdirSync(currentModelFile); + + // Look for GGUF model file + const ggufBinFile = files.find( + (file) => + file === path.basename(currentModelFile) || + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) + ); + + currentModelFile = path.join(currentModelFile, ggufBinFile); + if (wrapper.model.engine !== "nitro") { return Promise.resolve({ error: "Not a nitro model" }); } else { @@ -66,15 +78,31 @@ async function initModel(wrapper: any): Promise { async function loadModel(nitroResourceProbe: any | undefined) { // Gather system information for CPU physical cores and memory if (!nitroResourceProbe) nitroResourceProbe = await getResourcesInfo(); - return killSubprocess() - .then(() => spawnNitroProcess(nitroResourceProbe)) - .then(() => loadLLMModel(currentSettings)) - .then(validateModelStatus) - .catch((err) => { - console.error("error: ", err); - // TODO: Broadcast error so app could display proper error message - return { error: err, currentModelFile }; - }); + return ( + killSubprocess() + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + // wait for 500ms to make sure the port is free for windows platform + .then(() => { + if (process.platform === "win32") { + return sleep(500); + } else { + return sleep(0); + } + }) + .then(() => spawnNitroProcess(nitroResourceProbe)) + .then(() => loadLLMModel(currentSettings)) + .then(validateModelStatus) + .catch((err) => { + console.error("error: ", err); + // TODO: Broadcast error so app could display proper error message + return { error: err, currentModelFile }; + }) + ); +} + +// Add function sleep +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); } function promptTemplateConverter(promptTemplate) { diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 16adced5d..9580afd9b 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -5,9 +5,11 @@ import { abortDownload, getResourcePath, getUserSpace, + InferenceEngine, + joinPath, } from '@janhq/core' -import { ModelExtension, Model, ModelState } from '@janhq/core' -import { join } from 'path' +import { basename } from 'path' +import { ModelExtension, Model } from '@janhq/core' /** * A extension for models @@ -15,6 +17,9 @@ import { join } from 'path' export default class JanModelExtension implements ModelExtension { private static readonly _homeDir = 'models' private static readonly _modelMetadataFileName = 'model.json' + private static readonly _supportedModelFormat = '.gguf' + private static readonly _incompletedModelFileName = '.download' + private static readonly _offlineInferenceEngine = InferenceEngine.nitro /** * Implements type from JanExtension. @@ -54,10 +59,10 @@ export default class JanModelExtension implements ModelExtension { // copy models folder from resources to home directory const resourePath = await getResourcePath() - const srcPath = join(resourePath, 'models') + const srcPath = await joinPath([resourePath, 'models']) const userSpace = await getUserSpace() - const destPath = join(userSpace, JanModelExtension._homeDir) + const destPath = await joinPath([userSpace, JanModelExtension._homeDir]) await fs.syncFile(srcPath, destPath) @@ -88,11 +93,18 @@ export default class JanModelExtension implements ModelExtension { */ async downloadModel(model: Model): Promise { // create corresponding directory - const directoryPath = join(JanModelExtension._homeDir, model.id) - await fs.mkdir(directoryPath) + const modelDirPath = await joinPath([JanModelExtension._homeDir, model.id]) + await fs.mkdir(modelDirPath) - // path to model binary - const path = join(directoryPath, model.id) + // try to retrieve the download file name from the source url + // if it fails, use the model ID as the file name + const extractedFileName = basename(model.source_url) + const fileName = extractedFileName + .toLowerCase() + .endsWith(JanModelExtension._supportedModelFormat) + ? extractedFileName + : model.id + const path = await joinPath([modelDirPath, fileName]) downloadFile(model.source_url, path) } @@ -103,10 +115,12 @@ export default class JanModelExtension implements ModelExtension { */ async cancelModelDownload(modelId: string): Promise { return abortDownload( - join(JanModelExtension._homeDir, modelId, modelId) - ).then(() => { - fs.deleteFile(join(JanModelExtension._homeDir, modelId, modelId)) - }) + await joinPath([JanModelExtension._homeDir, modelId, modelId]) + ).then(async () => + fs.deleteFile( + await joinPath([JanModelExtension._homeDir, modelId, modelId]) + ) + ) } /** @@ -116,27 +130,16 @@ export default class JanModelExtension implements ModelExtension { */ async deleteModel(modelId: string): Promise { try { - const dirPath = join(JanModelExtension._homeDir, modelId) + const dirPath = await joinPath([JanModelExtension._homeDir, modelId]) // remove all files under dirPath except model.json const files = await fs.listFiles(dirPath) - const deletePromises = files.map((fileName: string) => { + const deletePromises = files.map(async (fileName: string) => { if (fileName !== JanModelExtension._modelMetadataFileName) { - return fs.deleteFile(join(dirPath, fileName)) + return fs.deleteFile(await joinPath([dirPath, fileName])) } }) await Promise.allSettled(deletePromises) - - // update the state as default - const jsonFilePath = join( - dirPath, - JanModelExtension._modelMetadataFileName - ) - const json = await fs.readFile(jsonFilePath) - const model = JSON.parse(json) as Model - delete model.state - - await fs.writeFile(jsonFilePath, JSON.stringify(model, null, 2)) } catch (err) { console.error(err) } @@ -148,24 +151,14 @@ export default class JanModelExtension implements ModelExtension { * @returns A Promise that resolves when the model is saved. */ async saveModel(model: Model): Promise { - const jsonFilePath = join( + const jsonFilePath = await joinPath([ JanModelExtension._homeDir, model.id, - JanModelExtension._modelMetadataFileName - ) + JanModelExtension._modelMetadataFileName, + ]) try { - await fs.writeFile( - jsonFilePath, - JSON.stringify( - { - ...model, - state: ModelState.Ready, - }, - null, - 2 - ) - ) + await fs.writeFile(jsonFilePath, JSON.stringify(model, null, 2)) } catch (err) { console.error(err) } @@ -176,11 +169,34 @@ export default class JanModelExtension implements ModelExtension { * @returns A Promise that resolves with an array of all models. */ async getDownloadedModels(): Promise { - const models = await this.getModelsMetadata() - return models.filter((model) => model.state === ModelState.Ready) + return await this.getModelsMetadata( + async (modelDir: string, model: Model) => { + if (model.engine !== JanModelExtension._offlineInferenceEngine) { + return true + } + return await fs + .listFiles(await joinPath([JanModelExtension._homeDir, modelDir])) + .then((files: string[]) => { + // or model binary exists in the directory + // model binary name can match model ID or be a .gguf file and not be an incompleted model file + return ( + files.includes(modelDir) || + files.some( + (file) => + file + .toLowerCase() + .includes(JanModelExtension._supportedModelFormat) && + !file.endsWith(JanModelExtension._incompletedModelFileName) + ) + ) + }) + } + ) } - private async getModelsMetadata(): Promise { + private async getModelsMetadata( + selector?: (path: string, model: Model) => Promise + ): Promise { try { const filesUnderJanRoot = await fs.listFiles('') if (!filesUnderJanRoot.includes(JanModelExtension._homeDir)) { @@ -193,26 +209,35 @@ export default class JanModelExtension implements ModelExtension { const allDirectories: string[] = [] for (const file of files) { const isDirectory = await fs.isDirectory( - join(JanModelExtension._homeDir, file) + await joinPath([JanModelExtension._homeDir, file]) ) if (isDirectory) { allDirectories.push(file) } } - const readJsonPromises = allDirectories.map((dirName) => { - const jsonPath = join( + const readJsonPromises = allDirectories.map(async (dirName) => { + // filter out directories that don't match the selector + + // read model.json + const jsonPath = await joinPath([ JanModelExtension._homeDir, dirName, - JanModelExtension._modelMetadataFileName - ) - return this.readModelMetadata(jsonPath) + JanModelExtension._modelMetadataFileName, + ]) + let model = await this.readModelMetadata(jsonPath) + model = typeof model === 'object' ? model : JSON.parse(model) + + if (selector && !(await selector?.(dirName, model))) { + return + } + return model }) const results = await Promise.allSettled(readJsonPromises) const modelData = results.map((result) => { if (result.status === 'fulfilled') { try { - return JSON.parse(result.value) as Model + return result.value as Model } catch { console.debug(`Unable to parse model metadata: ${result.value}`) return undefined @@ -230,7 +255,7 @@ export default class JanModelExtension implements ModelExtension { } private readModelMetadata(path: string) { - return fs.readFile(join(path)) + return fs.readFile(path) } /** diff --git a/web/containers/CardSidebar/index.tsx b/web/containers/CardSidebar/index.tsx index 8c7fe3314..e3851214b 100644 --- a/web/containers/CardSidebar/index.tsx +++ b/web/containers/CardSidebar/index.tsx @@ -29,6 +29,13 @@ export default function CardSidebar({ useClickOutside(() => setMore(false), null, [menu, toggle]) + let openFolderTitle: string = 'Open Containing Folder' + if (isMac) { + openFolderTitle = 'Reveal in Finder' + } else if (isWindows) { + openFolderTitle = 'Reveal in File Explorer' + } + return (
- Reveal in Finder + {openFolderTitle}
} -const Checkbox: React.FC = ({ name, title, checked, register }) => { +const Checkbox: React.FC = ({ name, title, checked }) => { const { updateModelParameter } = useUpdateModelParameters() const threadId = useAtomValue(getActiveThreadIdAtom) - const activeModelParams = useAtomValue(getActiveThreadModelRuntimeParamsAtom) const onCheckedChange = (checked: boolean) => { - if (!threadId || !activeModelParams) return + if (!threadId) return - const updatedModelParams: ModelRuntimeParams = { - ...activeModelParams, - [name]: checked, - } - - updateModelParameter(threadId, updatedModelParams) + updateModelParameter(threadId, name, checked) } return (
- - +

{title}

+
) } diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index 86203ddff..e40599344 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -9,10 +9,6 @@ import { SelectItem, SelectTrigger, SelectValue, - Tooltip, - TooltipContent, - TooltipTrigger, - TooltipArrow, Input, } from '@janhq/uikit' @@ -32,14 +28,22 @@ import useRecommendedModel from '@/hooks/useRecommendedModel' import { toGigabytes } from '@/utils/converter' -import { activeThreadAtom, threadStatesAtom } from '@/helpers/atoms/Thread.atom' +import { + activeThreadAtom, + getActiveThreadIdAtom, + setThreadModelParamsAtom, + threadStatesAtom, +} from '@/helpers/atoms/Thread.atom' export const selectedModelAtom = atom(undefined) export default function DropdownListSidebar() { - const setSelectedModel = useSetAtom(selectedModelAtom) - const threadStates = useAtomValue(threadStatesAtom) + const activeThreadId = useAtomValue(getActiveThreadIdAtom) const activeThread = useAtomValue(activeThreadAtom) + const threadStates = useAtomValue(threadStatesAtom) + const setSelectedModel = useSetAtom(selectedModelAtom) + const setThreadModelParams = useSetAtom(setThreadModelParamsAtom) + const [selected, setSelected] = useState() const { setMainViewState } = useMainViewState() const [openAISettings, setOpenAISettings] = useState< @@ -58,83 +62,93 @@ export default function DropdownListSidebar() { useEffect(() => { setSelected(recommendedModel) setSelectedModel(recommendedModel) - }, [recommendedModel, setSelectedModel]) + + if (activeThread) { + const finishInit = threadStates[activeThread.id].isFinishInit ?? true + if (finishInit) return + const modelParams = { + ...recommendedModel?.parameters, + ...recommendedModel?.settings, + } + setThreadModelParams(activeThread.id, modelParams) + } + }, [ + recommendedModel, + activeThread, + setSelectedModel, + setThreadModelParams, + threadStates, + ]) const onValueSelected = useCallback( (modelId: string) => { const model = downloadedModels.find((m) => m.id === modelId) setSelected(model) setSelectedModel(model) + + if (activeThreadId) { + const modelParams = { + ...model?.parameters, + ...model?.settings, + } + setThreadModelParams(activeThreadId, modelParams) + } }, - [downloadedModels, setSelectedModel] + [downloadedModels, activeThreadId, setSelectedModel, setThreadModelParams] ) if (!activeThread) { return null } - const finishInit = threadStates[activeThread.id].isFinishInit ?? true return ( - - - + + + {downloadedModels.filter((x) => x.id === selected?.id)[0]?.name} + + + +
+ + Local +
+
+ {downloadedModels.length === 0 ? ( +
+

{`Oops, you don't have a model yet.`}

-
- {downloadedModels.length === 0 ? ( -
-

{`Oops, you don't have a model yet.`}

-
- ) : ( - - {downloadedModels.map((x, i) => ( - -
- {x.name} - - {toGigabytes(x.metadata.size)} - -
- - ))} - - )} -
-
- -
- - - - - {finishInit && ( - - Start a new thread to change the model - - - )} + ) : ( + + {downloadedModels.map((x, i) => ( + +
+ {x.name} + + {toGigabytes(x.metadata.size)} + +
+ + ))} + + )} +
+
+ +
+ + {selected?.engine === InferenceEngine.openai && (
@@ -154,6 +168,6 @@ export default function DropdownListSidebar() { />
)} - + ) } diff --git a/web/containers/Layout/BottomBar/DownloadingState/index.tsx b/web/containers/Layout/BottomBar/DownloadingState/index.tsx index 0648508d0..7aef36caf 100644 --- a/web/containers/Layout/BottomBar/DownloadingState/index.tsx +++ b/web/containers/Layout/BottomBar/DownloadingState/index.tsx @@ -1,7 +1,5 @@ import { Fragment } from 'react' -import { ExtensionType } from '@janhq/core' -import { ModelExtension } from '@janhq/core' import { Progress, Modal, @@ -12,14 +10,19 @@ import { ModalTrigger, } from '@janhq/uikit' +import { useAtomValue } from 'jotai' + +import useDownloadModel from '@/hooks/useDownloadModel' import { useDownloadState } from '@/hooks/useDownloadState' import { formatDownloadPercentage } from '@/utils/converter' -import { extensionManager } from '@/extension' +import { downloadingModelsAtom } from '@/helpers/atoms/Model.atom' export default function DownloadingState() { const { downloadStates } = useDownloadState() + const downloadingModels = useAtomValue(downloadingModelsAtom) + const { abortModelDownload } = useDownloadModel() const totalCurrentProgress = downloadStates .map((a) => a.size.transferred + a.size.transferred) @@ -73,9 +76,10 @@ export default function DownloadingState() { size="sm" onClick={() => { if (item?.modelId) { - extensionManager - .get(ExtensionType.Model) - ?.cancelModelDownload(item.modelId) + const model = downloadingModels.find( + (model) => model.id === item.modelId + ) + if (model) abortModelDownload(model) } }} > diff --git a/web/containers/ModalCancelDownload/index.tsx b/web/containers/ModalCancelDownload/index.tsx index d1a6f1a44..2a5626183 100644 --- a/web/containers/ModalCancelDownload/index.tsx +++ b/web/containers/ModalCancelDownload/index.tsx @@ -1,6 +1,5 @@ import { useMemo } from 'react' -import { ModelExtension, ExtensionType } from '@janhq/core' import { Model } from '@janhq/core' import { @@ -17,11 +16,12 @@ import { import { atom, useAtomValue } from 'jotai' +import useDownloadModel from '@/hooks/useDownloadModel' import { useDownloadState } from '@/hooks/useDownloadState' import { formatDownloadPercentage } from '@/utils/converter' -import { extensionManager } from '@/extension' +import { downloadingModelsAtom } from '@/helpers/atoms/Model.atom' type Props = { model: Model @@ -30,6 +30,7 @@ type Props = { export default function ModalCancelDownload({ model, isFromList }: Props) { const { modelDownloadStateAtom } = useDownloadState() + const downloadingModels = useAtomValue(downloadingModelsAtom) const downloadAtom = useMemo( () => atom((get) => get(modelDownloadStateAtom)[model.id]), // eslint-disable-next-line react-hooks/exhaustive-deps @@ -37,6 +38,7 @@ export default function ModalCancelDownload({ model, isFromList }: Props) { ) const downloadState = useAtomValue(downloadAtom) const cancelText = `Cancel ${formatDownloadPercentage(downloadState.percent)}` + const { abortModelDownload } = useDownloadModel() return ( @@ -80,9 +82,10 @@ export default function ModalCancelDownload({ model, isFromList }: Props) { themes="danger" onClick={() => { if (downloadState?.modelId) { - extensionManager - .get(ExtensionType.Model) - ?.cancelModelDownload(downloadState.modelId) + const model = downloadingModels.find( + (model) => model.id === downloadState.modelId + ) + if (model) abortModelDownload(model) } }} > diff --git a/web/containers/ModelConfigInput/index.tsx b/web/containers/ModelConfigInput/index.tsx new file mode 100644 index 000000000..caa4f7fa3 --- /dev/null +++ b/web/containers/ModelConfigInput/index.tsx @@ -0,0 +1,43 @@ +import { Textarea } from '@janhq/uikit' + +import { useAtomValue } from 'jotai' + +import useUpdateModelParameters from '@/hooks/useUpdateModelParameters' + +import { getActiveThreadIdAtom } from '@/helpers/atoms/Thread.atom' + +type Props = { + title: string + name: string + placeholder: string + value: string +} + +const ModelConfigInput: React.FC = ({ + title, + name, + value, + placeholder, +}) => { + const { updateModelParameter } = useUpdateModelParameters() + const threadId = useAtomValue(getActiveThreadIdAtom) + + const onValueChanged = (e: React.ChangeEvent) => { + if (!threadId) return + + updateModelParameter(threadId, name, e.target.value) + } + + return ( +
+

{title}

+