From b3e24d7f51bc57723612014d83a3e0c87201e5a2 Mon Sep 17 00:00:00 2001 From: Nicole Zhu <69952136+0xSage@users.noreply.github.com> Date: Sun, 17 Mar 2024 14:47:30 -0700 Subject: [PATCH 01/16] Update broken-build.mdx --- docs/docs/guides/common-error/broken-build.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/docs/guides/common-error/broken-build.mdx b/docs/docs/guides/common-error/broken-build.mdx index 0e41d0b4d..2b3e7a128 100644 --- a/docs/docs/guides/common-error/broken-build.mdx +++ b/docs/docs/guides/common-error/broken-build.mdx @@ -67,7 +67,11 @@ This guide provides you steps to troubleshoot and to resolve the issue where you ### 2. Delete Application Data, Cache, and User Data ```sh - # You can delete the `/Jan` directory in Windows's AppData Directory by visiting the following path `%APPDATA%\Jan` + # Delete your own user data + cd ~ # Or where you moved the Jan Data Folder to + rm -r ./jan + + # Delete Application Cache cd C:\Users\YOUR_USERNAME\AppData\Roaming rm -r ./Jan ``` From 0348aa3321c11fcfdf304b3ed3b0033b1664e0a4 Mon Sep 17 00:00:00 2001 From: Meta Spartan Date: Sun, 17 Mar 2024 17:40:20 -0600 Subject: [PATCH 02/16] feat: Groq Inference Extension (#2263) * feat: Groq Inference Extension * Add Groq supported models * Fix folder typo * Add Groq options to interface and new API Key saving, tested working * Fix linting --- core/src/node/helper/config.ts | 2 +- core/src/types/model/modelEntity.ts | 1 + extensions/inference-groq-extension/README.md | 78 ++++++ .../inference-groq-extension/package.json | 41 ++++ .../src/@types/global.d.ts | 16 ++ .../src/helpers/sse.ts | 83 +++++++ .../inference-groq-extension/src/index.ts | 224 ++++++++++++++++++ .../inference-groq-extension/tsconfig.json | 14 ++ .../webpack.config.js | 39 +++ models/groq-llama2-70b/model.json | 27 +++ models/groq-mixtral-8x7b-instruct/model.json | 27 +++ web/containers/DropdownListSidebar/index.tsx | 4 +- web/containers/OpenAiKeyInput/index.tsx | 43 +++- web/hooks/useEngineSettings.ts | 40 +++- web/screens/Settings/Models/Row.tsx | 1 + 15 files changed, 631 insertions(+), 9 deletions(-) create mode 100644 extensions/inference-groq-extension/README.md create mode 100644 extensions/inference-groq-extension/package.json create mode 100644 extensions/inference-groq-extension/src/@types/global.d.ts create mode 100644 extensions/inference-groq-extension/src/helpers/sse.ts create mode 100644 extensions/inference-groq-extension/src/index.ts create mode 100644 extensions/inference-groq-extension/tsconfig.json create mode 100644 extensions/inference-groq-extension/webpack.config.js create mode 100644 models/groq-llama2-70b/model.json create mode 100644 models/groq-mixtral-8x7b-instruct/model.json diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index 06f2b03cd..81bc64611 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -118,7 +118,7 @@ const exec = async (command: string): Promise => { } export const getEngineConfiguration = async (engineId: string) => { - if (engineId !== 'openai') { + if (engineId !== 'openai' && engineId !== 'groq') { return undefined } const directoryPath = join(getJanDataFolderPath(), 'engines') diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts index 74568686b..d62a7c387 100644 --- a/core/src/types/model/modelEntity.ts +++ b/core/src/types/model/modelEntity.ts @@ -18,6 +18,7 @@ export type ModelInfo = { export enum InferenceEngine { nitro = 'nitro', openai = 'openai', + groq = 'groq', triton_trtllm = 'triton_trtllm', nitro_tensorrt_llm = 'nitro-tensorrt-llm', diff --git a/extensions/inference-groq-extension/README.md b/extensions/inference-groq-extension/README.md new file mode 100644 index 000000000..455783efb --- /dev/null +++ b/extensions/inference-groq-extension/README.md @@ -0,0 +1,78 @@ +# Jan inference plugin + +Created using Jan app example + +# Create a Jan Plugin using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 + +## Create Your Own Plugin + +To create your own plugin, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your plugin directory now + +## Update the Plugin Metadata + +The [`package.json`](package.json) file defines metadata about your plugin, such as +plugin name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your plugin. + +## Update the Plugin Code + +The [`src/`](./src/) directory is the heart of your plugin! This contains the +source code that will be run when your plugin extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your plugin code: + +- Most Jan Plugin Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { core } from "@janhq/core"; + + function onStart(): Promise { + return core.invokePluginFunc(MODULE_PATH, "run", 0); + } + ``` + + For more information about the Jan Plugin Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your plugin! + diff --git a/extensions/inference-groq-extension/package.json b/extensions/inference-groq-extension/package.json new file mode 100644 index 000000000..c6faf5418 --- /dev/null +++ b/extensions/inference-groq-extension/package.json @@ -0,0 +1,41 @@ +{ + "name": "@janhq/inference-groq-extension", + "version": "1.0.0", + "description": "This extension enables fast Groq chat completion API calls", + "main": "dist/index.js", + "module": "dist/module.js", + "author": "Carsen Klock & Jan", + "license": "AGPL-3.0", + "scripts": { + "build": "tsc -b . && webpack --config webpack.config.js", + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/module.js" + }, + "devDependencies": { + "cpx": "^1.5.0", + "rimraf": "^3.0.2", + "webpack": "^5.88.2", + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" + }, + "dependencies": { + "@janhq/core": "file:../../core", + "fetch-retry": "^5.0.6", + "path-browserify": "^1.0.1", + "ulid": "^2.3.0" + }, + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ], + "bundleDependencies": [ + "fetch-retry" + ] +} diff --git a/extensions/inference-groq-extension/src/@types/global.d.ts b/extensions/inference-groq-extension/src/@types/global.d.ts new file mode 100644 index 000000000..f817fb406 --- /dev/null +++ b/extensions/inference-groq-extension/src/@types/global.d.ts @@ -0,0 +1,16 @@ +declare const MODULE: string +declare const GROQ_DOMAIN: string + +declare interface EngineSettings { + full_url?: string + api_key?: string +} + +enum GroqChatCompletionModelName { + 'mixtral-8x7b-32768' = 'mixtral-8x7b-32768', + 'llama2-70b-4096' = 'llama2-70b-4096', +} + +declare type GroqModel = Omit & { + id: GroqChatCompletionModelName +} diff --git a/extensions/inference-groq-extension/src/helpers/sse.ts b/extensions/inference-groq-extension/src/helpers/sse.ts new file mode 100644 index 000000000..35c40053c --- /dev/null +++ b/extensions/inference-groq-extension/src/helpers/sse.ts @@ -0,0 +1,83 @@ +import { ErrorCode } from '@janhq/core' +import { Observable } from 'rxjs' + +/** + * Sends a request to the inference server to generate a response based on the recent messages. + * @param recentMessages - An array of recent messages to use as context for the inference. + * @param engine - The engine settings to use for the inference. + * @param model - The model to use for the inference. + * @returns An Observable that emits the generated response as a string. + */ +export function requestInference( + recentMessages: any[], + engine: EngineSettings, + model: GroqModel, + controller?: AbortController +): Observable { + return new Observable((subscriber) => { + // let model_id: string = model.id + + const requestBody = JSON.stringify({ + messages: recentMessages, + stream: true, + model: model.id, + ...model.parameters, + }) + fetch(`${engine.full_url}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Authorization': `Bearer ${engine.api_key}`, + // 'api-key': `${engine.api_key}`, + }, + body: requestBody, + signal: controller?.signal, + }) + .then(async (response) => { + if (!response.ok) { + const data = await response.json() + const error = { + message: data.error?.message ?? 'An error occurred.', + code: data.error?.code ?? ErrorCode.Unknown, + } + subscriber.error(error) + subscriber.complete() + return + } + if (model.parameters.stream === false) { + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') + } else { + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' + + while (true && reader) { + const { done, value } = await reader.read() + if (done) { + break + } + const text = decoder.decode(value) + const lines = text.trim().split('\n') + for (const line of lines) { + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') + } + subscriber.next(content) + } + } + } + } + subscriber.complete() + }) + .catch((err) => subscriber.error(err)) + }) +} diff --git a/extensions/inference-groq-extension/src/index.ts b/extensions/inference-groq-extension/src/index.ts new file mode 100644 index 000000000..0fe22a11c --- /dev/null +++ b/extensions/inference-groq-extension/src/index.ts @@ -0,0 +1,224 @@ +/** + * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + * @version 1.0.0 + * @module inference-groq-extension/src/index + */ + +import { + ChatCompletionRole, + ContentType, + MessageRequest, + MessageStatus, + ThreadContent, + ThreadMessage, + events, + fs, + InferenceEngine, + BaseExtension, + MessageEvent, + MessageRequestType, + ModelEvent, + InferenceEvent, + AppConfigurationEventName, + joinPath, +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' + +/** + * A class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + */ +export default class JanInferenceGroqExtension extends BaseExtension { + private static readonly _engineDir = 'file://engines' + private static readonly _engineMetadataFileName = 'groq.json' + + private static _currentModel: GroqModel + + private static _engineSettings: EngineSettings = { + full_url: 'https://api.groq.com/openai/v1/chat/completions', + api_key: 'gsk-', + } + + controller = new AbortController() + isCancelled = false + + /** + * Subscribes to events emitted by the @janhq/core package. + */ + async onLoad() { + if (!(await fs.existsSync(JanInferenceGroqExtension._engineDir))) { + await fs + .mkdirSync(JanInferenceGroqExtension._engineDir) + .catch((err) => console.debug(err)) + } + + JanInferenceGroqExtension.writeDefaultEngineSettings() + + // Events subscription + events.on(MessageEvent.OnMessageSent, (data) => + JanInferenceGroqExtension.handleMessageRequest(data, this) + ) + + events.on(ModelEvent.OnModelInit, (model: GroqModel) => { + JanInferenceGroqExtension.handleModelInit(model) + }) + + events.on(ModelEvent.OnModelStop, (model: GroqModel) => { + JanInferenceGroqExtension.handleModelStop(model) + }) + events.on(InferenceEvent.OnInferenceStopped, () => { + JanInferenceGroqExtension.handleInferenceStopped(this) + }) + + const settingsFilePath = await joinPath([ + JanInferenceGroqExtension._engineDir, + JanInferenceGroqExtension._engineMetadataFileName, + ]) + + events.on( + AppConfigurationEventName.OnConfigurationUpdate, + (settingsKey: string) => { + // Update settings on changes + if (settingsKey === settingsFilePath) + JanInferenceGroqExtension.writeDefaultEngineSettings() + } + ) + } + + /** + * Stops the model inference. + */ + onUnload(): void {} + + static async writeDefaultEngineSettings() { + try { + const engineFile = join( + JanInferenceGroqExtension._engineDir, + JanInferenceGroqExtension._engineMetadataFileName + ) + if (await fs.existsSync(engineFile)) { + const engine = await fs.readFileSync(engineFile, 'utf-8') + JanInferenceGroqExtension._engineSettings = + typeof engine === 'object' ? engine : JSON.parse(engine) + } else { + await fs.writeFileSync( + engineFile, + JSON.stringify(JanInferenceGroqExtension._engineSettings, null, 2) + ) + } + } catch (err) { + console.error(err) + } + } + private static async handleModelInit(model: GroqModel) { + if (model.engine !== InferenceEngine.groq) { + return + } else { + JanInferenceGroqExtension._currentModel = model + JanInferenceGroqExtension.writeDefaultEngineSettings() + // Todo: Check model list with API key + events.emit(ModelEvent.OnModelReady, model) + } + } + + private static async handleModelStop(model: GroqModel) { + if (model.engine !== 'groq') { + return + } + events.emit(ModelEvent.OnModelStopped, model) + } + + private static async handleInferenceStopped( + instance: JanInferenceGroqExtension + ) { + instance.isCancelled = true + instance.controller?.abort() + } + + /** + * Handles a new message request by making an inference request and emitting events. + * Function registered in event manager, should be static to avoid binding issues. + * Pass instance as a reference. + * @param {MessageRequest} data - The data for the new message request. + */ + private static async handleMessageRequest( + data: MessageRequest, + instance: JanInferenceGroqExtension + ) { + if (data.model.engine !== 'groq') { + return + } + + const timestamp = Date.now() + const message: ThreadMessage = { + id: ulid(), + thread_id: data.threadId, + type: data.type, + assistant_id: data.assistantId, + role: ChatCompletionRole.Assistant, + content: [], + status: MessageStatus.Pending, + created: timestamp, + updated: timestamp, + object: 'thread.message', + } + + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } + + instance.isCancelled = false + instance.controller = new AbortController() + + requestInference( + data?.messages ?? [], + this._engineSettings, + { + ...JanInferenceGroqExtension._currentModel, + parameters: data.model.parameters, + }, + instance.controller + ).subscribe({ + next: (content) => { + const messageContent: ThreadContent = { + type: ContentType.Text, + text: { + value: content.trim(), + annotations: [], + }, + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) + }, + complete: async () => { + message.status = message.content.length + ? MessageStatus.Ready + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + }, + error: async (err) => { + if (instance.isCancelled || message.content.length > 0) { + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return + } + const messageContent: ThreadContent = { + type: ContentType.Text, + text: { + value: 'An error occurred. ' + err.message, + annotations: [], + }, + } + message.content = [messageContent] + message.status = MessageStatus.Error + message.error_code = err.code + events.emit(MessageEvent.OnMessageUpdate, message) + }, + }) + } +} diff --git a/extensions/inference-groq-extension/tsconfig.json b/extensions/inference-groq-extension/tsconfig.json new file mode 100644 index 000000000..2477d58ce --- /dev/null +++ b/extensions/inference-groq-extension/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "ES6", + "moduleResolution": "node", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": false, + "skipLibCheck": true, + "rootDir": "./src" + }, + "include": ["./src"] +} diff --git a/extensions/inference-groq-extension/webpack.config.js b/extensions/inference-groq-extension/webpack.config.js new file mode 100644 index 000000000..96110e818 --- /dev/null +++ b/extensions/inference-groq-extension/webpack.config.js @@ -0,0 +1,39 @@ +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') + +module.exports = { + experiments: { outputModule: true }, + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', + module: { + rules: [ + { + test: /\.tsx?$/, + use: 'ts-loader', + exclude: /node_modules/, + }, + ], + }, + plugins: [ + new webpack.DefinePlugin({ + MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), + GROQ_DOMAIN: JSON.stringify('api.groq.com'), + }), + ], + output: { + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format + }, + resolve: { + extensions: ['.ts', '.js'], + fallback: { + path: require.resolve('path-browserify'), + }, + }, + optimization: { + minimize: false, + }, + // Add loaders and other configuration as needed for your project +} diff --git a/models/groq-llama2-70b/model.json b/models/groq-llama2-70b/model.json new file mode 100644 index 000000000..454591379 --- /dev/null +++ b/models/groq-llama2-70b/model.json @@ -0,0 +1,27 @@ +{ + "sources": [ + { + "url": "https://groq.com" + } + ], + "id": "llama2-70b-4096", + "object": "model", + "name": "Groq Llama 2 70b", + "version": "1.0", + "description": "Groq Llama 2 70b with supercharged speed!", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7, + "top_p": 1, + "stop": null, + "stream": true + }, + "metadata": { + "author": "Meta", + "tags": ["General", "Big Context Length"] + }, + "engine": "groq" + } + \ No newline at end of file diff --git a/models/groq-mixtral-8x7b-instruct/model.json b/models/groq-mixtral-8x7b-instruct/model.json new file mode 100644 index 000000000..241e7ae17 --- /dev/null +++ b/models/groq-mixtral-8x7b-instruct/model.json @@ -0,0 +1,27 @@ +{ + "sources": [ + { + "url": "https://groq.com" + } + ], + "id": "mixtral-8x7b-32768", + "object": "model", + "name": "Groq Mixtral 8x7b Instruct", + "version": "1.0", + "description": "Groq Mixtral 8x7b Instruct is Mixtral with supercharged speed!", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7, + "top_p": 1, + "stop": null, + "stream": true + }, + "metadata": { + "author": "Mistral", + "tags": ["General", "Big Context Length"] + }, + "engine": "groq" + } + \ No newline at end of file diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index 70651a4d4..fb51c521d 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -78,7 +78,9 @@ const DropdownListSidebar = ({ (model) => model.engine !== InferenceEngine.openai ) const remoteModel = downloadedModels.filter( - (model) => model.engine === InferenceEngine.openai + (model) => + model.engine === InferenceEngine.openai || + model.engine === InferenceEngine.groq ) const modelOptions = isTabActive === 0 ? localModel : remoteModel diff --git a/web/containers/OpenAiKeyInput/index.tsx b/web/containers/OpenAiKeyInput/index.tsx index 7ef97cf38..86bb2673d 100644 --- a/web/containers/OpenAiKeyInput/index.tsx +++ b/web/containers/OpenAiKeyInput/index.tsx @@ -19,16 +19,49 @@ const OpenAiKeyInput: React.FC = () => { >(undefined) const { readOpenAISettings, saveOpenAISettings } = useEngineSettings() + const [groqSettings, setGroqSettings] = useState< + { api_key: string } | undefined + >(undefined) + const { readGroqSettings, saveGroqSettings } = useEngineSettings() + useEffect(() => { readOpenAISettings().then((settings) => { setOpenAISettings(settings) }) }, [readOpenAISettings]) - if (!selectedModel || selectedModel.engine !== InferenceEngine.openai) { + useEffect(() => { + readGroqSettings().then((settings) => { + setGroqSettings(settings) + }) + }, [readGroqSettings]) + + if ( + !selectedModel || + (selectedModel.engine !== InferenceEngine.openai && + selectedModel.engine !== InferenceEngine.groq) + ) { return null } + const getCurrentApiKey = () => { + if (selectedModel.engine === InferenceEngine.openai) { + return openAISettings?.api_key + } else if (selectedModel.engine === InferenceEngine.groq) { + return groqSettings?.api_key + } + return '' // Default return value + } + + const handleApiKeyChange = (e: React.ChangeEvent) => { + const newApiKey = e.target.value + if (selectedModel.engine === InferenceEngine.openai) { + saveOpenAISettings({ apiKey: newApiKey }) + } else if (selectedModel.engine === InferenceEngine.groq) { + saveGroqSettings({ apiKey: newApiKey }) + } + } + return (
) diff --git a/web/hooks/useEngineSettings.ts b/web/hooks/useEngineSettings.ts index 4a17f91df..734edba36 100644 --- a/web/hooks/useEngineSettings.ts +++ b/web/hooks/useEngineSettings.ts @@ -36,5 +36,43 @@ export const useEngineSettings = () => { settingFilePath ) } - return { readOpenAISettings, saveOpenAISettings } + + const readGroqSettings = useCallback(async () => { + if (!(await fs.existsSync(await joinPath(['file://engines', 'groq.json'])))) + return {} + const settings = await fs.readFileSync( + await joinPath(['file://engines', 'groq.json']), + 'utf-8' + ) + if (settings) { + return typeof settings === 'object' ? settings : JSON.parse(settings) + } + return {} + }, []) + + const saveGroqSettings = async ({ + apiKey, + }: { + apiKey: string | undefined + }) => { + const settings = await readGroqSettings() + const settingFilePath = await joinPath(['file://engines', 'groq.json']) + + settings.api_key = apiKey + + await fs.writeFileSync(settingFilePath, JSON.stringify(settings)) + + // Sec: Don't attach the settings data to the event + events.emit( + AppConfigurationEventName.OnConfigurationUpdate, + settingFilePath + ) + } + + return { + readOpenAISettings, + saveOpenAISettings, + readGroqSettings, + saveGroqSettings, + } } diff --git a/web/screens/Settings/Models/Row.tsx b/web/screens/Settings/Models/Row.tsx index 5ade3bad6..b929c85f9 100644 --- a/web/screens/Settings/Models/Row.tsx +++ b/web/screens/Settings/Models/Row.tsx @@ -49,6 +49,7 @@ export default function RowModel(props: RowModelProps) { const isRemoteModel = props.data.engine === InferenceEngine.openai || + props.data.engine === InferenceEngine.groq || props.data.engine === InferenceEngine.triton_trtllm const onModelActionClick = (modelId: string) => { From a0c5c4be66be35cc66c66109b1484c7017973f06 Mon Sep 17 00:00:00 2001 From: Service Account Date: Mon, 18 Mar 2024 01:04:42 +0000 Subject: [PATCH 03/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4260db9fd..395b2cfcc 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From bb6cd4e96f2cb2e91132e49b576e41dde54ef3c0 Mon Sep 17 00:00:00 2001 From: Service Account Date: Mon, 18 Mar 2024 11:25:18 +0000 Subject: [PATCH 04/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 395b2cfcc..5f45f6edb 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From d52feecf2f51bf7b27413223629ad13ed7b64a3f Mon Sep 17 00:00:00 2001 From: Service Account Date: Mon, 18 Mar 2024 14:19:08 +0000 Subject: [PATCH 05/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5f45f6edb..37c41b05a 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From c27c146d90327fc9115506feb0c4bc21b5f216f3 Mon Sep 17 00:00:00 2001 From: Service Account Date: Mon, 18 Mar 2024 17:15:04 +0000 Subject: [PATCH 06/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 37c41b05a..a4e2391f8 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 93045b488b92b3aa3c01023ac0c90060a1f4ffce Mon Sep 17 00:00:00 2001 From: Service Account Date: Mon, 18 Mar 2024 20:17:51 +0000 Subject: [PATCH 07/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a4e2391f8..014f3661d 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 465e0e47ed1de7a9dd9b2daea00d228e609809ac Mon Sep 17 00:00:00 2001 From: Service Account Date: Tue, 19 Mar 2024 01:19:24 +0000 Subject: [PATCH 08/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 014f3661d..0db57d808 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From d1a17f0986816b699989e097c8788b0c8c59eec7 Mon Sep 17 00:00:00 2001 From: Service Account Date: Tue, 19 Mar 2024 03:44:42 +0000 Subject: [PATCH 09/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0db57d808..31fa4bfc9 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 4a267ff35b0c2aaa1a1986e980c2303e4146dcde Mon Sep 17 00:00:00 2001 From: hiro Date: Tue, 19 Mar 2024 00:15:30 +0700 Subject: [PATCH 10/16] feat: Add SEO header tag with OpenAI GPT-4 --- ...0-2024-bitdefender-false-positive-flag.mdx | 15 +++++ .../developer/01-overview/01-architecture.md | 14 ++++ .../developer/01-overview/02-file-based.md | 14 ++++ .../01-overview/03-user-interface.md | 14 ++++ .../04-install-and-prerequisites.md | 14 ++++ docs/docs/developer/01-overview/README.md | 14 ++++ .../01-your-first-extension.md | 20 +++++- .../guides/error-codes/permission-denied.mdx | 16 ++++- docs/docs/guides/extensions/setup-ext.mdx | 13 ++++ docs/docs/hardware/community.md | 33 ++++++++-- docs/docs/hardware/concepts/gpu-and-vram.md | 14 ++++ .../hardware/recommendations/by-budget.md | 14 ++++ .../hardware/recommendations/by-hardware.md | 14 ++++ .../docs/hardware/recommendations/by-model.md | 14 ++++ .../hardware/recommendations/by-usecase.md | 14 ++++ docs/docs/privacy/privacy.md | 18 ++++++ docs/docs/support/support.md | 18 ++++++ docs/docs/template/QA_script.md | 64 +++++++++++++++++-- 18 files changed, 324 insertions(+), 13 deletions(-) diff --git a/docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx b/docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx index ef418ff97..6c208764d 100644 --- a/docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx +++ b/docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx @@ -5,6 +5,21 @@ slug: /postmortems/january-10-2024-bitdefender-false-positive-flag tags: [Postmortem] --- + + Jan 10, 2024 Incident Postmortem - Bitdefender False Positive Flag on Jan AI Resolved + + + + + + + + + + + + + Following the recent incident related to Jan version 0.4.4 triggering Bitdefender on Windows with Gen:Variant.Tedy.258323 on January 10, 2024, we wanted to provide a comprehensive postmortem and outline the necessary follow-up actions. ## Incident Overview diff --git a/docs/docs/developer/01-overview/01-architecture.md b/docs/docs/developer/01-overview/01-architecture.md index 432b12537..218964bed 100644 --- a/docs/docs/developer/01-overview/01-architecture.md +++ b/docs/docs/developer/01-overview/01-architecture.md @@ -15,6 +15,20 @@ keywords: ] --- + + Jan AI Architecture - Modular and Extensible Framework + + + + + + + + + + + + :::warning This page is still under construction, and should be read as a scratchpad diff --git a/docs/docs/developer/01-overview/02-file-based.md b/docs/docs/developer/01-overview/02-file-based.md index 653eba3f5..2cd8a554c 100644 --- a/docs/docs/developer/01-overview/02-file-based.md +++ b/docs/docs/developer/01-overview/02-file-based.md @@ -15,6 +15,20 @@ keywords: ] --- + + Jan AI File-based Data Persistence Approach + + + + + + + + + + + + :::warning This page is still under construction, and should be read as a scratchpad diff --git a/docs/docs/developer/01-overview/03-user-interface.md b/docs/docs/developer/01-overview/03-user-interface.md index eb6eac89e..fa5a3de79 100644 --- a/docs/docs/developer/01-overview/03-user-interface.md +++ b/docs/docs/developer/01-overview/03-user-interface.md @@ -15,6 +15,20 @@ keywords: ] --- + + Jan AI User Interface - Customizable UI Kit + + + + + + + + + + + + :::warning This page is still under construction, and should be read as a scratchpad diff --git a/docs/docs/developer/01-overview/04-install-and-prerequisites.md b/docs/docs/developer/01-overview/04-install-and-prerequisites.md index 9752f7b72..a3e6ccfc1 100644 --- a/docs/docs/developer/01-overview/04-install-and-prerequisites.md +++ b/docs/docs/developer/01-overview/04-install-and-prerequisites.md @@ -18,6 +18,20 @@ keywords: ] --- + + Jan AI Installation and Setup Guide - Developer Prerequisites + + + + + + + + + + + + ## Requirements ### Hardware Requirements diff --git a/docs/docs/developer/01-overview/README.md b/docs/docs/developer/01-overview/README.md index 7bc3524de..4f094685a 100644 --- a/docs/docs/developer/01-overview/README.md +++ b/docs/docs/developer/01-overview/README.md @@ -15,6 +15,20 @@ keywords: ] --- + + Jan AI Developer Documentation - Building Extensions and SDK Overview + + + + + + + + + + + + The following docs are aimed at developers who want to build extensions on top of the Jan Framework. :::tip diff --git a/docs/docs/developer/04-build-extension/01-your-first-extension.md b/docs/docs/developer/04-build-extension/01-your-first-extension.md index f89f34053..3fa9f5da5 100644 --- a/docs/docs/developer/04-build-extension/01-your-first-extension.md +++ b/docs/docs/developer/04-build-extension/01-your-first-extension.md @@ -17,6 +17,20 @@ keywords: ] --- + + Building Your First Jan AI Extension - Quick Start Guide + + + + + + + + + + + + :::caution This is currently under development. ::: @@ -76,13 +90,13 @@ There are a few things to keep in mind when writing your extension code: In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { core } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return core.invokePluginFunc(MODULE_PATH, 'run', 0) } ``` For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -Now, go ahead and start customizing your extension! Happy coding! \ No newline at end of file +Now, go ahead and start customizing your extension! Happy coding! diff --git a/docs/docs/guides/error-codes/permission-denied.mdx b/docs/docs/guides/error-codes/permission-denied.mdx index 1d41d3b03..a1c3577f9 100644 --- a/docs/docs/guides/error-codes/permission-denied.mdx +++ b/docs/docs/guides/error-codes/permission-denied.mdx @@ -17,7 +17,21 @@ keywords: ] --- -When running Jan, you might encounter the following error message: + + Resolving "Permission Denied" Error in Jan AI + + + + + + + + + + + + +When you run Jan, you may encounter the following error: ``` Uncaught (in promise) Error: Error invoking layout-480796bff433a3a3.js:538 remote method 'installExtension': diff --git a/docs/docs/guides/extensions/setup-ext.mdx b/docs/docs/guides/extensions/setup-ext.mdx index c080283e9..b8e1f757d 100644 --- a/docs/docs/guides/extensions/setup-ext.mdx +++ b/docs/docs/guides/extensions/setup-ext.mdx @@ -16,6 +16,19 @@ keywords: ] --- + + Configuring Extension Settings in Jan AI - User Guide + + + + + + + + + + + The current Jan Desktop Client has some default extensions built on top of this framework to enhance the user experience. In this guide, we will show you the list of default extensions and how to configure extension settings. diff --git a/docs/docs/hardware/community.md b/docs/docs/hardware/community.md index e1825b24b..5ba920d89 100644 --- a/docs/docs/hardware/community.md +++ b/docs/docs/hardware/community.md @@ -1,12 +1,36 @@ --- title: Hardware Examples description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -keywords: [Jan AI, Jan, ChatGPT alternative, local AI, private AI, conversational AI, no-subscription fee, large language model ] +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + ] --- + + Hardware Examples + + + + + + + + + + + + ## Add your own example -Add your own examples to this page by creating a new file in the `docs/docs/hardware/examples` directory. +Add your own examples to this page by creating a new file in the `docs/docs/hardware/examples` directory. ```shell docs @@ -18,9 +42,10 @@ docs // highlight-next-line └── .md ``` + ### File and Title Convention -We use a specific naming convention for the file name. +We use a specific naming convention for the file name. ```shell # Filename @@ -52,4 +77,4 @@ You are allowed to include affiliate links in your example. ## Longer-Term -We will likely build a simple web app to make it easier to add your own examples, sort and retrieve. \ No newline at end of file +We will likely build a simple web app to make it easier to add your own examples, sort and retrieve. diff --git a/docs/docs/hardware/concepts/gpu-and-vram.md b/docs/docs/hardware/concepts/gpu-and-vram.md index 57387e8d2..0a9316a90 100644 --- a/docs/docs/hardware/concepts/gpu-and-vram.md +++ b/docs/docs/hardware/concepts/gpu-and-vram.md @@ -2,6 +2,20 @@ title: GPUs and VRAM --- + + Understanding GPUs and VRAM for AI and Gaming + + + + + + + + + + + + ## What Is a GPU? A Graphics Card, or GPU (Graphics Processing Unit), is a fundamental component in modern computing. Think of it as the powerhouse behind rendering the stunning visuals you see on your screen. Similar to the motherboard in your computer, the graphics card is a printed circuit board. However, it's not just a passive piece of hardware; it's a sophisticated device equipped with essential components like fans, onboard RAM, a dedicated memory controller, BIOS, and various other features. If you want to learn more about GPUs then read here to [Understand the architecture of a GPU.](https://medium.com/codex/understanding-the-architecture-of-a-gpu-d5d2d2e8978b) diff --git a/docs/docs/hardware/recommendations/by-budget.md b/docs/docs/hardware/recommendations/by-budget.md index 9e640fbc9..e556c77d3 100644 --- a/docs/docs/hardware/recommendations/by-budget.md +++ b/docs/docs/hardware/recommendations/by-budget.md @@ -2,6 +2,20 @@ title: Recommended AI Hardware by Budget --- + + Recommended AI Hardware Builds by Budget + + + + + + + + + + + + > :warning: **Warning:** Do your own research before any purchase. Jan is not liable for compatibility, performance or other issues. Products can become outdated quickly. ## Entry-level PC Build at $1000 diff --git a/docs/docs/hardware/recommendations/by-hardware.md b/docs/docs/hardware/recommendations/by-hardware.md index ee80a290c..dcd744d8b 100644 --- a/docs/docs/hardware/recommendations/by-hardware.md +++ b/docs/docs/hardware/recommendations/by-hardware.md @@ -2,6 +2,20 @@ title: Selecting AI Hardware --- + + Selecting AI Hardware + + + + + + + + + + + + When selecting a GPU for LLMs, remember that it's not just about the GPU itself. Consider the synergy with other components in your PC: - **CPU**: To ensure efficient processing, pair your GPU with a powerful CPU. LLMs benefit from fast processors, so having a capable CPU is essential. diff --git a/docs/docs/hardware/recommendations/by-model.md b/docs/docs/hardware/recommendations/by-model.md index 99d1ca8a2..e9fe1c3e8 100644 --- a/docs/docs/hardware/recommendations/by-model.md +++ b/docs/docs/hardware/recommendations/by-model.md @@ -2,6 +2,20 @@ title: Recommended AI Hardware by Model --- + + Recommended AI Hardware by Model + + + + + + + + + + + + ## Codellama 34b ### System Requirements: diff --git a/docs/docs/hardware/recommendations/by-usecase.md b/docs/docs/hardware/recommendations/by-usecase.md index 2ae0cb906..aa7a1bf75 100644 --- a/docs/docs/hardware/recommendations/by-usecase.md +++ b/docs/docs/hardware/recommendations/by-usecase.md @@ -2,6 +2,20 @@ title: Recommended AI Hardware by Use Case --- + + Recommended AI Hardware by Model + + + + + + + + + + + + ## Which AI Hardware to Choose Based on Your Use Case Artificial intelligence (AI) is rapidly changing the world, and AI hardware is becoming increasingly important for businesses and individuals alike. Choosing the right hardware for your AI needs is crucial to get the best performance and results. Here are some tips for selecting AI hardware based on your specific use case and requirements. diff --git a/docs/docs/privacy/privacy.md b/docs/docs/privacy/privacy.md index 56e81f3a1..e1f5d0f10 100644 --- a/docs/docs/privacy/privacy.md +++ b/docs/docs/privacy/privacy.md @@ -1,3 +1,21 @@ +--- +title: Privacy - Jan +--- + + + Privacy Policy - Jan + + + + + + + + + + + + # Privacy Policy Jan is committed to protecting your privacy and ensuring that your personal information is handled in a safe and responsible way. This policy outlines how we collect, store, and use your personal information when you use our mobile application. diff --git a/docs/docs/support/support.md b/docs/docs/support/support.md index 5a1ec2097..856041f86 100644 --- a/docs/docs/support/support.md +++ b/docs/docs/support/support.md @@ -1,3 +1,21 @@ +--- +title: Support - Jan +--- + + + Support - Jan + + + + + + + + + + + + # Support - Bugs & requests: file a GitHub ticket [here](https://github.com/janhq/jan/issues) diff --git a/docs/docs/template/QA_script.md b/docs/docs/template/QA_script.md index 9c7eeaf18..d0ba4b90f 100644 --- a/docs/docs/template/QA_script.md +++ b/docs/docs/template/QA_script.md @@ -1,5 +1,25 @@ +--- +title: Jan version release template +--- + + + Release Version QA Script template + + + + + + + + + + + + # Regression test +# [Release Version] QA Script + **Release Version:** v0.4.7 **Operating System:** MacOS @@ -26,7 +46,6 @@ - [ ] :key::warning: Check that the uninstallation process removes the app successfully from the system. - [ ] Clean the Jan root directory and open the app to check if it creates all the necessary folders, especially models and extensions. - ## B. Overview ### 1. Shortcut key, memory usage / CPU usage @@ -71,10 +90,18 @@ - [ ] :key: Ensure that users switch between threads with different models, the app can handle it. ### 3. Model dropdown + - [ ] :key: Model list should highlight recommended based on user RAM - [ ] Model size should display (for both installed and imported models) ### 4. Users can click on a history thread + +# <<<<<<< HEAD + +- [ ] Test the ability to click on any thread in the history panel. +- [ ] :key: Verify that clicking a thread brings up the past conversation in the main chat window. +- [ ] :key: Ensure that the selected thread is highlighted or otherwise indicated in the history panel. + > > > > > > > f2847d56c (feat: Add SEO header tag with OpenAI GPT-4) - [ ] Confirm that the chat window displays the entire conversation from the selected history thread without any missing messages. - [ ] :key: Check the performance and accuracy of the history feature when dealing with a large number of threads. - [ ] Validate that historical threads reflect the exact state of the chat at that time, including settings. @@ -82,12 +109,16 @@ - [ ] Confirm that changing the title of the thread updates correctly. ### 5. Users can config instructions for the assistant. + +# <<<<<<< HEAD + +- [ ] Ensure there is a clear interface to input or change instructions for the assistant. + > > > > > > > f2847d56c (feat: Add SEO header tag with OpenAI GPT-4) - [ ] Test if the instructions set by the user are being followed by the assistant in subsequent conversations. - [ ] :key: Validate that changes to instructions are updated in real time and do not require a restart of the application or session. - [ ] :key: Check for the ability to reset instructions to default or clear them completely. - [ ] :key: RAG - Users can import documents and the system should process queries about the uploaded file, providing accurate and appropriate responses in the conversation thread. - ## D. Hub ### 1. Users can discover recommended models (Jan ships with a few preconfigured model.json files) @@ -117,13 +148,14 @@ ### 5. Users can use the model as they want -- [ ] :key: Check `start` / `stop` / `delete` button response exactly what it does. +- [ ] :key: Check `start` / `stop` / `delete` button response exactly what it does. - [ ] Check if starting another model stops the other model entirely. - [x] :rocket: Check the `Explore models` navigate correctly to the model panel. - [ ] :key: Check when deleting a model it will delete all the files on the user's computer. - [ ] :warning:The recommended tags should present right for the user's hardware. ### 6. Users can Integrate With a Remote Server + - [ ] :key: Import openAI GPT model https://jan.ai/guides/using-models/integrate-with-remote-server/ and the model displayed in Hub / Thread dropdown - [ ] Users can use the remote model properly @@ -175,6 +207,27 @@ - [ ] :key: Test the `Experimental Mode` toggle to confirm it enables or disables experimental features as intended. - [ ] :key: Check the functionality of `Open App Directory` to ensure it opens the correct folder in the system file explorer. + <<<<<<< HEAD + ======= +- [ ] Validate that changes in advanced settings are applied immediately or provide appropriate instructions if a restart is needed. +- [ ] Test the application's stability when experimental features are enabled. + +### 4. Users can add custom plugins via manual installation [TBU] + +- [ ] Verify that the `Manual Installation` option is clearly visible and accessible in the `Extensions` section. +- [ ] Test the functionality of the `Select` button within the `Manual Installation` area. +- [ ] :warning: Check that the file picker dialog allows for the correct plugin file types (e.g., .tgz). +- [ ] :key: Validate that the selected plugin file installs correctly and the plugin becomes functional. +- [ ] Ensure that there is a progress indicator or confirmation message once the installation is complete. +- [ ] Confirm that if the installation is interrupted or fails, the user is given a clear error message. +- [ ] :key: Test that the application prevents the installation of incompatible or corrupt plugin files. +- [ ] :key: Check that the user can uninstall or disable custom plugins as easily as pre-installed ones. +- [ ] Verify that the application's performance remains stable after the installation of custom plugins. + +### 5. Advanced Settings + +- [ ] Attemp to test downloading model from hub using **HTTP Proxy** [guideline](https://github.com/janhq/jan/pull/1562) + > > > > > > > f2847d56c (feat: Add SEO header tag with OpenAI GPT-4) - [ ] Users can move **Jan data folder** - [ ] Validate that changes in advanced settings are applied immediately or provide appropriate instructions if a restart is needed. - [ ] Attemp to test downloading model from hub using **HTTP Proxy** [guideline](https://github.com/janhq/jan/pull/1562) @@ -184,9 +237,10 @@ ## G. Local API server ### 1. Local Server Usage with Server Options + - [ ] :key: Explore API Reference: Swagger API for sending/receiving requests - - [ ] Use default server option - - [ ] Configure and use custom server options + - [ ] Use default server option + - [ ] Configure and use custom server options - [ ] Test starting/stopping the local API server with different Model/Model settings - [ ] Server logs captured with correct Server Options provided - [ ] Verify functionality of Open logs/Clear feature From f8d970401a97da23620b6a6e783cf10e474c1f4b Mon Sep 17 00:00:00 2001 From: hiro Date: Tue, 19 Mar 2024 10:19:55 +0700 Subject: [PATCH 11/16] chore: Update head metadata --- README.md | 1 + docs/docs/about/2035.mdx | 14 ++++++++++++ docs/docs/about/about.md | 14 ++++++++++++ docs/docs/about/faq.md | 18 +++++++++++++++ docs/docs/about/roadmap.md | 16 +++++++++++++- docs/docs/acknowledgements.md | 13 +++++++++++ docs/docs/community/community.mdx | 14 ++++++++++++ .../01-your-first-assistant.md | 15 ++++++++++++- .../05-framework/03-engineering/assistants.md | 15 ++++++++++++- .../05-framework/03-engineering/chats.md | 13 +++++++++++ .../05-framework/03-engineering/engine.md | 13 +++++++++++ .../05-framework/03-engineering/files.md | 15 ++++++++++++- .../05-framework/03-engineering/messages.md | 13 +++++++++++ .../05-framework/03-engineering/models.md | 13 +++++++++++ .../05-framework/03-engineering/threads.md | 13 +++++++++++ .../developer/05-framework/03-product/chat.md | 13 +++++++++++ .../developer/05-framework/03-product/hub.md | 13 +++++++++++ .../developer/05-framework/03-product/jan.md | 13 +++++++++++ .../05-framework/03-product/settings.md | 13 +++++++++++ .../05-framework/03-product/system-monitor.md | 13 +++++++++++ docs/docs/events/hcmc-oct23.md | 15 ++++++++++++- docs/docs/events/nvidia-llm-day-nov-23.md | 22 ++++++++++++++----- .../advanced-settings/advanced-settings.mdx | 14 ++++++++++++ .../guides/advanced-settings/http-proxy.mdx | 13 +++++++++++ docs/docs/guides/best-practices.mdx | 12 ++++++++++ .../docs/guides/common-error/broken-build.mdx | 14 ++++++++++++ .../guides/common-error/not-using-gpu.mdx | 14 ++++++++++++ .../error-codes/how-to-get-error-logs.mdx | 14 ++++++++++++ .../error-codes/no-assistant-available.mdx | 14 ++++++++++++ .../guides/error-codes/something-amiss.mdx | 12 ++++++++++ .../error-codes/stuck-on-loading-model.mdx | 12 ++++++++++ .../error-codes/thread-disappreance.mdx | 12 ++++++++++ .../guides/error-codes/undefined-issue.mdx | 12 ++++++++++ .../guides/error-codes/unexpected-token.mdx | 12 ++++++++++ docs/docs/guides/faq.mdx | 12 ++++++++++ docs/docs/guides/install.mdx | 12 ++++++++++ docs/docs/guides/integration/azure.mdx | 12 ++++++++++ docs/docs/guides/integration/discord.mdx | 12 ++++++++++ docs/docs/guides/integration/groq.mdx | 12 ++++++++++ docs/docs/guides/integration/lmstudio.mdx | 12 ++++++++++ docs/docs/guides/integration/mistral.mdx | 12 ++++++++++ docs/docs/guides/integration/ollama.mdx | 12 ++++++++++ .../guides/integration/openinterpreter.mdx | 11 ++++++++++ docs/docs/guides/integration/openrouter.mdx | 13 ++++++++++- docs/docs/guides/integration/raycast.mdx | 11 ++++++++++ docs/docs/guides/integration/vscode.mdx | 12 ++++++++++ docs/docs/guides/models-list.mdx | 12 ++++++++++ docs/docs/guides/models/customize-engine.mdx | 12 ++++++++++ docs/docs/guides/models/import-models.mdx | 12 ++++++++++ docs/docs/guides/models/integrate-remote.mdx | 12 ++++++++++ docs/docs/guides/providers/llama-cpp.md | 14 +++++++++++- docs/docs/guides/providers/tensorrt-llm.md | 19 +++++++++++++--- docs/docs/guides/quickstart.mdx | 12 ++++++++++ docs/docs/guides/start-server.mdx | 11 ++++++++++ docs/docs/guides/thread.mdx | 11 ++++++++++ .../overview/cloud-vs-self-hosting.md | 12 ++++++++++ docs/docs/hardware/overview/cpu-vs-gpu.md | 14 ++++++++++++ docs/docs/how-we-work.md | 12 ++++++++++ docs/docs/how-we-work/analytics/analytics.md | 12 ++++++++++ docs/docs/how-we-work/engineering/qa.mdx | 12 ++++++++++ .../project-management/project-management.md | 13 ++++++++++- docs/docs/how-we-work/strategy/strategy.md | 14 +++++++++++- .../how-we-work/website-docs/website-docs.md | 12 ++++++++++ docs/docs/platforms/desktop.md | 12 ++++++++++ docs/docs/server-suite/enterprise.md | 12 ++++++++++ docs/docs/server-suite/home-server.md | 12 ++++++++++ docs/docs/team/team.md | 15 +++++++++++-- docs/docs/wall-of-love.md | 12 ++++++++++ 68 files changed, 860 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 31fa4bfc9..9e9d83fc9 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,7 @@ Jan builds on top of other open-source projects: - [llama.cpp](https://github.com/ggerganov/llama.cpp) - [LangChain](https://github.com/langchain-ai) - [TensorRT](https://github.com/NVIDIA/TensorRT) +- [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) ## Contact diff --git a/docs/docs/about/2035.mdx b/docs/docs/about/2035.mdx index 3af7a3197..2806e7ade 100644 --- a/docs/docs/about/2035.mdx +++ b/docs/docs/about/2035.mdx @@ -2,6 +2,20 @@ title: Jan's Vision for 2035 --- + + Jan's Vision for 2035 + + + + + + + + + + + + [Jan 2035: A Robotics Company](https://hackmd.io/QIWyYbNNQVWVbupuI3kjAA) We only have 2 planning parameters: diff --git a/docs/docs/about/about.md b/docs/docs/about/about.md index a047ab910..32f4a3e4f 100644 --- a/docs/docs/about/about.md +++ b/docs/docs/about/about.md @@ -18,6 +18,20 @@ keywords: ] --- + + About Jan + + + + + + + + + + + + Jan turns computers into thinking machines to change how we use them. Jan is created and maintained by Jan Labs, a robotics company. diff --git a/docs/docs/about/faq.md b/docs/docs/about/faq.md index 29832e211..b4e05b3a6 100644 --- a/docs/docs/about/faq.md +++ b/docs/docs/about/faq.md @@ -1,3 +1,21 @@ +--- +title: Frequently Asked Questions (FAQ) - Jan +--- + + + Frequently Asked Questions (FAQ) + + + + + + + + + + + + # Frequently Asked Questions (FAQ) ## What is Jan? diff --git a/docs/docs/about/roadmap.md b/docs/docs/about/roadmap.md index 1c789d733..d818aa647 100644 --- a/docs/docs/about/roadmap.md +++ b/docs/docs/about/roadmap.md @@ -2,5 +2,19 @@ title: Roadmap --- + + Roadmap + + + + + + + + + + + + - [ ] [Immediate Roadmap on Github](https://github.com/orgs/janhq/projects/5/views/16) -- [ ] [Longer-term Roadmap on Discord](https://discord.gg/Ey62mynnYr) \ No newline at end of file +- [ ] [Longer-term Roadmap on Discord](https://discord.gg/Ey62mynnYr) diff --git a/docs/docs/acknowledgements.md b/docs/docs/acknowledgements.md index c68c4ed86..a7f77218e 100644 --- a/docs/docs/acknowledgements.md +++ b/docs/docs/acknowledgements.md @@ -17,6 +17,18 @@ keywords: ] --- + + Acknowledgements - Jan + + + + + + + + + + # Acknowledgements We would like to express our gratitude to the following third-party libraries that have made the development of Jan possible. @@ -24,3 +36,4 @@ We would like to express our gratitude to the following third-party libraries th - [llama.cpp](https://github.com/ggerganov/llama.cpp/blob/master/LICENSE) - [LangChain.js](https://github.com/langchain-ai/langchainjs/blob/main/LICENSE) - [TensorRT](https://github.com/NVIDIA/TensorRT/blob/main/LICENSE) +- [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE) diff --git a/docs/docs/community/community.mdx b/docs/docs/community/community.mdx index d4866490e..7c5ad9367 100644 --- a/docs/docs/community/community.mdx +++ b/docs/docs/community/community.mdx @@ -15,6 +15,20 @@ keywords: ] --- + + Jan's Community + + + + + + + + + + + + ## Socials - [Discord](https://discord.gg/SH3DGmUs6b) diff --git a/docs/docs/developer/02-build-assistant/01-your-first-assistant.md b/docs/docs/developer/02-build-assistant/01-your-first-assistant.md index 16b80fc5e..863d275fe 100644 --- a/docs/docs/developer/02-build-assistant/01-your-first-assistant.md +++ b/docs/docs/developer/02-build-assistant/01-your-first-assistant.md @@ -17,7 +17,20 @@ keywords: ] --- + + Your First Assistant + + + + + + + + + + + + :::caution This is currently under development. ::: - diff --git a/docs/docs/developer/05-framework/03-engineering/assistants.md b/docs/docs/developer/05-framework/03-engineering/assistants.md index fa9c593ab..90b52ab38 100644 --- a/docs/docs/developer/05-framework/03-engineering/assistants.md +++ b/docs/docs/developer/05-framework/03-engineering/assistants.md @@ -1,5 +1,5 @@ --- -title: "Assistants" +title: 'Assistants' description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. keywords: [ @@ -14,6 +14,19 @@ keywords: ] --- + + Assistants + + + + + + + + + + + :::caution This is currently under development. diff --git a/docs/docs/developer/05-framework/03-engineering/chats.md b/docs/docs/developer/05-framework/03-engineering/chats.md index eb0ae287a..654621e30 100644 --- a/docs/docs/developer/05-framework/03-engineering/chats.md +++ b/docs/docs/developer/05-framework/03-engineering/chats.md @@ -14,6 +14,19 @@ keywords: ] --- + + Chats + + + + + + + + + + + :::caution This is currently under development. diff --git a/docs/docs/developer/05-framework/03-engineering/engine.md b/docs/docs/developer/05-framework/03-engineering/engine.md index 653576f1b..8ebfff88d 100644 --- a/docs/docs/developer/05-framework/03-engineering/engine.md +++ b/docs/docs/developer/05-framework/03-engineering/engine.md @@ -2,6 +2,19 @@ title: Engine --- + + Engine + + + + + + + + + + + :::caution Currently Under Development diff --git a/docs/docs/developer/05-framework/03-engineering/files.md b/docs/docs/developer/05-framework/03-engineering/files.md index 59ca27ec9..9f572af11 100644 --- a/docs/docs/developer/05-framework/03-engineering/files.md +++ b/docs/docs/developer/05-framework/03-engineering/files.md @@ -1,5 +1,5 @@ --- -title: "Files" +title: 'Files' description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. keywords: [ @@ -14,6 +14,19 @@ keywords: ] --- + + Files + + + + + + + + + + + :::warning Draft Specification: functionality has not been implemented yet. diff --git a/docs/docs/developer/05-framework/03-engineering/messages.md b/docs/docs/developer/05-framework/03-engineering/messages.md index 8f2497002..6ddaba45d 100644 --- a/docs/docs/developer/05-framework/03-engineering/messages.md +++ b/docs/docs/developer/05-framework/03-engineering/messages.md @@ -14,6 +14,19 @@ keywords: ] --- + + Messages + + + + + + + + + + + :::caution This is currently under development. diff --git a/docs/docs/developer/05-framework/03-engineering/models.md b/docs/docs/developer/05-framework/03-engineering/models.md index 4e4c3c604..dbe134f07 100644 --- a/docs/docs/developer/05-framework/03-engineering/models.md +++ b/docs/docs/developer/05-framework/03-engineering/models.md @@ -14,6 +14,19 @@ keywords: ] --- + + Models + + + + + + + + + + + :::caution This is currently under development. diff --git a/docs/docs/developer/05-framework/03-engineering/threads.md b/docs/docs/developer/05-framework/03-engineering/threads.md index a1cd2b4df..f8ba018f8 100644 --- a/docs/docs/developer/05-framework/03-engineering/threads.md +++ b/docs/docs/developer/05-framework/03-engineering/threads.md @@ -14,6 +14,19 @@ keywords: ] --- + + Threads + + + + + + + + + + + :::caution This is currently under development. diff --git a/docs/docs/developer/05-framework/03-product/chat.md b/docs/docs/developer/05-framework/03-product/chat.md index b0dcce2d6..3b98485b8 100644 --- a/docs/docs/developer/05-framework/03-product/chat.md +++ b/docs/docs/developer/05-framework/03-product/chat.md @@ -14,6 +14,19 @@ keywords: ] --- + + Chat + + + + + + + + + + + ## Overview A home screen for users to chat with [assistants](/docs/engineering/assistants) via conversation [threads](/docs/engineering/threads). diff --git a/docs/docs/developer/05-framework/03-product/hub.md b/docs/docs/developer/05-framework/03-product/hub.md index 7171f8378..ea8dd81a5 100644 --- a/docs/docs/developer/05-framework/03-product/hub.md +++ b/docs/docs/developer/05-framework/03-product/hub.md @@ -14,6 +14,19 @@ keywords: ] --- + + Hub + + + + + + + + + + + ## Overview The Hub is like a store for everything, where users can discover and download models, assistants, and more. diff --git a/docs/docs/developer/05-framework/03-product/jan.md b/docs/docs/developer/05-framework/03-product/jan.md index 9e8973360..b906be09d 100644 --- a/docs/docs/developer/05-framework/03-product/jan.md +++ b/docs/docs/developer/05-framework/03-product/jan.md @@ -14,6 +14,19 @@ keywords: ] --- + + Jan (The Default Assistant) + + + + + + + + + + + Jan ships with a default assistant "Jan" that lets users chat with any open source model out-of-the-box. This assistant is defined in `/jan`. It is a generic assistant to illustrate power of Jan. In the future, it will support additional features e.g. multi-assistant conversations diff --git a/docs/docs/developer/05-framework/03-product/settings.md b/docs/docs/developer/05-framework/03-product/settings.md index 514139a00..515b5e802 100644 --- a/docs/docs/developer/05-framework/03-product/settings.md +++ b/docs/docs/developer/05-framework/03-product/settings.md @@ -14,6 +14,19 @@ keywords: ] --- + + Settings + + + + + + + + + + + ## Overview A settings page for users to add extensions, configure model settings, change app appearance, add keyboard shortcuts, and a plethora of other personalizations. diff --git a/docs/docs/developer/05-framework/03-product/system-monitor.md b/docs/docs/developer/05-framework/03-product/system-monitor.md index 761d9a7bf..15dae09ea 100644 --- a/docs/docs/developer/05-framework/03-product/system-monitor.md +++ b/docs/docs/developer/05-framework/03-product/system-monitor.md @@ -14,6 +14,19 @@ keywords: ] --- + + System Monitor + + + + + + + + + + + ## Overview An activity screen to monitor system health and running models. diff --git a/docs/docs/events/hcmc-oct23.md b/docs/docs/events/hcmc-oct23.md index 73898efcd..e70329b2d 100644 --- a/docs/docs/events/hcmc-oct23.md +++ b/docs/docs/events/hcmc-oct23.md @@ -1,10 +1,23 @@ --- title: "Jan's AI Hacker House (Ho Chi Minh City)" -description: "24-27 Oct 2023, District 3, HCMC. AI-focused talks, workshops and social events. Hosted by Jan.ai" +description: '24-27 Oct 2023, District 3, HCMC. AI-focused talks, workshops and social events. Hosted by Jan.ai' slug: /events/hcmc-oct23 image: /img/hcmc-launch-party.png --- + + Jan's AI Hacker House (Ho Chi Minh City) + + + + + + + + + + + ![](/img/hcmc-launch-party.png) 🎉 Join us at our Friday Launch Party for an evening of AI talks from other builders! [(RSVP here)](https://jan-launch-party.eventbrite.sg/) 🎉 diff --git a/docs/docs/events/nvidia-llm-day-nov-23.md b/docs/docs/events/nvidia-llm-day-nov-23.md index d467dcb6e..f739fb4ff 100644 --- a/docs/docs/events/nvidia-llm-day-nov-23.md +++ b/docs/docs/events/nvidia-llm-day-nov-23.md @@ -1,21 +1,33 @@ --- -title: "Nov 23: Nvidia GenAI Day" -description: Nvidia's LLM Day +title: 'Nov 23: Nvidia GenAI Day' +description: Nvidia's LLM Day --- + + Nov 23: Nvidia GenAI Day + + + + + + + + + + + ![](/img/nvidia-llm-day-header.png) ## Nvidia GenAI Innovation Day -Jan will be at Nvidia's GenAI Innovation Day in Nov '23, focusing on Enterprise use-cases of LLMs. +Jan will be at Nvidia's GenAI Innovation Day in Nov '23, focusing on Enterprise use-cases of LLMs. ### Location -- JW Marriott Hanoi Hotel +- JW Marriott Hanoi Hotel - 8:30am November 8th 2023 - Registration: [https://gmcgroup.com.vn/nvidia-genai-event/](https://gmcgroup.com.vn/nvidia-genai-event/) ### Programme ![](/img/nvidia-llm-day.png) - diff --git a/docs/docs/guides/advanced-settings/advanced-settings.mdx b/docs/docs/guides/advanced-settings/advanced-settings.mdx index ae3244cda..1ce65867a 100644 --- a/docs/docs/guides/advanced-settings/advanced-settings.mdx +++ b/docs/docs/guides/advanced-settings/advanced-settings.mdx @@ -15,6 +15,20 @@ keywords: ] --- + + Advanced Settings + + + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/docs/guides/advanced-settings/http-proxy.mdx b/docs/docs/guides/advanced-settings/http-proxy.mdx index 7b2de339c..297133aae 100644 --- a/docs/docs/guides/advanced-settings/http-proxy.mdx +++ b/docs/docs/guides/advanced-settings/http-proxy.mdx @@ -16,6 +16,19 @@ keywords: ] --- + + HTTPS Proxy + + + + + + + + + + + ## Why HTTPS Proxy? diff --git a/docs/docs/guides/best-practices.mdx b/docs/docs/guides/best-practices.mdx index 9dabef8dc..d6e8c4584 100644 --- a/docs/docs/guides/best-practices.mdx +++ b/docs/docs/guides/best-practices.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Best Practices - Jan Guides + + + + + + + + + + Jan is a versatile platform offering solutions for integrating AI locally across various platforms. This guide outlines best practices for developers, analysts, and AI enthusiasts to enhance their experience with Jan when adding AI locally to their computers. Implementing these practices will optimize the performance of AI models. ## Follow the Quickstart Guide diff --git a/docs/docs/guides/common-error/broken-build.mdx b/docs/docs/guides/common-error/broken-build.mdx index 2b3e7a128..9a8809955 100644 --- a/docs/docs/guides/common-error/broken-build.mdx +++ b/docs/docs/guides/common-error/broken-build.mdx @@ -17,6 +17,20 @@ keywords: ] --- + + Broken Build + + + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/docs/guides/common-error/not-using-gpu.mdx b/docs/docs/guides/common-error/not-using-gpu.mdx index a7dd788f8..b932a6822 100644 --- a/docs/docs/guides/common-error/not-using-gpu.mdx +++ b/docs/docs/guides/common-error/not-using-gpu.mdx @@ -17,6 +17,20 @@ keywords: [ ] --- + + Troubleshooting NVIDIA GPU + + + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/docs/guides/error-codes/how-to-get-error-logs.mdx b/docs/docs/guides/error-codes/how-to-get-error-logs.mdx index 045468e33..74a7508c0 100644 --- a/docs/docs/guides/error-codes/how-to-get-error-logs.mdx +++ b/docs/docs/guides/error-codes/how-to-get-error-logs.mdx @@ -17,6 +17,20 @@ keywords: ] --- + + How to Get Error Logs + + + + + + + + + + + + To get the error logs of your Jan application, follow the steps below: ### Jan Application 1. Navigate to the main dashboard. diff --git a/docs/docs/guides/error-codes/no-assistant-available.mdx b/docs/docs/guides/error-codes/no-assistant-available.mdx index 31d9a75e9..7716dc5b0 100644 --- a/docs/docs/guides/error-codes/no-assistant-available.mdx +++ b/docs/docs/guides/error-codes/no-assistant-available.mdx @@ -17,6 +17,20 @@ keywords: ] --- + + No Assistant Available + + + + + + + + + + + + When you encounter the following error message: ``` No assistant available. diff --git a/docs/docs/guides/error-codes/something-amiss.mdx b/docs/docs/guides/error-codes/something-amiss.mdx index 0975754e3..63f918441 100644 --- a/docs/docs/guides/error-codes/something-amiss.mdx +++ b/docs/docs/guides/error-codes/something-amiss.mdx @@ -4,6 +4,18 @@ sidebar_position: 4 description: A step-by-step guide to resolve an unspecified or general error. --- + + Something's Amiss + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/docs/guides/error-codes/stuck-on-loading-model.mdx b/docs/docs/guides/error-codes/stuck-on-loading-model.mdx index 86a16b5fc..3c5adf035 100644 --- a/docs/docs/guides/error-codes/stuck-on-loading-model.mdx +++ b/docs/docs/guides/error-codes/stuck-on-loading-model.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Stuck on Loading Model + + + + + + + + + + ## 1. Issue: Model Loading Stuck Due To Missing Windows Management Instrumentation Command-line (WMIC) Encountering a stuck-on-loading model issue in Jan is caused by errors related to the `Windows Management Instrumentation Command-line (WMIC)` path not being included in the system's PATH environment variable. diff --git a/docs/docs/guides/error-codes/thread-disappreance.mdx b/docs/docs/guides/error-codes/thread-disappreance.mdx index 06235df56..120e8585c 100644 --- a/docs/docs/guides/error-codes/thread-disappreance.mdx +++ b/docs/docs/guides/error-codes/thread-disappreance.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Thread Disappearance + + + + + + + + + + When you encounter the error of old threads suddenly disappear. This can happen when a new, unintentional file is created in `/jan/threads`. It can be resolved through the following steps: diff --git a/docs/docs/guides/error-codes/undefined-issue.mdx b/docs/docs/guides/error-codes/undefined-issue.mdx index 223f686d1..9373044a4 100644 --- a/docs/docs/guides/error-codes/undefined-issue.mdx +++ b/docs/docs/guides/error-codes/undefined-issue.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Undefined Issue + + + + + + + + + + Encountering an `undefined issue` in Jan is caused by errors related to the Nitro tool or other internal processes. It can be resolved through the following steps: 1. Clearing the Jan folder and then reopen the application to determine if the problem persists diff --git a/docs/docs/guides/error-codes/unexpected-token.mdx b/docs/docs/guides/error-codes/unexpected-token.mdx index 4a00e447d..252cd97fa 100644 --- a/docs/docs/guides/error-codes/unexpected-token.mdx +++ b/docs/docs/guides/error-codes/unexpected-token.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Unexpected Token + + + + + + + + + + Encountering the `Unexpected token` error when initiating a chat with OpenAI models mainly caused by either your OpenAI key or where you access your OpenAI from. This issue can be solved through the following steps: 1. Obtain an OpenAI API key from [OpenAI's developer platform](https://platform.openai.com/) and integrate it into your application. diff --git a/docs/docs/guides/faq.mdx b/docs/docs/guides/faq.mdx index 7e3d7d13d..862d43620 100644 --- a/docs/docs/guides/faq.mdx +++ b/docs/docs/guides/faq.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + FAQs - Jan Guides + + + + + + + + + + ## General Issues - **Why can't I download models like Pandora 11B Q4 and Solar Instruct 10.7B Q4?** diff --git a/docs/docs/guides/install.mdx b/docs/docs/guides/install.mdx index c8dcbf3c3..11fc96685 100644 --- a/docs/docs/guides/install.mdx +++ b/docs/docs/guides/install.mdx @@ -16,6 +16,18 @@ keywords: ] --- + + Installation - Jan Guides + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import installImageURL from './assets/jan-ai-download.png'; diff --git a/docs/docs/guides/integration/azure.mdx b/docs/docs/guides/integration/azure.mdx index 6c344a199..24f457d28 100644 --- a/docs/docs/guides/integration/azure.mdx +++ b/docs/docs/guides/integration/azure.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Azure OpenAI + + + + + + + + + + ## How to Integrate Azure OpenAI with Jan The [Azure OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview?source=docs) offers robust APIs, making it simple for you to incorporate OpenAI's language models into your applications. You can integrate Azure OpenAI with Jan by following the steps below: diff --git a/docs/docs/guides/integration/discord.mdx b/docs/docs/guides/integration/discord.mdx index 5cf846883..7b91cc972 100644 --- a/docs/docs/guides/integration/discord.mdx +++ b/docs/docs/guides/integration/discord.mdx @@ -4,6 +4,18 @@ sidebar_position: 5 description: A step-by-step guide on how to integrate Jan with a Discord bot. --- + + Discord + + + + + + + + + + ## How to Integrate Discord Bot with Jan Discord bot can enhances your discord server interactions. By integrating Jan with it, you can significantly boost responsiveness and user engaggement in your discord server. diff --git a/docs/docs/guides/integration/groq.mdx b/docs/docs/guides/integration/groq.mdx index a57bf16dd..2edc1caa5 100644 --- a/docs/docs/guides/integration/groq.mdx +++ b/docs/docs/guides/integration/groq.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Groq + + + + + + + + + + ## How to Integrate Mistral AI with Jan This guide provides step-by-step instructions on integrating the Groq API with Jan, enabling users to leverage Groq's capabilities within Jan's conversational interface. diff --git a/docs/docs/guides/integration/lmstudio.mdx b/docs/docs/guides/integration/lmstudio.mdx index 33e48f33a..9b018c2a7 100644 --- a/docs/docs/guides/integration/lmstudio.mdx +++ b/docs/docs/guides/integration/lmstudio.mdx @@ -16,6 +16,18 @@ keywords: ] --- + + LM Studio + + + + + + + + + + ## How to Integrate LM Studio with Jan [LM Studio](https://lmstudio.ai/) enables you to explore, download, and run local Large Language Models (LLMs). You can integrate Jan with LM Studio using two methods: diff --git a/docs/docs/guides/integration/mistral.mdx b/docs/docs/guides/integration/mistral.mdx index a44e23205..1772fefd9 100644 --- a/docs/docs/guides/integration/mistral.mdx +++ b/docs/docs/guides/integration/mistral.mdx @@ -16,6 +16,18 @@ keywords: ] --- + + Mistral AI + + + + + + + + + + ## How to Integrate Mistral AI with Jan [Mistral AI](https://docs.mistral.ai/) provides two ways to use their Large Language Models (LLM): diff --git a/docs/docs/guides/integration/ollama.mdx b/docs/docs/guides/integration/ollama.mdx index 6c55bc856..e50431f79 100644 --- a/docs/docs/guides/integration/ollama.mdx +++ b/docs/docs/guides/integration/ollama.mdx @@ -16,6 +16,18 @@ keywords: ] --- + + Ollama + + + + + + + + + + ## How to Integrate Ollama with Jan Ollama provides you with largen language that you can run locally. There are two methods to integrate Ollama with Jan: diff --git a/docs/docs/guides/integration/openinterpreter.mdx b/docs/docs/guides/integration/openinterpreter.mdx index a844155f5..ff706a3bc 100644 --- a/docs/docs/guides/integration/openinterpreter.mdx +++ b/docs/docs/guides/integration/openinterpreter.mdx @@ -4,6 +4,17 @@ sidebar_position: 6 description: A step-by-step guide on how to integrate Jan with Open Interpreter. --- + + Open Interpreter + + + + + + + + + ## How to Integrate Open Interpreter with Jan diff --git a/docs/docs/guides/integration/openrouter.mdx b/docs/docs/guides/integration/openrouter.mdx index 2189db0d9..89e90c6d5 100644 --- a/docs/docs/guides/integration/openrouter.mdx +++ b/docs/docs/guides/integration/openrouter.mdx @@ -4,6 +4,17 @@ sidebar_position: 2 description: A step-by-step guide on how to integrate Jan with OpenRouter. --- + + OpenRouter + + + + + + + + + ## How to Integrate OpenRouter with Jan @@ -16,7 +27,7 @@ To connect Jan with OpenRouter for accessing remote Large Language Models (LLMs) 1. Find your API keys in the [OpenRouter API Key](https://openrouter.ai/keys). 2. Set the OpenRouter API key in `~/jan/engines/openai.json` file. -### Step 2: MModel Configuration +### Step 2: Model Configuration 1. Go to the directory `~/jan/models`. 2. Make a new folder called `openrouter-(modelname)`, like `openrouter-dolphin-mixtral-8x7b`. diff --git a/docs/docs/guides/integration/raycast.mdx b/docs/docs/guides/integration/raycast.mdx index a626b0061..c9cb092b7 100644 --- a/docs/docs/guides/integration/raycast.mdx +++ b/docs/docs/guides/integration/raycast.mdx @@ -4,6 +4,17 @@ sidebar_position: 4 description: A step-by-step guide on how to integrate Jan with Raycast. --- + + Raycast + + + + + + + + + ## How to Integrate Raycast [Raycast](https://www.raycast.com/) is a productivity tool designed for macOS that enhances workflow efficiency by providing quick access to various tasks and functionalities through a keyboard-driven interface. To integrate Raycast with Jan, follow the steps below: diff --git a/docs/docs/guides/integration/vscode.mdx b/docs/docs/guides/integration/vscode.mdx index 0bc112186..e40f91a78 100644 --- a/docs/docs/guides/integration/vscode.mdx +++ b/docs/docs/guides/integration/vscode.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Continue + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/docs/guides/models-list.mdx b/docs/docs/guides/models-list.mdx index cd7107a92..149d73733 100644 --- a/docs/docs/guides/models-list.mdx +++ b/docs/docs/guides/models-list.mdx @@ -3,6 +3,18 @@ title: Pre-configured Models sidebar_position: 3 --- + + Pre-configured Models - Jan Guides + + + + + + + + + + ## Overview Jan provides various pre-configured AI models with different capabilities. Please see the following list for details. diff --git a/docs/docs/guides/models/customize-engine.mdx b/docs/docs/guides/models/customize-engine.mdx index 2f54204a8..440e02243 100644 --- a/docs/docs/guides/models/customize-engine.mdx +++ b/docs/docs/guides/models/customize-engine.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Customize Engine Settings + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/docs/guides/models/import-models.mdx b/docs/docs/guides/models/import-models.mdx index 9ed8953c7..a1f333db8 100644 --- a/docs/docs/guides/models/import-models.mdx +++ b/docs/docs/guides/models/import-models.mdx @@ -17,6 +17,18 @@ keywords: ] --- + + Manual Import + + + + + + + + + + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import janModel from './assets/jan-model-hub.png'; diff --git a/docs/docs/guides/models/integrate-remote.mdx b/docs/docs/guides/models/integrate-remote.mdx index af881f999..c99d7b98f 100644 --- a/docs/docs/guides/models/integrate-remote.mdx +++ b/docs/docs/guides/models/integrate-remote.mdx @@ -18,6 +18,18 @@ keywords: ] --- + + Remote Server Integration + + + + + + + + + + This guide will show you how to configure Jan as a client and point it to any remote & local (self-hosted) API server. ## OpenAI Platform Configuration diff --git a/docs/docs/guides/providers/llama-cpp.md b/docs/docs/guides/providers/llama-cpp.md index d2b0daa2a..8d9db5398 100644 --- a/docs/docs/guides/providers/llama-cpp.md +++ b/docs/docs/guides/providers/llama-cpp.md @@ -3,8 +3,20 @@ title: llama.cpp slug: /guides/providers/llama-cpp --- + + llama.cpp - Jan Guides + + + + + + + + + + ## Overview [Nitro](https://github.com/janhq/nitro) is an inference server on top of [llama.cpp](https://github.com/ggerganov/llama.cpp). It provides an OpenAI-compatible API, queue, & scaling. -Nitro is the default AI engine downloaded with Jan. There is no additional setup needed. \ No newline at end of file +Nitro is the default AI engine downloaded with Jan. There is no additional setup needed. diff --git a/docs/docs/guides/providers/tensorrt-llm.md b/docs/docs/guides/providers/tensorrt-llm.md index 52da83b36..b30d3510f 100644 --- a/docs/docs/guides/providers/tensorrt-llm.md +++ b/docs/docs/guides/providers/tensorrt-llm.md @@ -3,11 +3,23 @@ title: TensorRT-LLM slug: /guides/providers/tensorrt-llm --- + + TensorRT-LLM - Jan Guides + + + + + + + + + + Users with Nvidia GPUs can get **20-40% faster\* token speeds** on their laptop or desktops by using [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). The greater implication is that you are running FP16, which is also more accurate than quantized models. This guide walks you through how to install Jan's official [TensorRT-LLM Extension](https://github.com/janhq/nitro-tensorrt-llm). This extension uses [Nitro-TensorRT-LLM](https://github.com/janhq/nitro-tensorrt-llm) as the AI engine, instead of the default [Nitro-Llama-CPP](https://github.com/janhq/nitro). It includes an efficient C++ server to natively execute the [TRT-LLM C++ runtime](https://nvidia.github.io/TensorRT-LLM/gpt_runtime.html). It also comes with additional feature and performance improvements like OpenAI compatibility, tokenizer improvements, and queues. -*Compared to using LlamaCPP engine. +\*Compared to using LlamaCPP engine. :::warning This feature is only available for Windows users. Linux is coming soon. @@ -37,6 +49,7 @@ ls ~\jan\extensions\@janhq\tensorrt-llm-extension\dist\bin ``` ## Download a Compatible Model + TensorRT-LLM can only run models in `TensorRT` format. These models, aka "TensorRT Engines", are prebuilt specifically for each target OS+GPU architecture. We offer a handful of precompiled models for Ampere and Ada cards that you can immediately download and play with: @@ -47,7 +60,7 @@ We offer a handful of precompiled models for Ampere and Ada cards that you can i ![image](https://hackmd.io/_uploads/rJewrEgRp.png) 3. Click use and start chatting! -4. You may need to allow Nitro in your network +4. You may need to allow Nitro in your network ![alt text](image.png) @@ -57,7 +70,7 @@ If you are our nightly builds, you may have to reinstall the TensorRT-LLM extens ## Configure Settings -You can customize the default parameters for how Jan runs TensorRT-LLM. +You can customize the default parameters for how Jan runs TensorRT-LLM. :::info coming soon diff --git a/docs/docs/guides/quickstart.mdx b/docs/docs/guides/quickstart.mdx index 84612716a..4ef86b589 100644 --- a/docs/docs/guides/quickstart.mdx +++ b/docs/docs/guides/quickstart.mdx @@ -16,6 +16,18 @@ keywords: ] --- + + Quickstart - Jan Docs + + + + + + + + + + import installImageURL from './assets/jan-ai-quickstart.png'; import flow from './assets/quick.png'; diff --git a/docs/docs/guides/start-server.mdx b/docs/docs/guides/start-server.mdx index d293bc646..2ae72c92b 100644 --- a/docs/docs/guides/start-server.mdx +++ b/docs/docs/guides/start-server.mdx @@ -4,6 +4,17 @@ sidebar_position: 4 description: A step-by-step guide to start Jan Local Server. --- + + Local Server - Jan Docs + + + + + + + + + Jan provides a built-in API server that can be used as a drop-in for OpenAI's API local replacement. This guide will walk you through on how to start the local server and use it to make request to the local server. diff --git a/docs/docs/guides/thread.mdx b/docs/docs/guides/thread.mdx index fdd8fb603..57f036bef 100644 --- a/docs/docs/guides/thread.mdx +++ b/docs/docs/guides/thread.mdx @@ -5,6 +5,17 @@ hide_table_of_contents: true description: Manage your interaction with AI locally. --- + + Thread Management - Jan Docs + + + + + + + + + Jan provides a straightforward and private solution for managing your threads with AI on your own device. As you interact with AI using Jan, you'll accumulate a history of threads. Jan offers easy tools to organize, delete, or review your past threads with AI. This guide will show you how to keep your threads private and well-organized. diff --git a/docs/docs/hardware/overview/cloud-vs-self-hosting.md b/docs/docs/hardware/overview/cloud-vs-self-hosting.md index 0d34bb1a9..8a24ed5af 100644 --- a/docs/docs/hardware/overview/cloud-vs-self-hosting.md +++ b/docs/docs/hardware/overview/cloud-vs-self-hosting.md @@ -2,6 +2,18 @@ title: Cloud vs. Self-hosting Your AI --- + + Cloud vs. Self-hosting Your AI + + + + + + + + + + The choice of how to run your AI - on GPU cloud services, on-prem, or just using an API provider - involves various trade-offs. The following is a naive exploration of the pros and cons of renting vs self-hosting. ## Cost Comparison diff --git a/docs/docs/hardware/overview/cpu-vs-gpu.md b/docs/docs/hardware/overview/cpu-vs-gpu.md index f0f20d8d6..e22dc43dc 100644 --- a/docs/docs/hardware/overview/cpu-vs-gpu.md +++ b/docs/docs/hardware/overview/cpu-vs-gpu.md @@ -2,6 +2,20 @@ title: GPU vs CPU What's the Difference? --- + + GPU vs CPU What's the Difference? + + + + + + + + + + + + ## CPU vs. GPU | | CPU | GPU | diff --git a/docs/docs/how-we-work.md b/docs/docs/how-we-work.md index e81099d18..20f3fb685 100644 --- a/docs/docs/how-we-work.md +++ b/docs/docs/how-we-work.md @@ -2,6 +2,18 @@ title: How We Work --- + + How We Work - Jan + + + + + + + + + + ### Open Source Jan is a startup with an open source business model. We believe in the need for an open source AI ecosystem, and are committed to building it. diff --git a/docs/docs/how-we-work/analytics/analytics.md b/docs/docs/how-we-work/analytics/analytics.md index 79e107a83..a2333a13a 100644 --- a/docs/docs/how-we-work/analytics/analytics.md +++ b/docs/docs/how-we-work/analytics/analytics.md @@ -2,6 +2,18 @@ title: Analytics --- + + Analytics + + + + + + + + + + Adhering to Jan's privacy preserving philosophy, our analytics philosophy is to get "barely-enough-to-function'. #### What is tracked diff --git a/docs/docs/how-we-work/engineering/qa.mdx b/docs/docs/how-we-work/engineering/qa.mdx index f43caae4a..43abdc12a 100644 --- a/docs/docs/how-we-work/engineering/qa.mdx +++ b/docs/docs/how-we-work/engineering/qa.mdx @@ -15,6 +15,18 @@ keywords: ] --- + + QA + + + + + + + + + + ### Phase 1: Planning #### Definition of Ready (DoR): diff --git a/docs/docs/how-we-work/project-management/project-management.md b/docs/docs/how-we-work/project-management/project-management.md index 58af4a0d3..d7429b66e 100644 --- a/docs/docs/how-we-work/project-management/project-management.md +++ b/docs/docs/how-we-work/project-management/project-management.md @@ -2,6 +2,18 @@ title: Project Management --- + + Project Management + + + + + + + + + + We use the [Jan Monorepo Project](https://github.com/orgs/janhq/projects/5) in Github to manage our roadmap and sprint Kanbans. As much as possible, everyone owns their respective `epics` and `tasks`. @@ -58,7 +70,6 @@ We aim to always sprint on `tasks` that are a part of the [current roadmap](http - `Urgent bugs`: assign to an owner (or @engineers if you are not sure) && tag the current `sprint` & `milestone` - `All else`: assign the correct roadmap `label(s)` and owner (if any) - #### Request for help As a result, our feature prioritization can feel a bit black box at times. diff --git a/docs/docs/how-we-work/strategy/strategy.md b/docs/docs/how-we-work/strategy/strategy.md index 09d9b9fb4..c0dc5c44c 100644 --- a/docs/docs/how-we-work/strategy/strategy.md +++ b/docs/docs/how-we-work/strategy/strategy.md @@ -2,7 +2,20 @@ title: Strategy --- + + Strategy + + + + + + + + + + We only have 2 planning parameters: + - 10 year vision - 2 week sprint - Quarterly OKRs @@ -46,7 +59,6 @@ Jan is a seamless user experience that runs on your personal computer, that glue - We run on top of a local folder of non-proprietary files, that anyone can tinker with (yes, even other apps!) - We provide open formats for packaging and distributing AI to run reproducibly across devices - ## Prerequisites - [Figma](https://figma.com) diff --git a/docs/docs/how-we-work/website-docs/website-docs.md b/docs/docs/how-we-work/website-docs/website-docs.md index 19fdc1676..f2ec98e9a 100644 --- a/docs/docs/how-we-work/website-docs/website-docs.md +++ b/docs/docs/how-we-work/website-docs/website-docs.md @@ -2,6 +2,18 @@ title: Website & Docs --- + + Website & Docs + + + + + + + + + + This website is built using [Docusaurus 3.0](https://docusaurus.io/), a modern static website generator. ### Information Architecture diff --git a/docs/docs/platforms/desktop.md b/docs/docs/platforms/desktop.md index fb4ea8389..d2b0efce5 100644 --- a/docs/docs/platforms/desktop.md +++ b/docs/docs/platforms/desktop.md @@ -15,6 +15,18 @@ keywords: ] --- + + Jan Desktop + + + + + + + + + + # Turn any computer into an AI computer ![Alt text](image.png) diff --git a/docs/docs/server-suite/enterprise.md b/docs/docs/server-suite/enterprise.md index 565c14fde..1823bfddd 100644 --- a/docs/docs/server-suite/enterprise.md +++ b/docs/docs/server-suite/enterprise.md @@ -15,6 +15,18 @@ keywords: ] --- + + Jan Enterprise + + + + + + + + + + # Customize and run AI across your organization Jan can professional backend to create, customize and run AIs at scale, for production-grade data centers. diff --git a/docs/docs/server-suite/home-server.md b/docs/docs/server-suite/home-server.md index 97f3afbc7..9e766564c 100644 --- a/docs/docs/server-suite/home-server.md +++ b/docs/docs/server-suite/home-server.md @@ -15,6 +15,18 @@ keywords: ] --- + + Jan Home Server + + + + + + + + + + # Customize and run AI across all of your devices Self-host and access your AI from anywhere with Jan server suite. diff --git a/docs/docs/team/team.md b/docs/docs/team/team.md index 7d5e07cfb..e6ef906f1 100644 --- a/docs/docs/team/team.md +++ b/docs/docs/team/team.md @@ -2,6 +2,18 @@ title: Who we are --- + + Who we are - Jan + + + + + + + + + + What's Jan the company about? We aim to build the cognitive framework for future robots @@ -13,7 +25,6 @@ Jan is a startup with an open source business model. We believe in the need for - [Jan Desktop Client & Local server](https://jan.ai) (AGPLv3, built on Jan Framework) - [Nitro: run Local AI](https://github.com/janhq/nitro) (AGPLv3) - ### Bootstrapped Jan is currently a bootstrapped startup. @@ -25,4 +36,4 @@ We balance technical invention with the search for a sustainable business model. ## Our Team - Contributors -- Core Team \ No newline at end of file +- Core Team diff --git a/docs/docs/wall-of-love.md b/docs/docs/wall-of-love.md index f6bfe79d8..7d130f373 100644 --- a/docs/docs/wall-of-love.md +++ b/docs/docs/wall-of-love.md @@ -2,6 +2,18 @@ title: Wall of Love ❤️ --- + + Wall of Love ❤️ - Jan + + + + + + + + + + ## Twitter Check out our amazing users and what they are saying about Jan! From 5edc6e429c24a635433066d6b0e6f4af54ce8802 Mon Sep 17 00:00:00 2001 From: hiro Date: Tue, 19 Mar 2024 10:47:24 +0700 Subject: [PATCH 12/16] fix: metadata end tag --- docs/docs/acknowledgements.md | 17 +++---- .../advanced-settings/advanced-settings.mdx | 20 ++++---- .../guides/advanced-settings/http-proxy.mdx | 20 ++++---- docs/docs/guides/best-practices.mdx | 16 +++--- .../docs/guides/common-error/broken-build.mdx | 20 ++++---- .../guides/common-error/not-using-gpu.mdx | 20 ++++---- .../error-codes/how-to-get-error-logs.mdx | 20 ++++---- .../error-codes/no-assistant-available.mdx | 20 ++++---- .../guides/error-codes/something-amiss.mdx | 16 +++--- .../error-codes/stuck-on-loading-model.mdx | 16 +++--- .../error-codes/thread-disappreance.mdx | 16 +++--- .../guides/error-codes/undefined-issue.mdx | 16 +++--- .../guides/error-codes/unexpected-token.mdx | 16 +++--- docs/docs/guides/faq.mdx | 16 +++--- docs/docs/guides/install.mdx | 16 +++--- docs/docs/guides/integration/azure.mdx | 16 +++--- docs/docs/guides/integration/discord.mdx | 16 +++--- docs/docs/guides/integration/groq.mdx | 16 +++--- docs/docs/guides/integration/lmstudio.mdx | 16 +++--- docs/docs/guides/integration/mistral.mdx | 16 +++--- docs/docs/guides/integration/ollama.mdx | 16 +++--- .../guides/integration/openinterpreter.mdx | 16 +++--- docs/docs/guides/integration/openrouter.mdx | 16 +++--- docs/docs/guides/integration/raycast.mdx | 16 +++--- docs/docs/guides/integration/vscode.mdx | 16 +++--- docs/docs/guides/models-list.mdx | 16 +++--- docs/docs/guides/models/customize-engine.mdx | 16 +++--- docs/docs/guides/models/import-models.mdx | 16 +++--- docs/docs/guides/models/integrate-remote.mdx | 16 +++--- docs/docs/guides/providers/llama-cpp.md | 16 +++--- docs/docs/guides/providers/tensorrt-llm.md | 16 +++--- docs/docs/guides/quickstart.mdx | 16 +++--- docs/docs/guides/start-server.mdx | 16 +++--- docs/docs/guides/thread.mdx | 16 +++--- .../overview/cloud-vs-self-hosting.md | 16 +++--- docs/docs/hardware/overview/cpu-vs-gpu.md | 20 ++++---- docs/docs/how-we-work.md | 16 +++--- docs/docs/how-we-work/analytics/analytics.md | 16 +++--- docs/docs/how-we-work/engineering/qa.mdx | 16 +++--- .../project-management/project-management.md | 16 +++--- docs/docs/how-we-work/strategy/strategy.md | 16 +++--- .../how-we-work/website-docs/website-docs.md | 16 +++--- docs/docs/platforms/desktop.md | 16 +++--- docs/docs/server-suite/enterprise.md | 16 +++--- docs/docs/server-suite/home-server.md | 16 +++--- docs/docs/team/team.md | 16 +++--- docs/docs/template/QA_script.md | 51 ------------------- docs/docs/wall-of-love.md | 16 +++--- 48 files changed, 390 insertions(+), 442 deletions(-) diff --git a/docs/docs/acknowledgements.md b/docs/docs/acknowledgements.md index a7f77218e..6807d90fd 100644 --- a/docs/docs/acknowledgements.md +++ b/docs/docs/acknowledgements.md @@ -18,15 +18,14 @@ keywords: --- - Acknowledgements - Jan - - - - - - - - + + + + + + + + # Acknowledgements diff --git a/docs/docs/guides/advanced-settings/advanced-settings.mdx b/docs/docs/guides/advanced-settings/advanced-settings.mdx index 1ce65867a..7099843ca 100644 --- a/docs/docs/guides/advanced-settings/advanced-settings.mdx +++ b/docs/docs/guides/advanced-settings/advanced-settings.mdx @@ -17,16 +17,16 @@ keywords: Advanced Settings - - - - - - - - - - + + + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/advanced-settings/http-proxy.mdx b/docs/docs/guides/advanced-settings/http-proxy.mdx index 297133aae..c3e3b0b52 100644 --- a/docs/docs/guides/advanced-settings/http-proxy.mdx +++ b/docs/docs/guides/advanced-settings/http-proxy.mdx @@ -18,16 +18,16 @@ keywords: HTTPS Proxy - - - - - - - - - - + + + + + + + + + + ## Why HTTPS Proxy? diff --git a/docs/docs/guides/best-practices.mdx b/docs/docs/guides/best-practices.mdx index d6e8c4584..9f06ac68a 100644 --- a/docs/docs/guides/best-practices.mdx +++ b/docs/docs/guides/best-practices.mdx @@ -19,14 +19,14 @@ keywords: Best Practices - Jan Guides - - - - - - - - + + + + + + + + Jan is a versatile platform offering solutions for integrating AI locally across various platforms. This guide outlines best practices for developers, analysts, and AI enthusiasts to enhance their experience with Jan when adding AI locally to their computers. Implementing these practices will optimize the performance of AI models. diff --git a/docs/docs/guides/common-error/broken-build.mdx b/docs/docs/guides/common-error/broken-build.mdx index 9a8809955..39004068b 100644 --- a/docs/docs/guides/common-error/broken-build.mdx +++ b/docs/docs/guides/common-error/broken-build.mdx @@ -19,16 +19,16 @@ keywords: Broken Build - - - - - - - - - - + + + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/common-error/not-using-gpu.mdx b/docs/docs/guides/common-error/not-using-gpu.mdx index b932a6822..d0f719919 100644 --- a/docs/docs/guides/common-error/not-using-gpu.mdx +++ b/docs/docs/guides/common-error/not-using-gpu.mdx @@ -19,16 +19,16 @@ keywords: [ Troubleshooting NVIDIA GPU - - - - - - - - - - + + + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/error-codes/how-to-get-error-logs.mdx b/docs/docs/guides/error-codes/how-to-get-error-logs.mdx index 74a7508c0..03c6ec87c 100644 --- a/docs/docs/guides/error-codes/how-to-get-error-logs.mdx +++ b/docs/docs/guides/error-codes/how-to-get-error-logs.mdx @@ -19,16 +19,16 @@ keywords: How to Get Error Logs - - - - - - - - - - + + + + + + + + + + To get the error logs of your Jan application, follow the steps below: diff --git a/docs/docs/guides/error-codes/no-assistant-available.mdx b/docs/docs/guides/error-codes/no-assistant-available.mdx index 7716dc5b0..2d298aca0 100644 --- a/docs/docs/guides/error-codes/no-assistant-available.mdx +++ b/docs/docs/guides/error-codes/no-assistant-available.mdx @@ -19,16 +19,16 @@ keywords: No Assistant Available - - - - - - - - - - + + + + + + + + + + When you encounter the following error message: diff --git a/docs/docs/guides/error-codes/something-amiss.mdx b/docs/docs/guides/error-codes/something-amiss.mdx index 63f918441..8ecaeacf3 100644 --- a/docs/docs/guides/error-codes/something-amiss.mdx +++ b/docs/docs/guides/error-codes/something-amiss.mdx @@ -6,14 +6,14 @@ description: A step-by-step guide to resolve an unspecified or general error. Something's Amiss - - - - - - - - + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/error-codes/stuck-on-loading-model.mdx b/docs/docs/guides/error-codes/stuck-on-loading-model.mdx index 3c5adf035..80f55d403 100644 --- a/docs/docs/guides/error-codes/stuck-on-loading-model.mdx +++ b/docs/docs/guides/error-codes/stuck-on-loading-model.mdx @@ -19,14 +19,14 @@ keywords: Stuck on Loading Model - - - - - - - - + + + + + + + + ## 1. Issue: Model Loading Stuck Due To Missing Windows Management Instrumentation Command-line (WMIC) diff --git a/docs/docs/guides/error-codes/thread-disappreance.mdx b/docs/docs/guides/error-codes/thread-disappreance.mdx index 120e8585c..ef8c35f64 100644 --- a/docs/docs/guides/error-codes/thread-disappreance.mdx +++ b/docs/docs/guides/error-codes/thread-disappreance.mdx @@ -19,14 +19,14 @@ keywords: Thread Disappearance - - - - - - - - + + + + + + + + When you encounter the error of old threads suddenly disappear. This can happen when a new, unintentional file is created in `/jan/threads`. diff --git a/docs/docs/guides/error-codes/undefined-issue.mdx b/docs/docs/guides/error-codes/undefined-issue.mdx index 9373044a4..5f4281155 100644 --- a/docs/docs/guides/error-codes/undefined-issue.mdx +++ b/docs/docs/guides/error-codes/undefined-issue.mdx @@ -19,14 +19,14 @@ keywords: Undefined Issue - - - - - - - - + + + + + + + + Encountering an `undefined issue` in Jan is caused by errors related to the Nitro tool or other internal processes. It can be resolved through the following steps: diff --git a/docs/docs/guides/error-codes/unexpected-token.mdx b/docs/docs/guides/error-codes/unexpected-token.mdx index 252cd97fa..28f69af51 100644 --- a/docs/docs/guides/error-codes/unexpected-token.mdx +++ b/docs/docs/guides/error-codes/unexpected-token.mdx @@ -19,14 +19,14 @@ keywords: Unexpected Token - - - - - - - - + + + + + + + + Encountering the `Unexpected token` error when initiating a chat with OpenAI models mainly caused by either your OpenAI key or where you access your OpenAI from. This issue can be solved through the following steps: diff --git a/docs/docs/guides/faq.mdx b/docs/docs/guides/faq.mdx index 862d43620..f1d1fca2c 100644 --- a/docs/docs/guides/faq.mdx +++ b/docs/docs/guides/faq.mdx @@ -19,14 +19,14 @@ keywords: FAQs - Jan Guides - - - - - - - - + + + + + + + + ## General Issues diff --git a/docs/docs/guides/install.mdx b/docs/docs/guides/install.mdx index 11fc96685..b64469411 100644 --- a/docs/docs/guides/install.mdx +++ b/docs/docs/guides/install.mdx @@ -18,14 +18,14 @@ keywords: Installation - Jan Guides - - - - - - - - + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/integration/azure.mdx b/docs/docs/guides/integration/azure.mdx index 24f457d28..71f070ffb 100644 --- a/docs/docs/guides/integration/azure.mdx +++ b/docs/docs/guides/integration/azure.mdx @@ -19,14 +19,14 @@ keywords: Azure OpenAI - - - - - - - - + + + + + + + + ## How to Integrate Azure OpenAI with Jan diff --git a/docs/docs/guides/integration/discord.mdx b/docs/docs/guides/integration/discord.mdx index 7b91cc972..9bff868d9 100644 --- a/docs/docs/guides/integration/discord.mdx +++ b/docs/docs/guides/integration/discord.mdx @@ -6,14 +6,14 @@ description: A step-by-step guide on how to integrate Jan with a Discord bot. Discord - - - - - - - - + + + + + + + + ## How to Integrate Discord Bot with Jan diff --git a/docs/docs/guides/integration/groq.mdx b/docs/docs/guides/integration/groq.mdx index 2edc1caa5..db97461a7 100644 --- a/docs/docs/guides/integration/groq.mdx +++ b/docs/docs/guides/integration/groq.mdx @@ -19,14 +19,14 @@ keywords: Groq - - - - - - - - + + + + + + + + ## How to Integrate Mistral AI with Jan diff --git a/docs/docs/guides/integration/lmstudio.mdx b/docs/docs/guides/integration/lmstudio.mdx index 9b018c2a7..058c2df49 100644 --- a/docs/docs/guides/integration/lmstudio.mdx +++ b/docs/docs/guides/integration/lmstudio.mdx @@ -18,14 +18,14 @@ keywords: LM Studio - - - - - - - - + + + + + + + + ## How to Integrate LM Studio with Jan diff --git a/docs/docs/guides/integration/mistral.mdx b/docs/docs/guides/integration/mistral.mdx index 1772fefd9..200713422 100644 --- a/docs/docs/guides/integration/mistral.mdx +++ b/docs/docs/guides/integration/mistral.mdx @@ -18,14 +18,14 @@ keywords: Mistral AI - - - - - - - - + + + + + + + + ## How to Integrate Mistral AI with Jan diff --git a/docs/docs/guides/integration/ollama.mdx b/docs/docs/guides/integration/ollama.mdx index e50431f79..bbf6cc826 100644 --- a/docs/docs/guides/integration/ollama.mdx +++ b/docs/docs/guides/integration/ollama.mdx @@ -18,14 +18,14 @@ keywords: Ollama - - - - - - - - + + + + + + + + ## How to Integrate Ollama with Jan diff --git a/docs/docs/guides/integration/openinterpreter.mdx b/docs/docs/guides/integration/openinterpreter.mdx index ff706a3bc..a637736dc 100644 --- a/docs/docs/guides/integration/openinterpreter.mdx +++ b/docs/docs/guides/integration/openinterpreter.mdx @@ -6,14 +6,14 @@ description: A step-by-step guide on how to integrate Jan with Open Interpreter. Open Interpreter - - - - - - - - + + + + + + + + ## How to Integrate Open Interpreter with Jan diff --git a/docs/docs/guides/integration/openrouter.mdx b/docs/docs/guides/integration/openrouter.mdx index 89e90c6d5..01f8077ba 100644 --- a/docs/docs/guides/integration/openrouter.mdx +++ b/docs/docs/guides/integration/openrouter.mdx @@ -6,14 +6,14 @@ description: A step-by-step guide on how to integrate Jan with OpenRouter. OpenRouter - - - - - - - - + + + + + + + + ## How to Integrate OpenRouter with Jan diff --git a/docs/docs/guides/integration/raycast.mdx b/docs/docs/guides/integration/raycast.mdx index c9cb092b7..4e255872b 100644 --- a/docs/docs/guides/integration/raycast.mdx +++ b/docs/docs/guides/integration/raycast.mdx @@ -6,14 +6,14 @@ description: A step-by-step guide on how to integrate Jan with Raycast. Raycast - - - - - - - - + + + + + + + + ## How to Integrate Raycast diff --git a/docs/docs/guides/integration/vscode.mdx b/docs/docs/guides/integration/vscode.mdx index e40f91a78..75c477e15 100644 --- a/docs/docs/guides/integration/vscode.mdx +++ b/docs/docs/guides/integration/vscode.mdx @@ -19,14 +19,14 @@ keywords: Continue - - - - - - - - + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/models-list.mdx b/docs/docs/guides/models-list.mdx index 149d73733..48f6c7ba5 100644 --- a/docs/docs/guides/models-list.mdx +++ b/docs/docs/guides/models-list.mdx @@ -5,14 +5,14 @@ sidebar_position: 3 Pre-configured Models - Jan Guides - - - - - - - - + + + + + + + + ## Overview diff --git a/docs/docs/guides/models/customize-engine.mdx b/docs/docs/guides/models/customize-engine.mdx index 440e02243..9f3eb5eb9 100644 --- a/docs/docs/guides/models/customize-engine.mdx +++ b/docs/docs/guides/models/customize-engine.mdx @@ -19,14 +19,14 @@ keywords: Customize Engine Settings - - - - - - - - + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/models/import-models.mdx b/docs/docs/guides/models/import-models.mdx index a1f333db8..5f1a75ab5 100644 --- a/docs/docs/guides/models/import-models.mdx +++ b/docs/docs/guides/models/import-models.mdx @@ -19,14 +19,14 @@ keywords: Manual Import - - - - - - - - + + + + + + + + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/models/integrate-remote.mdx b/docs/docs/guides/models/integrate-remote.mdx index c99d7b98f..46e2bf508 100644 --- a/docs/docs/guides/models/integrate-remote.mdx +++ b/docs/docs/guides/models/integrate-remote.mdx @@ -20,14 +20,14 @@ keywords: Remote Server Integration - - - - - - - - + + + + + + + + This guide will show you how to configure Jan as a client and point it to any remote & local (self-hosted) API server. diff --git a/docs/docs/guides/providers/llama-cpp.md b/docs/docs/guides/providers/llama-cpp.md index 8d9db5398..0116973bd 100644 --- a/docs/docs/guides/providers/llama-cpp.md +++ b/docs/docs/guides/providers/llama-cpp.md @@ -5,14 +5,14 @@ slug: /guides/providers/llama-cpp llama.cpp - Jan Guides - - - - - - - - + + + + + + + + ## Overview diff --git a/docs/docs/guides/providers/tensorrt-llm.md b/docs/docs/guides/providers/tensorrt-llm.md index b30d3510f..ad37c1969 100644 --- a/docs/docs/guides/providers/tensorrt-llm.md +++ b/docs/docs/guides/providers/tensorrt-llm.md @@ -5,14 +5,14 @@ slug: /guides/providers/tensorrt-llm TensorRT-LLM - Jan Guides - - - - - - - - + + + + + + + + Users with Nvidia GPUs can get **20-40% faster\* token speeds** on their laptop or desktops by using [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). The greater implication is that you are running FP16, which is also more accurate than quantized models. diff --git a/docs/docs/guides/quickstart.mdx b/docs/docs/guides/quickstart.mdx index 4ef86b589..e0702b0eb 100644 --- a/docs/docs/guides/quickstart.mdx +++ b/docs/docs/guides/quickstart.mdx @@ -18,14 +18,14 @@ keywords: Quickstart - Jan Docs - - - - - - - - + + + + + + + + import installImageURL from './assets/jan-ai-quickstart.png'; diff --git a/docs/docs/guides/start-server.mdx b/docs/docs/guides/start-server.mdx index 2ae72c92b..741eb2ffc 100644 --- a/docs/docs/guides/start-server.mdx +++ b/docs/docs/guides/start-server.mdx @@ -6,14 +6,14 @@ description: A step-by-step guide to start Jan Local Server. Local Server - Jan Docs - - - - - - - - + + + + + + + + Jan provides a built-in API server that can be used as a drop-in for OpenAI's API local replacement. This guide will walk you through on how to start the local server and use it to make request to the local server. diff --git a/docs/docs/guides/thread.mdx b/docs/docs/guides/thread.mdx index 57f036bef..cb6940a89 100644 --- a/docs/docs/guides/thread.mdx +++ b/docs/docs/guides/thread.mdx @@ -7,14 +7,14 @@ description: Manage your interaction with AI locally. Thread Management - Jan Docs - - - - - - - - + + + + + + + + Jan provides a straightforward and private solution for managing your threads with AI on your own device. As you interact with AI using Jan, you'll accumulate a history of threads. diff --git a/docs/docs/hardware/overview/cloud-vs-self-hosting.md b/docs/docs/hardware/overview/cloud-vs-self-hosting.md index 8a24ed5af..8452d6967 100644 --- a/docs/docs/hardware/overview/cloud-vs-self-hosting.md +++ b/docs/docs/hardware/overview/cloud-vs-self-hosting.md @@ -4,14 +4,14 @@ title: Cloud vs. Self-hosting Your AI Cloud vs. Self-hosting Your AI - - - - - - - - + + + + + + + + The choice of how to run your AI - on GPU cloud services, on-prem, or just using an API provider - involves various trade-offs. The following is a naive exploration of the pros and cons of renting vs self-hosting. diff --git a/docs/docs/hardware/overview/cpu-vs-gpu.md b/docs/docs/hardware/overview/cpu-vs-gpu.md index e22dc43dc..7ebd327b4 100644 --- a/docs/docs/hardware/overview/cpu-vs-gpu.md +++ b/docs/docs/hardware/overview/cpu-vs-gpu.md @@ -4,16 +4,16 @@ title: GPU vs CPU What's the Difference? GPU vs CPU What's the Difference? - - - - - - - - - - + + + + + + + + + + ## CPU vs. GPU diff --git a/docs/docs/how-we-work.md b/docs/docs/how-we-work.md index 20f3fb685..b8b24bdb2 100644 --- a/docs/docs/how-we-work.md +++ b/docs/docs/how-we-work.md @@ -4,14 +4,14 @@ title: How We Work How We Work - Jan - - - - - - - - + + + + + + + + ### Open Source diff --git a/docs/docs/how-we-work/analytics/analytics.md b/docs/docs/how-we-work/analytics/analytics.md index a2333a13a..e938a9734 100644 --- a/docs/docs/how-we-work/analytics/analytics.md +++ b/docs/docs/how-we-work/analytics/analytics.md @@ -4,14 +4,14 @@ title: Analytics Analytics - - - - - - - - + + + + + + + + Adhering to Jan's privacy preserving philosophy, our analytics philosophy is to get "barely-enough-to-function'. diff --git a/docs/docs/how-we-work/engineering/qa.mdx b/docs/docs/how-we-work/engineering/qa.mdx index 43abdc12a..8e10ef0b1 100644 --- a/docs/docs/how-we-work/engineering/qa.mdx +++ b/docs/docs/how-we-work/engineering/qa.mdx @@ -17,14 +17,14 @@ keywords: QA - - - - - - - - + + + + + + + + ### Phase 1: Planning diff --git a/docs/docs/how-we-work/project-management/project-management.md b/docs/docs/how-we-work/project-management/project-management.md index d7429b66e..f35a6ba77 100644 --- a/docs/docs/how-we-work/project-management/project-management.md +++ b/docs/docs/how-we-work/project-management/project-management.md @@ -4,14 +4,14 @@ title: Project Management Project Management - - - - - - - - + + + + + + + + We use the [Jan Monorepo Project](https://github.com/orgs/janhq/projects/5) in Github to manage our roadmap and sprint Kanbans. diff --git a/docs/docs/how-we-work/strategy/strategy.md b/docs/docs/how-we-work/strategy/strategy.md index c0dc5c44c..26c1ca74b 100644 --- a/docs/docs/how-we-work/strategy/strategy.md +++ b/docs/docs/how-we-work/strategy/strategy.md @@ -4,14 +4,14 @@ title: Strategy Strategy - - - - - - - - + + + + + + + + We only have 2 planning parameters: diff --git a/docs/docs/how-we-work/website-docs/website-docs.md b/docs/docs/how-we-work/website-docs/website-docs.md index f2ec98e9a..9288fa735 100644 --- a/docs/docs/how-we-work/website-docs/website-docs.md +++ b/docs/docs/how-we-work/website-docs/website-docs.md @@ -4,14 +4,14 @@ title: Website & Docs Website & Docs - - - - - - - - + + + + + + + + This website is built using [Docusaurus 3.0](https://docusaurus.io/), a modern static website generator. diff --git a/docs/docs/platforms/desktop.md b/docs/docs/platforms/desktop.md index d2b0efce5..f09013b6b 100644 --- a/docs/docs/platforms/desktop.md +++ b/docs/docs/platforms/desktop.md @@ -17,14 +17,14 @@ keywords: Jan Desktop - - - - - - - - + + + + + + + + # Turn any computer into an AI computer diff --git a/docs/docs/server-suite/enterprise.md b/docs/docs/server-suite/enterprise.md index 1823bfddd..187e97121 100644 --- a/docs/docs/server-suite/enterprise.md +++ b/docs/docs/server-suite/enterprise.md @@ -17,14 +17,14 @@ keywords: Jan Enterprise - - - - - - - - + + + + + + + + # Customize and run AI across your organization diff --git a/docs/docs/server-suite/home-server.md b/docs/docs/server-suite/home-server.md index 9e766564c..9317f2f76 100644 --- a/docs/docs/server-suite/home-server.md +++ b/docs/docs/server-suite/home-server.md @@ -17,14 +17,14 @@ keywords: Jan Home Server - - - - - - - - + + + + + + + + # Customize and run AI across all of your devices diff --git a/docs/docs/team/team.md b/docs/docs/team/team.md index e6ef906f1..3af811933 100644 --- a/docs/docs/team/team.md +++ b/docs/docs/team/team.md @@ -4,14 +4,14 @@ title: Who we are Who we are - Jan - - - - - - - - + + + + + + + + What's Jan the company about? diff --git a/docs/docs/template/QA_script.md b/docs/docs/template/QA_script.md index d0ba4b90f..de006c629 100644 --- a/docs/docs/template/QA_script.md +++ b/docs/docs/template/QA_script.md @@ -1,25 +1,5 @@ ---- -title: Jan version release template ---- - - - Release Version QA Script template - - - - - - - - - - - - # Regression test -# [Release Version] QA Script - **Release Version:** v0.4.7 **Operating System:** MacOS @@ -96,12 +76,6 @@ title: Jan version release template ### 4. Users can click on a history thread -# <<<<<<< HEAD - -- [ ] Test the ability to click on any thread in the history panel. -- [ ] :key: Verify that clicking a thread brings up the past conversation in the main chat window. -- [ ] :key: Ensure that the selected thread is highlighted or otherwise indicated in the history panel. - > > > > > > > f2847d56c (feat: Add SEO header tag with OpenAI GPT-4) - [ ] Confirm that the chat window displays the entire conversation from the selected history thread without any missing messages. - [ ] :key: Check the performance and accuracy of the history feature when dealing with a large number of threads. - [ ] Validate that historical threads reflect the exact state of the chat at that time, including settings. @@ -110,10 +84,6 @@ title: Jan version release template ### 5. Users can config instructions for the assistant. -# <<<<<<< HEAD - -- [ ] Ensure there is a clear interface to input or change instructions for the assistant. - > > > > > > > f2847d56c (feat: Add SEO header tag with OpenAI GPT-4) - [ ] Test if the instructions set by the user are being followed by the assistant in subsequent conversations. - [ ] :key: Validate that changes to instructions are updated in real time and do not require a restart of the application or session. - [ ] :key: Check for the ability to reset instructions to default or clear them completely. @@ -207,27 +177,6 @@ title: Jan version release template - [ ] :key: Test the `Experimental Mode` toggle to confirm it enables or disables experimental features as intended. - [ ] :key: Check the functionality of `Open App Directory` to ensure it opens the correct folder in the system file explorer. - <<<<<<< HEAD - ======= -- [ ] Validate that changes in advanced settings are applied immediately or provide appropriate instructions if a restart is needed. -- [ ] Test the application's stability when experimental features are enabled. - -### 4. Users can add custom plugins via manual installation [TBU] - -- [ ] Verify that the `Manual Installation` option is clearly visible and accessible in the `Extensions` section. -- [ ] Test the functionality of the `Select` button within the `Manual Installation` area. -- [ ] :warning: Check that the file picker dialog allows for the correct plugin file types (e.g., .tgz). -- [ ] :key: Validate that the selected plugin file installs correctly and the plugin becomes functional. -- [ ] Ensure that there is a progress indicator or confirmation message once the installation is complete. -- [ ] Confirm that if the installation is interrupted or fails, the user is given a clear error message. -- [ ] :key: Test that the application prevents the installation of incompatible or corrupt plugin files. -- [ ] :key: Check that the user can uninstall or disable custom plugins as easily as pre-installed ones. -- [ ] Verify that the application's performance remains stable after the installation of custom plugins. - -### 5. Advanced Settings - -- [ ] Attemp to test downloading model from hub using **HTTP Proxy** [guideline](https://github.com/janhq/jan/pull/1562) - > > > > > > > f2847d56c (feat: Add SEO header tag with OpenAI GPT-4) - [ ] Users can move **Jan data folder** - [ ] Validate that changes in advanced settings are applied immediately or provide appropriate instructions if a restart is needed. - [ ] Attemp to test downloading model from hub using **HTTP Proxy** [guideline](https://github.com/janhq/jan/pull/1562) diff --git a/docs/docs/wall-of-love.md b/docs/docs/wall-of-love.md index 7d130f373..359494162 100644 --- a/docs/docs/wall-of-love.md +++ b/docs/docs/wall-of-love.md @@ -4,14 +4,14 @@ title: Wall of Love ❤️ Wall of Love ❤️ - Jan - - - - - - - - + + + + + + + + ## Twitter From 642ee431d7cc8252ae3731d49e9a8158d54915e6 Mon Sep 17 00:00:00 2001 From: hiro Date: Tue, 19 Mar 2024 10:56:45 +0700 Subject: [PATCH 13/16] chore: Update misc --- docs/docs/acknowledgements.md | 2 +- docs/docs/guides/advanced-settings/advanced-settings.mdx | 6 +++--- docs/docs/guides/advanced-settings/http-proxy.mdx | 6 +++--- docs/docs/guides/best-practices.mdx | 2 +- docs/docs/guides/common-error/broken-build.mdx | 6 +++--- docs/docs/guides/common-error/not-using-gpu.mdx | 6 +++--- docs/docs/guides/error-codes/how-to-get-error-logs.mdx | 6 +++--- docs/docs/guides/error-codes/no-assistant-available.mdx | 6 +++--- docs/docs/guides/error-codes/something-amiss.mdx | 2 +- docs/docs/guides/error-codes/stuck-on-loading-model.mdx | 2 +- docs/docs/guides/error-codes/thread-disappreance.mdx | 2 +- docs/docs/guides/error-codes/undefined-issue.mdx | 2 +- docs/docs/guides/error-codes/unexpected-token.mdx | 2 +- docs/docs/guides/faq.mdx | 2 +- docs/docs/guides/install.mdx | 2 +- docs/docs/guides/integration/azure.mdx | 2 +- docs/docs/guides/integration/discord.mdx | 2 +- docs/docs/guides/integration/groq.mdx | 2 +- docs/docs/guides/integration/lmstudio.mdx | 2 +- docs/docs/guides/integration/mistral.mdx | 2 +- docs/docs/guides/integration/ollama.mdx | 2 +- docs/docs/guides/integration/openinterpreter.mdx | 2 +- docs/docs/guides/integration/openrouter.mdx | 2 +- docs/docs/guides/integration/raycast.mdx | 2 +- docs/docs/guides/integration/vscode.mdx | 2 +- docs/docs/guides/models-list.mdx | 2 +- docs/docs/guides/models/customize-engine.mdx | 2 +- docs/docs/guides/models/import-models.mdx | 2 +- docs/docs/guides/models/integrate-remote.mdx | 2 +- docs/docs/guides/providers/llama-cpp.md | 2 +- docs/docs/guides/providers/tensorrt-llm.md | 2 +- docs/docs/guides/quickstart.mdx | 2 +- docs/docs/guides/start-server.mdx | 2 +- docs/docs/guides/thread.mdx | 2 +- docs/docs/hardware/overview/cloud-vs-self-hosting.md | 2 +- docs/docs/hardware/overview/cpu-vs-gpu.md | 2 +- docs/docs/how-we-work.md | 2 +- docs/docs/how-we-work/analytics/analytics.md | 2 +- docs/docs/how-we-work/engineering/qa.mdx | 2 +- .../how-we-work/project-management/project-management.md | 2 +- docs/docs/how-we-work/strategy/strategy.md | 2 +- docs/docs/how-we-work/website-docs/website-docs.md | 2 +- docs/docs/platforms/desktop.md | 2 +- docs/docs/server-suite/enterprise.md | 2 +- docs/docs/server-suite/home-server.md | 2 +- docs/docs/team/team.md | 2 +- docs/docs/wall-of-love.md | 2 +- 47 files changed, 59 insertions(+), 59 deletions(-) diff --git a/docs/docs/acknowledgements.md b/docs/docs/acknowledgements.md index 6807d90fd..46418fc79 100644 --- a/docs/docs/acknowledgements.md +++ b/docs/docs/acknowledgements.md @@ -22,7 +22,7 @@ keywords: - + diff --git a/docs/docs/guides/advanced-settings/advanced-settings.mdx b/docs/docs/guides/advanced-settings/advanced-settings.mdx index 7099843ca..f59d2b4ce 100644 --- a/docs/docs/guides/advanced-settings/advanced-settings.mdx +++ b/docs/docs/guides/advanced-settings/advanced-settings.mdx @@ -21,12 +21,12 @@ keywords: - - + + - + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/advanced-settings/http-proxy.mdx b/docs/docs/guides/advanced-settings/http-proxy.mdx index c3e3b0b52..b2b8c9e24 100644 --- a/docs/docs/guides/advanced-settings/http-proxy.mdx +++ b/docs/docs/guides/advanced-settings/http-proxy.mdx @@ -22,12 +22,12 @@ keywords: - - + + - + ## Why HTTPS Proxy? diff --git a/docs/docs/guides/best-practices.mdx b/docs/docs/guides/best-practices.mdx index 9f06ac68a..a170e07de 100644 --- a/docs/docs/guides/best-practices.mdx +++ b/docs/docs/guides/best-practices.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/common-error/broken-build.mdx b/docs/docs/guides/common-error/broken-build.mdx index 39004068b..ffa470e22 100644 --- a/docs/docs/guides/common-error/broken-build.mdx +++ b/docs/docs/guides/common-error/broken-build.mdx @@ -23,12 +23,12 @@ keywords: - - + + - + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/common-error/not-using-gpu.mdx b/docs/docs/guides/common-error/not-using-gpu.mdx index d0f719919..acc359093 100644 --- a/docs/docs/guides/common-error/not-using-gpu.mdx +++ b/docs/docs/guides/common-error/not-using-gpu.mdx @@ -23,12 +23,12 @@ keywords: [ - - + + - + import Tabs from '@theme/Tabs'; diff --git a/docs/docs/guides/error-codes/how-to-get-error-logs.mdx b/docs/docs/guides/error-codes/how-to-get-error-logs.mdx index 03c6ec87c..aeec5b183 100644 --- a/docs/docs/guides/error-codes/how-to-get-error-logs.mdx +++ b/docs/docs/guides/error-codes/how-to-get-error-logs.mdx @@ -23,12 +23,12 @@ keywords: - - + + - + To get the error logs of your Jan application, follow the steps below: diff --git a/docs/docs/guides/error-codes/no-assistant-available.mdx b/docs/docs/guides/error-codes/no-assistant-available.mdx index 2d298aca0..01f2cf6a2 100644 --- a/docs/docs/guides/error-codes/no-assistant-available.mdx +++ b/docs/docs/guides/error-codes/no-assistant-available.mdx @@ -23,12 +23,12 @@ keywords: - - + + - + When you encounter the following error message: diff --git a/docs/docs/guides/error-codes/something-amiss.mdx b/docs/docs/guides/error-codes/something-amiss.mdx index 8ecaeacf3..51b8c0cf4 100644 --- a/docs/docs/guides/error-codes/something-amiss.mdx +++ b/docs/docs/guides/error-codes/something-amiss.mdx @@ -10,7 +10,7 @@ description: A step-by-step guide to resolve an unspecified or general error. - + diff --git a/docs/docs/guides/error-codes/stuck-on-loading-model.mdx b/docs/docs/guides/error-codes/stuck-on-loading-model.mdx index 80f55d403..a44e93919 100644 --- a/docs/docs/guides/error-codes/stuck-on-loading-model.mdx +++ b/docs/docs/guides/error-codes/stuck-on-loading-model.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/error-codes/thread-disappreance.mdx b/docs/docs/guides/error-codes/thread-disappreance.mdx index ef8c35f64..edae02ac0 100644 --- a/docs/docs/guides/error-codes/thread-disappreance.mdx +++ b/docs/docs/guides/error-codes/thread-disappreance.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/error-codes/undefined-issue.mdx b/docs/docs/guides/error-codes/undefined-issue.mdx index 5f4281155..cf0814977 100644 --- a/docs/docs/guides/error-codes/undefined-issue.mdx +++ b/docs/docs/guides/error-codes/undefined-issue.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/error-codes/unexpected-token.mdx b/docs/docs/guides/error-codes/unexpected-token.mdx index 28f69af51..f62da648b 100644 --- a/docs/docs/guides/error-codes/unexpected-token.mdx +++ b/docs/docs/guides/error-codes/unexpected-token.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/faq.mdx b/docs/docs/guides/faq.mdx index f1d1fca2c..21cf1a232 100644 --- a/docs/docs/guides/faq.mdx +++ b/docs/docs/guides/faq.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/install.mdx b/docs/docs/guides/install.mdx index b64469411..e4e863c1d 100644 --- a/docs/docs/guides/install.mdx +++ b/docs/docs/guides/install.mdx @@ -22,7 +22,7 @@ keywords: - + diff --git a/docs/docs/guides/integration/azure.mdx b/docs/docs/guides/integration/azure.mdx index 71f070ffb..fa22549de 100644 --- a/docs/docs/guides/integration/azure.mdx +++ b/docs/docs/guides/integration/azure.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/integration/discord.mdx b/docs/docs/guides/integration/discord.mdx index 9bff868d9..79ada5396 100644 --- a/docs/docs/guides/integration/discord.mdx +++ b/docs/docs/guides/integration/discord.mdx @@ -10,7 +10,7 @@ description: A step-by-step guide on how to integrate Jan with a Discord bot. - + diff --git a/docs/docs/guides/integration/groq.mdx b/docs/docs/guides/integration/groq.mdx index db97461a7..b1af58c9f 100644 --- a/docs/docs/guides/integration/groq.mdx +++ b/docs/docs/guides/integration/groq.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/integration/lmstudio.mdx b/docs/docs/guides/integration/lmstudio.mdx index 058c2df49..6fdf36cee 100644 --- a/docs/docs/guides/integration/lmstudio.mdx +++ b/docs/docs/guides/integration/lmstudio.mdx @@ -22,7 +22,7 @@ keywords: - + diff --git a/docs/docs/guides/integration/mistral.mdx b/docs/docs/guides/integration/mistral.mdx index 200713422..129bdee21 100644 --- a/docs/docs/guides/integration/mistral.mdx +++ b/docs/docs/guides/integration/mistral.mdx @@ -22,7 +22,7 @@ keywords: - + diff --git a/docs/docs/guides/integration/ollama.mdx b/docs/docs/guides/integration/ollama.mdx index bbf6cc826..c3ee4cd0f 100644 --- a/docs/docs/guides/integration/ollama.mdx +++ b/docs/docs/guides/integration/ollama.mdx @@ -22,7 +22,7 @@ keywords: - + diff --git a/docs/docs/guides/integration/openinterpreter.mdx b/docs/docs/guides/integration/openinterpreter.mdx index a637736dc..ba77738e3 100644 --- a/docs/docs/guides/integration/openinterpreter.mdx +++ b/docs/docs/guides/integration/openinterpreter.mdx @@ -10,7 +10,7 @@ description: A step-by-step guide on how to integrate Jan with Open Interpreter. - + diff --git a/docs/docs/guides/integration/openrouter.mdx b/docs/docs/guides/integration/openrouter.mdx index 01f8077ba..e095a42f1 100644 --- a/docs/docs/guides/integration/openrouter.mdx +++ b/docs/docs/guides/integration/openrouter.mdx @@ -10,7 +10,7 @@ description: A step-by-step guide on how to integrate Jan with OpenRouter. - + diff --git a/docs/docs/guides/integration/raycast.mdx b/docs/docs/guides/integration/raycast.mdx index 4e255872b..4da6e849b 100644 --- a/docs/docs/guides/integration/raycast.mdx +++ b/docs/docs/guides/integration/raycast.mdx @@ -10,7 +10,7 @@ description: A step-by-step guide on how to integrate Jan with Raycast. - + diff --git a/docs/docs/guides/integration/vscode.mdx b/docs/docs/guides/integration/vscode.mdx index 75c477e15..05b141180 100644 --- a/docs/docs/guides/integration/vscode.mdx +++ b/docs/docs/guides/integration/vscode.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/models-list.mdx b/docs/docs/guides/models-list.mdx index 48f6c7ba5..7d80c0220 100644 --- a/docs/docs/guides/models-list.mdx +++ b/docs/docs/guides/models-list.mdx @@ -9,7 +9,7 @@ sidebar_position: 3 - + diff --git a/docs/docs/guides/models/customize-engine.mdx b/docs/docs/guides/models/customize-engine.mdx index 9f3eb5eb9..91d9615f3 100644 --- a/docs/docs/guides/models/customize-engine.mdx +++ b/docs/docs/guides/models/customize-engine.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/models/import-models.mdx b/docs/docs/guides/models/import-models.mdx index 5f1a75ab5..1c131503f 100644 --- a/docs/docs/guides/models/import-models.mdx +++ b/docs/docs/guides/models/import-models.mdx @@ -23,7 +23,7 @@ keywords: - + diff --git a/docs/docs/guides/models/integrate-remote.mdx b/docs/docs/guides/models/integrate-remote.mdx index 46e2bf508..1a0435926 100644 --- a/docs/docs/guides/models/integrate-remote.mdx +++ b/docs/docs/guides/models/integrate-remote.mdx @@ -24,7 +24,7 @@ keywords: - + diff --git a/docs/docs/guides/providers/llama-cpp.md b/docs/docs/guides/providers/llama-cpp.md index 0116973bd..6e1e294b0 100644 --- a/docs/docs/guides/providers/llama-cpp.md +++ b/docs/docs/guides/providers/llama-cpp.md @@ -9,7 +9,7 @@ slug: /guides/providers/llama-cpp - + diff --git a/docs/docs/guides/providers/tensorrt-llm.md b/docs/docs/guides/providers/tensorrt-llm.md index ad37c1969..b0485fd57 100644 --- a/docs/docs/guides/providers/tensorrt-llm.md +++ b/docs/docs/guides/providers/tensorrt-llm.md @@ -9,7 +9,7 @@ slug: /guides/providers/tensorrt-llm - + diff --git a/docs/docs/guides/quickstart.mdx b/docs/docs/guides/quickstart.mdx index e0702b0eb..5fea2f978 100644 --- a/docs/docs/guides/quickstart.mdx +++ b/docs/docs/guides/quickstart.mdx @@ -22,7 +22,7 @@ keywords: - + diff --git a/docs/docs/guides/start-server.mdx b/docs/docs/guides/start-server.mdx index 741eb2ffc..8d394c9d4 100644 --- a/docs/docs/guides/start-server.mdx +++ b/docs/docs/guides/start-server.mdx @@ -10,7 +10,7 @@ description: A step-by-step guide to start Jan Local Server. - + diff --git a/docs/docs/guides/thread.mdx b/docs/docs/guides/thread.mdx index cb6940a89..4852aa0c2 100644 --- a/docs/docs/guides/thread.mdx +++ b/docs/docs/guides/thread.mdx @@ -11,7 +11,7 @@ description: Manage your interaction with AI locally. - + diff --git a/docs/docs/hardware/overview/cloud-vs-self-hosting.md b/docs/docs/hardware/overview/cloud-vs-self-hosting.md index 8452d6967..f4d4b2236 100644 --- a/docs/docs/hardware/overview/cloud-vs-self-hosting.md +++ b/docs/docs/hardware/overview/cloud-vs-self-hosting.md @@ -8,7 +8,7 @@ title: Cloud vs. Self-hosting Your AI - + diff --git a/docs/docs/hardware/overview/cpu-vs-gpu.md b/docs/docs/hardware/overview/cpu-vs-gpu.md index 7ebd327b4..b50655574 100644 --- a/docs/docs/hardware/overview/cpu-vs-gpu.md +++ b/docs/docs/hardware/overview/cpu-vs-gpu.md @@ -9,7 +9,7 @@ title: GPU vs CPU What's the Difference? - + diff --git a/docs/docs/how-we-work.md b/docs/docs/how-we-work.md index b8b24bdb2..602f7c902 100644 --- a/docs/docs/how-we-work.md +++ b/docs/docs/how-we-work.md @@ -8,7 +8,7 @@ title: How We Work - + diff --git a/docs/docs/how-we-work/analytics/analytics.md b/docs/docs/how-we-work/analytics/analytics.md index e938a9734..5991263cc 100644 --- a/docs/docs/how-we-work/analytics/analytics.md +++ b/docs/docs/how-we-work/analytics/analytics.md @@ -8,7 +8,7 @@ title: Analytics - + diff --git a/docs/docs/how-we-work/engineering/qa.mdx b/docs/docs/how-we-work/engineering/qa.mdx index 8e10ef0b1..aa851dfa3 100644 --- a/docs/docs/how-we-work/engineering/qa.mdx +++ b/docs/docs/how-we-work/engineering/qa.mdx @@ -21,7 +21,7 @@ keywords: - + diff --git a/docs/docs/how-we-work/project-management/project-management.md b/docs/docs/how-we-work/project-management/project-management.md index f35a6ba77..85bbe0d75 100644 --- a/docs/docs/how-we-work/project-management/project-management.md +++ b/docs/docs/how-we-work/project-management/project-management.md @@ -8,7 +8,7 @@ title: Project Management - + diff --git a/docs/docs/how-we-work/strategy/strategy.md b/docs/docs/how-we-work/strategy/strategy.md index 26c1ca74b..a448c090e 100644 --- a/docs/docs/how-we-work/strategy/strategy.md +++ b/docs/docs/how-we-work/strategy/strategy.md @@ -8,7 +8,7 @@ title: Strategy - + diff --git a/docs/docs/how-we-work/website-docs/website-docs.md b/docs/docs/how-we-work/website-docs/website-docs.md index 9288fa735..9dcedb0b8 100644 --- a/docs/docs/how-we-work/website-docs/website-docs.md +++ b/docs/docs/how-we-work/website-docs/website-docs.md @@ -8,7 +8,7 @@ title: Website & Docs - + diff --git a/docs/docs/platforms/desktop.md b/docs/docs/platforms/desktop.md index f09013b6b..d8c8a38cc 100644 --- a/docs/docs/platforms/desktop.md +++ b/docs/docs/platforms/desktop.md @@ -21,7 +21,7 @@ keywords: - + diff --git a/docs/docs/server-suite/enterprise.md b/docs/docs/server-suite/enterprise.md index 187e97121..e08356954 100644 --- a/docs/docs/server-suite/enterprise.md +++ b/docs/docs/server-suite/enterprise.md @@ -21,7 +21,7 @@ keywords: - + diff --git a/docs/docs/server-suite/home-server.md b/docs/docs/server-suite/home-server.md index 9317f2f76..630d2b9d9 100644 --- a/docs/docs/server-suite/home-server.md +++ b/docs/docs/server-suite/home-server.md @@ -21,7 +21,7 @@ keywords: - + diff --git a/docs/docs/team/team.md b/docs/docs/team/team.md index 3af811933..2b8b24d01 100644 --- a/docs/docs/team/team.md +++ b/docs/docs/team/team.md @@ -8,7 +8,7 @@ title: Who we are - + diff --git a/docs/docs/wall-of-love.md b/docs/docs/wall-of-love.md index 359494162..41f68b0f2 100644 --- a/docs/docs/wall-of-love.md +++ b/docs/docs/wall-of-love.md @@ -8,7 +8,7 @@ title: Wall of Love ❤️ - + From dd11ba293bc58b974a8beddffb565adaf97e5a67 Mon Sep 17 00:00:00 2001 From: Service Account Date: Tue, 19 Mar 2024 04:46:01 +0000 Subject: [PATCH 14/16] Update README.md with Stable Download URLs --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 31fa4bfc9..623be084c 100644 --- a/README.md +++ b/README.md @@ -43,31 +43,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Stable (Recommended) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 489e8aab24c94a66bf80b3688085cc5c177f9601 Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 19 Mar 2024 12:20:09 +0700 Subject: [PATCH 15/16] Sync release 0.4.9 to dev (#2407) * fix: move tensorrt executable to engine (#2400) * fix: move tensorrt executable to engine Signed-off-by: James * some update Signed-off-by: hiro * chore: bump tensorrt version * fix: wrong destroy path * fix: install extensions in parallel * chore: update path for tensorrt engine (#2404) Signed-off-by: James Co-authored-by: James --------- Signed-off-by: James Signed-off-by: hiro Co-authored-by: James Co-authored-by: hiro Co-authored-by: Louis * Release/v0.4.9 (#2421) * fix: turn off experimental settings should also turn off quick ask (#2411) * fix: app glitches 1s generating response before starting model (#2412) * fix: disable experimental feature should also disable vulkan (#2414) * fix: model load stuck on windows when can't get CPU core count (#2413) Signed-off-by: James Co-authored-by: James * feat: TensorRT-LLM engine update support (#2415) * fix: engine update * chore: add remove prepopulated models Signed-off-by: James * update tinyjensen url Signed-off-by: James * update llamacorn Signed-off-by: James * update Mistral 7B Instruct v0.1 int4 Signed-off-by: James * update tensorrt Signed-off-by: James * update Signed-off-by: hiro * update Signed-off-by: James * prettier Signed-off-by: James * update mistral config Signed-off-by: James * fix some lint Signed-off-by: James --------- Signed-off-by: James Signed-off-by: hiro Co-authored-by: James Co-authored-by: hiro * Tensorrt LLM disable turing support (#2418) Co-authored-by: Hien To * chore: add prompt template tensorrtllm (#2375) * chore: add prompt template tensorrtllm * Add Prompt template for mistral and correct model metadata --------- Co-authored-by: Hien To * fix: correct tensorrt mistral model.json (#2419) --------- Signed-off-by: James Signed-off-by: hiro Co-authored-by: Louis Co-authored-by: James Co-authored-by: hiro Co-authored-by: hiento09 <136591877+hiento09@users.noreply.github.com> Co-authored-by: Hien To --------- Signed-off-by: James Signed-off-by: hiro Co-authored-by: NamH Co-authored-by: James Co-authored-by: hiro Co-authored-by: hiento09 <136591877+hiento09@users.noreply.github.com> Co-authored-by: Hien To --- .../jan-electron-linter-and-test.yml | 1 + core/package.json | 2 +- core/src/api/index.ts | 4 +- core/src/core.ts | 7 +- core/src/extension.ts | 8 + .../extensions/ai-engines/LocalOAIEngine.ts | 6 +- core/src/extensions/monitoring.ts | 3 +- core/src/fs.ts | 7 + core/src/node/api/processors/fsExt.ts | 24 +++ core/src/node/extension/store.ts | 19 +- core/src/node/helper/config.ts | 48 +++--- core/src/node/helper/resource.ts | 2 +- .../types/miscellaneous/systemResourceInfo.ts | 24 +++ extensions/model-extension/src/index.ts | 2 +- extensions/monitoring-extension/src/index.ts | 11 +- .../monitoring-extension/src/node/index.ts | 29 +++- extensions/tensorrt-llm-extension/models.json | 88 ++++++++-- .../tensorrt-llm-extension/package.json | 2 + .../tensorrt-llm-extension/rollup.config.ts | 12 +- .../src/@types/global.d.ts | 1 + .../tensorrt-llm-extension/src/index.ts | 128 +++++++++++--- .../tensorrt-llm-extension/src/node/index.ts | 162 ++++++++++++++---- web/containers/Providers/index.tsx | 8 +- web/hooks/useSendChatMessage.ts | 6 +- web/hooks/useSettings.ts | 5 - web/screens/Settings/Advanced/index.tsx | 43 +++-- .../CoreExtensions/TensorRtExtensionItem.tsx | 24 ++- .../CoreExtensions/UpdateExtensionModal.tsx | 58 +++++++ web/services/appService.ts | 24 ++- 29 files changed, 607 insertions(+), 151 deletions(-) create mode 100644 web/screens/Settings/CoreExtensions/UpdateExtensionModal.tsx diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml index 96258e709..55c3308da 100644 --- a/.github/workflows/jan-electron-linter-and-test.yml +++ b/.github/workflows/jan-electron-linter-and-test.yml @@ -22,6 +22,7 @@ on: branches: - main - dev + - release/** paths: - "electron/**" - .github/workflows/jan-electron-linter-and-test.yml diff --git a/core/package.json b/core/package.json index 2f4f6b576..c4d0d475d 100644 --- a/core/package.json +++ b/core/package.json @@ -46,7 +46,7 @@ }, "devDependencies": { "@types/jest": "^29.5.12", - "@types/node": "^12.0.2", + "@types/node": "^20.11.4", "eslint": "8.57.0", "eslint-plugin-jest": "^27.9.0", "jest": "^29.7.0", diff --git a/core/src/api/index.ts b/core/src/api/index.ts index f97593934..8e41da0d1 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -33,7 +33,7 @@ export enum AppRoute { stopServer = 'stopServer', log = 'log', logServer = 'logServer', - systemInformations = 'systemInformations', + systemInformation = 'systemInformation', showToast = 'showToast', } @@ -95,6 +95,8 @@ export enum FileManagerRoute { getUserHomePath = 'getUserHomePath', fileStat = 'fileStat', writeBlob = 'writeBlob', + mkdir = 'mkdir', + rm = 'rm', } export type ApiFunction = (...args: any[]) => any diff --git a/core/src/core.ts b/core/src/core.ts index b8cbd3162..47c0fe6f2 100644 --- a/core/src/core.ts +++ b/core/src/core.ts @@ -1,4 +1,4 @@ -import { DownloadRequest, FileStat, NetworkConfig } from './types' +import { DownloadRequest, FileStat, NetworkConfig, SystemInformation } from './types' /** * Execute a extension module function in main process @@ -110,7 +110,8 @@ const isSubdirectory: (from: string, to: string) => Promise = (from: st * Get system information * @returns {Promise} - A promise that resolves with the system information. */ -const systemInformations: () => Promise = () => global.core.api?.systemInformations() +const systemInformation: () => Promise = () => + global.core.api?.systemInformation() /** * Show toast message from browser processes. @@ -146,7 +147,7 @@ export { log, isSubdirectory, getUserHomePath, - systemInformations, + systemInformation, showToast, FileStat, } diff --git a/core/src/extension.ts b/core/src/extension.ts index 22accb4b4..973d4778a 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -19,6 +19,7 @@ export interface Compatibility { const ALL_INSTALLATION_STATE = [ 'NotRequired', // not required. 'Installed', // require and installed. Good to go. + 'Updatable', // require and installed but need to be updated. 'NotInstalled', // require to be installed. 'Corrupted', // require but corrupted. Need to redownload. ] as const @@ -59,6 +60,13 @@ export abstract class BaseExtension implements ExtensionType { return undefined } + /** + * Determine if the extension is updatable. + */ + updatable(): boolean { + return false + } + /** * Determine if the prerequisites for the extension are installed. * diff --git a/core/src/extensions/ai-engines/LocalOAIEngine.ts b/core/src/extensions/ai-engines/LocalOAIEngine.ts index 79dbcbf5e..89444ff0f 100644 --- a/core/src/extensions/ai-engines/LocalOAIEngine.ts +++ b/core/src/extensions/ai-engines/LocalOAIEngine.ts @@ -1,4 +1,4 @@ -import { executeOnMain, getJanDataFolderPath, joinPath } from '../../core' +import { executeOnMain, getJanDataFolderPath, joinPath, systemInformation } from '../../core' import { events } from '../../events' import { Model, ModelEvent } from '../../types' import { OAIEngine } from './OAIEngine' @@ -30,11 +30,11 @@ export abstract class LocalOAIEngine extends OAIEngine { if (model.engine.toString() !== this.provider) return const modelFolder = await joinPath([await getJanDataFolderPath(), this.modelFolder, model.id]) - + const systemInfo = await systemInformation() const res = await executeOnMain(this.nodeModule, this.loadModelFunctionName, { modelFolder, model, - }) + }, systemInfo) if (res?.error) { events.emit(ModelEvent.OnModelFail, { diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts index 8d61580fc..2d75e0218 100644 --- a/core/src/extensions/monitoring.ts +++ b/core/src/extensions/monitoring.ts @@ -1,5 +1,5 @@ import { BaseExtension, ExtensionTypeEnum } from '../extension' -import { GpuSetting, MonitoringInterface } from '../index' +import { GpuSetting, MonitoringInterface, OperatingSystemInfo } from '../index' /** * Monitoring extension for system monitoring. @@ -16,4 +16,5 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit abstract getGpuSetting(): Promise abstract getResourcesInfo(): Promise abstract getCurrentLoad(): Promise + abstract getOsInfo(): Promise } diff --git a/core/src/fs.ts b/core/src/fs.ts index 71538ae9c..dacdbb6d6 100644 --- a/core/src/fs.ts +++ b/core/src/fs.ts @@ -37,12 +37,17 @@ const readdirSync = (...args: any[]) => global.core.api?.readdirSync(...args) */ const mkdirSync = (...args: any[]) => global.core.api?.mkdirSync(...args) +const mkdir = (...args: any[]) => global.core.api?.mkdir(...args) + /** * Removes a directory at the specified path. * @returns {Promise} A Promise that resolves when the directory is removed successfully. */ const rmdirSync = (...args: any[]) => global.core.api?.rmdirSync(...args, { recursive: true, force: true }) + +const rm = (path: string) => global.core.api?.rm(path) + /** * Deletes a file from the local file system. * @param {string} path - The path of the file to delete. @@ -92,7 +97,9 @@ export const fs = { existsSync, readdirSync, mkdirSync, + mkdir, rmdirSync, + rm, unlinkSync, appendFileSync, copyFileSync, diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts index 4787da65b..9b88cfef9 100644 --- a/core/src/node/api/processors/fsExt.ts +++ b/core/src/node/api/processors/fsExt.ts @@ -88,4 +88,28 @@ export class FSExt implements Processor { }) }) } + + mkdir(path: string): Promise { + return new Promise((resolve, reject) => { + fs.mkdir(path, { recursive: true }, (err) => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) + } + + rmdir(path: string): Promise { + return new Promise((resolve, reject) => { + fs.rm(path, { recursive: true }, (err) => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) + } } diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts index 93b1aeb2b..630756485 100644 --- a/core/src/node/extension/store.ts +++ b/core/src/node/extension/store.ts @@ -93,8 +93,7 @@ export function persistExtensions() { */ export async function installExtensions(extensions: any) { const installed: Extension[] = [] - for (const ext of extensions) { - // Set install options and activation based on input type + const installations = extensions.map((ext: any): Promise => { const isObject = typeof ext === 'object' const spec = isObject ? [ext.specifier, ext] : [ext] const activate = isObject ? ext.activate !== false : true @@ -102,15 +101,17 @@ export async function installExtensions(extensions: any) { // Install and possibly activate extension const extension = new Extension(...spec) if (!extension.origin) { - continue + return Promise.resolve() } - await extension._install() - if (activate) extension.setActive(true) + return extension._install().then(() => { + if (activate) extension.setActive(true) + // Add extension to store if needed + addExtension(extension) + installed.push(extension) + }) + }) - // Add extension to store if needed - addExtension(extension) - installed.push(extension) - } + await Promise.all(installations) // Return list of all installed extensions return installed diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index 81bc64611..b5ec2e029 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -82,26 +82,34 @@ export const getJanExtensionsPath = (): string => { */ export const physicalCpuCount = async (): Promise => { const platform = os.platform() - if (platform === 'linux') { - const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l') - return parseInt(output.trim(), 10) - } else if (platform === 'darwin') { - const output = await exec('sysctl -n hw.physicalcpu_max') - return parseInt(output.trim(), 10) - } else if (platform === 'win32') { - const output = await exec('WMIC CPU Get NumberOfCores') - return output - .split(os.EOL) - .map((line: string) => parseInt(line)) - .filter((value: number) => !isNaN(value)) - .reduce((sum: number, number: number) => sum + number, 1) - } else { - const cores = os.cpus().filter((cpu: any, index: number) => { - const hasHyperthreading = cpu.model.includes('Intel') - const isOdd = index % 2 === 1 - return !hasHyperthreading || isOdd - }) - return cores.length + try { + if (platform === 'linux') { + const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l') + return parseInt(output.trim(), 10) + } else if (platform === 'darwin') { + const output = await exec('sysctl -n hw.physicalcpu_max') + return parseInt(output.trim(), 10) + } else if (platform === 'win32') { + const output = await exec('WMIC CPU Get NumberOfCores') + return output + .split(os.EOL) + .map((line: string) => parseInt(line)) + .filter((value: number) => !isNaN(value)) + .reduce((sum: number, number: number) => sum + number, 1) + } else { + const cores = os.cpus().filter((cpu: any, index: number) => { + const hasHyperthreading = cpu.model.includes('Intel') + const isOdd = index % 2 === 1 + return !hasHyperthreading || isOdd + }) + return cores.length + } + } catch (err) { + console.warn('Failed to get physical CPU count', err) + // Divide by 2 to get rid of hyper threading + const coreCount = Math.ceil(os.cpus().length / 2) + console.debug('Using node API to get physical CPU count:', coreCount) + return coreCount } } diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts index c79a63688..faaaace05 100644 --- a/core/src/node/helper/resource.ts +++ b/core/src/node/helper/resource.ts @@ -1,6 +1,6 @@ import { SystemResourceInfo } from '../../types' import { physicalCpuCount } from './config' -import { log, logServer } from './log' +import { log } from './log' export const getSystemResourceInfo = async (): Promise => { const cpu = await physicalCpuCount() diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts index f7dd4a82b..fb059b1ba 100644 --- a/core/src/types/miscellaneous/systemResourceInfo.ts +++ b/core/src/types/miscellaneous/systemResourceInfo.ts @@ -30,3 +30,27 @@ export type GpuSettingInfo = { name: string arch?: string } + +export type SystemInformation = { + gpuSetting: GpuSetting + osInfo?: OperatingSystemInfo +} + +export const SupportedPlatforms = ['win32', 'linux', 'darwin'] as const +export type SupportedPlatformTuple = typeof SupportedPlatforms +export type SupportedPlatform = SupportedPlatformTuple[number] + +export type OperatingSystemInfo = { + platform: SupportedPlatform | 'unknown' + arch: string + release: string + machine: string + version: string + totalMem: number + freeMem: number +} + +export type CpuCoreInfo = { + model: string + speed: number +} diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index e2970b8f9..607275884 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -38,7 +38,7 @@ export default class JanModelExtension extends ModelExtension { private static readonly _tensorRtEngineFormat = '.engine' private static readonly _configDirName = 'config' private static readonly _defaultModelFileName = 'default-model.json' - private static readonly _supportedGpuArch = ['turing', 'ampere', 'ada'] + private static readonly _supportedGpuArch = ['ampere', 'ada'] /** * Called when the extension is loaded. diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index c7f53455d..7ef40e7be 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,4 +1,9 @@ -import { GpuSetting, MonitoringExtension, executeOnMain } from '@janhq/core' +import { + GpuSetting, + MonitoringExtension, + OperatingSystemInfo, + executeOnMain, +} from '@janhq/core' /** * JanMonitoringExtension is a extension that provides system monitoring functionality. @@ -41,4 +46,8 @@ export default class JanMonitoringExtension extends MonitoringExtension { getCurrentLoad(): Promise { return executeOnMain(NODE, 'getCurrentLoad') } + + getOsInfo(): Promise { + return executeOnMain(NODE, 'getOsInfo') + } } diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts index 25f151112..ca767d348 100644 --- a/extensions/monitoring-extension/src/node/index.ts +++ b/extensions/monitoring-extension/src/node/index.ts @@ -1,9 +1,16 @@ -import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core' +import { + GpuSetting, + GpuSettingInfo, + OperatingSystemInfo, + ResourceInfo, + SupportedPlatforms, +} from '@janhq/core' import { getJanDataFolderPath, log } from '@janhq/core/node' import { mem, cpu } from 'node-os-utils' import { exec } from 'child_process' import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs' import path from 'path' +import os from 'os' /** * Path to the settings directory @@ -174,8 +181,7 @@ const updateNvidiaDriverInfo = async () => const getGpuArch = (gpuName: string): string => { if (!gpuName.toLowerCase().includes('nvidia')) return 'unknown' - if (gpuName.includes('20')) return 'turing' - else if (gpuName.includes('30')) return 'ampere' + if (gpuName.includes('30')) return 'ampere' else if (gpuName.includes('40')) return 'ada' else return 'unknown' } @@ -320,3 +326,20 @@ const updateCudaExistence = ( data.is_initial = false return data } + +export const getOsInfo = (): OperatingSystemInfo => { + const platform = + SupportedPlatforms.find((p) => p === process.platform) || 'unknown' + + const osInfo: OperatingSystemInfo = { + platform: platform, + arch: process.arch, + release: os.release(), + machine: os.machine(), + version: os.version(), + totalMem: os.totalmem(), + freeMem: os.freemem(), + } + + return osInfo +} diff --git a/extensions/tensorrt-llm-extension/models.json b/extensions/tensorrt-llm-extension/models.json index 7f95940b7..a27cf059d 100644 --- a/extensions/tensorrt-llm-extension/models.json +++ b/extensions/tensorrt-llm-extension/models.json @@ -3,27 +3,31 @@ "sources": [ { "filename": "config.json", - "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/config.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/config.json" }, { - "filename": "rank0.engine", - "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/rank0.engine" + "filename": "mistral_float16_tp1_rank0.engine", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/mistral_float16_tp1_rank0.engine" }, { "filename": "tokenizer.model", - "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer.model" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer.model" }, { "filename": "special_tokens_map.json", - "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/special_tokens_map.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/special_tokens_map.json" }, { "filename": "tokenizer.json", - "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer.json" }, { "filename": "tokenizer_config.json", - "url": "https://delta.jan.ai/dist/models///LlamaCorn-1.1B-Chat-fp16/tokenizer_config.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer_config.json" + }, + { + "filename": "model.cache", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/model.cache" } ], "id": "llamacorn-1.1b-chat-fp16", @@ -50,27 +54,31 @@ "sources": [ { "filename": "config.json", - "url": "https://delta.jan.ai/dist/models///TinyJensen-1.1B-Chat-fp16/config.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/config.json" }, { - "filename": "rank0.engine", - "url": "https://delta.jan.ai/dist/models///TinyJensen-1.1B-Chat-fp16/rank0.engine" + "filename": "mistral_float16_tp1_rank0.engine", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/mistral_float16_tp1_rank0.engine" }, { "filename": "tokenizer.model", - "url": "https://delta.jan.ai/dist/models///TinyJensen-1.1B-Chat-fp16/tokenizer.model" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer.model" }, { "filename": "special_tokens_map.json", - "url": "https://delta.jan.ai/dist/models///TinyJensen-1.1B-Chat-fp16/special_tokens_map.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/special_tokens_map.json" }, { "filename": "tokenizer.json", - "url": "https://delta.jan.ai/dist/models///TinyJensen-1.1B-Chat-fp16/tokenizer.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer.json" }, { "filename": "tokenizer_config.json", - "url": "https://delta.jan.ai/dist/models///TinyJensen-1.1B-Chat-fp16/tokenizer_config.json" + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer_config.json" + }, + { + "filename": "model.cache", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/model.cache" } ], "id": "tinyjensen-1.1b-chat-fp16", @@ -92,5 +100,57 @@ "size": 2151000000 }, "engine": "nitro-tensorrt-llm" + }, + { + "sources": [ + { + "filename": "config.json", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/config.json" + }, + { + "filename": "mistral_float16_tp1_rank0.engine", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/mistral_float16_tp1_rank0.engine" + }, + { + "filename": "tokenizer.model", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer.model" + }, + { + "filename": "special_tokens_map.json", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/special_tokens_map.json" + }, + { + "filename": "tokenizer.json", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer.json" + }, + { + "filename": "tokenizer_config.json", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer_config.json" + }, + { + "filename": "model.cache", + "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/model.cache" + } + ], + "id": "mistral-7b-instruct-int4", + "object": "model", + "name": "Mistral 7B Instruct v0.1 INT4", + "version": "1.0", + "description": "Mistral 7B Instruct v0.1 INT4", + "format": "TensorRT-LLM", + "settings": { + "ctx_len": 2048, + "text_model": false, + "prompt_template": "[INST] {prompt} [/INST]" + }, + "parameters": { + "max_tokens": 4096 + }, + "metadata": { + "author": "MistralAI", + "tags": ["TensorRT-LLM", "7B", "Finetuned"], + "size": 3840000000 + }, + "engine": "nitro-tensorrt-llm" } ] diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json index af1dba853..d3521669e 100644 --- a/extensions/tensorrt-llm-extension/package.json +++ b/extensions/tensorrt-llm-extension/package.json @@ -18,6 +18,8 @@ "0.1.0" ] }, + "tensorrtVersion": "0.1.8", + "provider": "nitro-tensorrt-llm", "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", "build:publish:win32": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", diff --git a/extensions/tensorrt-llm-extension/rollup.config.ts b/extensions/tensorrt-llm-extension/rollup.config.ts index 33e45823b..e602bc720 100644 --- a/extensions/tensorrt-llm-extension/rollup.config.ts +++ b/extensions/tensorrt-llm-extension/rollup.config.ts @@ -16,11 +16,12 @@ export default [ plugins: [ replace({ EXTENSION_NAME: JSON.stringify(packageJson.name), - TENSORRT_VERSION: JSON.stringify('0.1.5'), + TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion), + PROVIDER: JSON.stringify(packageJson.provider), DOWNLOAD_RUNNER_URL: - process.platform === 'darwin' || process.platform === 'win32' + process.platform === 'win32' ? JSON.stringify( - 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/windows-v/nitro-windows-v-amd64-tensorrt-llm-.tar.gz' + 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/windows-v-tensorrt-llm-v0.7.1/nitro-windows-v-tensorrt-llm-v0.7.1-amd64-all-arch.tar.gz' ) : JSON.stringify( 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/linux-v/nitro-linux-v-amd64-tensorrt-llm-.tar.gz' @@ -52,11 +53,14 @@ export default [ }, plugins: [ replace({ + EXTENSION_NAME: JSON.stringify(packageJson.name), + TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion), + PROVIDER: JSON.stringify(packageJson.provider), LOAD_MODEL_URL: JSON.stringify( `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/tensorrtllm/loadmodel` ), TERMINATE_ENGINE_URL: JSON.stringify( - `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/processmanager/destroy` + `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/processmanager/destroy` ), ENGINE_HOST: JSON.stringify(packageJson.config?.host ?? '127.0.0.1'), ENGINE_PORT: JSON.stringify(packageJson.config?.port ?? '3928'), diff --git a/extensions/tensorrt-llm-extension/src/@types/global.d.ts b/extensions/tensorrt-llm-extension/src/@types/global.d.ts index 905e86380..9cf5b6090 100644 --- a/extensions/tensorrt-llm-extension/src/@types/global.d.ts +++ b/extensions/tensorrt-llm-extension/src/@types/global.d.ts @@ -8,3 +8,4 @@ declare const DOWNLOAD_RUNNER_URL: string declare const TENSORRT_VERSION: string declare const COMPATIBILITY: object declare const EXTENSION_NAME: string +declare const PROVIDER: string diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index 02c676841..d2d08e8a7 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -16,11 +16,12 @@ import { executeOnMain, joinPath, showToast, - systemInformations, + systemInformation, LocalOAIEngine, fs, MessageRequest, ModelEvent, + getJanDataFolderPath, } from '@janhq/core' import models from '../models.json' @@ -34,11 +35,13 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { * Override custom function name for loading and unloading model * Which are implemented from node module */ - override provider = 'nitro-tensorrt-llm' + override provider = PROVIDER override inferenceUrl = INFERENCE_URL override nodeModule = NODE - private supportedGpuArch = ['turing', 'ampere', 'ada'] + private supportedGpuArch = ['ampere', 'ada'] + private supportedPlatform = ['win32', 'linux'] + private isUpdateAvailable = false compatibility() { return COMPATIBILITY as unknown as Compatibility @@ -54,7 +57,9 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { } override async install(): Promise { - const info = await systemInformations() + await this.removePopulatedModels() + + const info = await systemInformation() console.debug( `TensorRTLLMExtension installing pre-requisites... ${JSON.stringify(info)}` ) @@ -83,12 +88,19 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { return } - const binaryFolderPath = await executeOnMain( - this.nodeModule, - 'binaryFolder' - ) - if (!(await fs.existsSync(binaryFolderPath))) { - await fs.mkdirSync(binaryFolderPath) + const janDataFolderPath = await getJanDataFolderPath() + const engineVersion = TENSORRT_VERSION + + const executableFolderPath = await joinPath([ + janDataFolderPath, + 'engines', + this.provider, + engineVersion, + firstGpu.arch, + ]) + + if (!(await fs.existsSync(executableFolderPath))) { + await fs.mkdir(executableFolderPath) } const placeholderUrl = DOWNLOAD_RUNNER_URL @@ -100,7 +112,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { const tarball = await baseName(url) - const tarballFullPath = await joinPath([binaryFolderPath, tarball]) + const tarballFullPath = await joinPath([executableFolderPath, tarball]) const downloadRequest: DownloadRequest = { url, localPath: tarballFullPath, @@ -109,12 +121,16 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { } downloadFile(downloadRequest) - // TODO: wrap this into a Promise const onFileDownloadSuccess = async (state: DownloadState) => { // if other download, ignore if (state.fileName !== tarball) return events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) - await executeOnMain(this.nodeModule, 'decompressRunner', tarballFullPath) + await executeOnMain( + this.nodeModule, + 'decompressRunner', + tarballFullPath, + executableFolderPath + ) events.emit(DownloadEvent.onFileUnzipSuccess, state) // Prepopulate models as soon as it's ready @@ -128,6 +144,22 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) } + async removePopulatedModels(): Promise { + console.debug(`removePopulatedModels`, JSON.stringify(models)) + const janDataFolderPath = await getJanDataFolderPath() + const modelFolderPath = await joinPath([janDataFolderPath, 'models']) + + for (const model of models) { + const modelPath = await joinPath([modelFolderPath, model.id]) + console.debug(`modelPath: ${modelPath}`) + if (await fs.existsSync(modelPath)) { + console.debug(`Removing model ${modelPath}`) + await fs.rmdirSync(modelPath) + } + } + events.emit(ModelEvent.OnModelsUpdate, {}) + } + async onModelInit(model: Model): Promise { if (model.engine !== this.provider) return @@ -143,14 +175,70 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { } } - override async installationState(): Promise { - // For now, we just check the executable of nitro x tensor rt - const isNitroExecutableAvailable = await executeOnMain( - this.nodeModule, - 'isNitroExecutableAvailable' - ) + updatable() { + return this.isUpdateAvailable + } - return isNitroExecutableAvailable ? 'Installed' : 'NotInstalled' + override async installationState(): Promise { + const info = await systemInformation() + + const gpuSetting: GpuSetting | undefined = info.gpuSetting + if (gpuSetting === undefined) { + console.warn( + 'No GPU setting found. TensorRT-LLM extension is not installed' + ) + return 'NotInstalled' // TODO: maybe disabled / incompatible is more appropriate + } + + if (gpuSetting.gpus.length === 0) { + console.warn('No GPU found. TensorRT-LLM extension is not installed') + return 'NotInstalled' + } + + const firstGpu = gpuSetting.gpus[0] + if (!firstGpu.name.toLowerCase().includes('nvidia')) { + console.error('No Nvidia GPU found. Please check your GPU setting.') + return 'NotInstalled' + } + + if (firstGpu.arch === undefined) { + console.error('No GPU architecture found. Please check your GPU setting.') + return 'NotInstalled' + } + + if (!this.supportedGpuArch.includes(firstGpu.arch)) { + console.error( + `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` + ) + return 'NotInstalled' + } + + const osInfo = info.osInfo + if (!osInfo) { + console.error('No OS information found. Please check your OS setting.') + return 'NotInstalled' + } + + if (!this.supportedPlatform.includes(osInfo.platform)) { + console.error( + `Your OS: ${osInfo.platform} is not supported. Only Windows and Linux are supported.` + ) + return 'NotInstalled' + } + const janDataFolderPath = await getJanDataFolderPath() + const engineVersion = TENSORRT_VERSION + + const enginePath = await joinPath([ + janDataFolderPath, + 'engines', + this.provider, + engineVersion, + firstGpu.arch, + osInfo.platform === 'win32' ? 'nitro.exe' : 'nitro', + ]) + + // For now, we just check the executable of nitro x tensor rt + return (await fs.existsSync(enginePath)) ? 'Installed' : 'NotInstalled' } override onInferenceStopped() { diff --git a/extensions/tensorrt-llm-extension/src/node/index.ts b/extensions/tensorrt-llm-extension/src/node/index.ts index 252468fc1..1afebb950 100644 --- a/extensions/tensorrt-llm-extension/src/node/index.ts +++ b/extensions/tensorrt-llm-extension/src/node/index.ts @@ -2,13 +2,17 @@ import path from 'path' import { ChildProcessWithoutNullStreams, spawn } from 'child_process' import tcpPortUsed from 'tcp-port-used' import fetchRT from 'fetch-retry' -import { log } from '@janhq/core/node' -import { existsSync } from 'fs' +import { log, getJanDataFolderPath } from '@janhq/core/node' import decompress from 'decompress' +import { SystemInformation } from '@janhq/core' +import { PromptTemplate } from '@janhq/core' // Polyfill fetch with retry const fetchRetry = fetchRT(fetch) +const supportedPlatform = (): string[] => ['win32', 'linux'] +const supportedGpuArch = (): string[] => ['ampere', 'ada'] + /** * The response object for model init operation. */ @@ -24,16 +28,34 @@ let subprocess: ChildProcessWithoutNullStreams | undefined = undefined * Initializes a engine subprocess to load a machine learning model. * @param params - The model load settings. */ -async function loadModel(params: any): Promise<{ error: Error | undefined }> { +async function loadModel( + params: any, + systemInfo?: SystemInformation +): Promise<{ error: Error | undefined }> { // modelFolder is the absolute path to the running model folder // e.g. ~/jan/models/llama-2 let modelFolder = params.modelFolder + if (params.model.settings.prompt_template) { + const promptTemplate = params.model.settings.prompt_template + const prompt = promptTemplateConverter(promptTemplate) + if (prompt?.error) { + return Promise.reject(prompt.error) + } + params.model.settings.system_prompt = prompt.system_prompt + params.model.settings.user_prompt = prompt.user_prompt + params.model.settings.ai_prompt = prompt.ai_prompt + } + const settings: ModelLoadParams = { engine_path: modelFolder, ctx_len: params.model.settings.ctx_len ?? 2048, + ...params.model.settings, } - return runEngineAndLoadModel(settings) + if (!systemInfo) { + throw new Error('Cannot get system info. Unable to start nitro x tensorrt.') + } + return runEngineAndLoadModel(settings, systemInfo) } /** @@ -67,9 +89,12 @@ function unloadModel(): Promise { * 2. Load model into engine subprocess * @returns */ -async function runEngineAndLoadModel(settings: ModelLoadParams) { +async function runEngineAndLoadModel( + settings: ModelLoadParams, + systemInfo: SystemInformation +) { return unloadModel() - .then(runEngine) + .then(() => runEngine(systemInfo)) .then(() => loadModelRequest(settings)) .catch((err) => { // TODO: Broadcast error so app could display proper error message @@ -81,7 +106,7 @@ async function runEngineAndLoadModel(settings: ModelLoadParams) { /** * Loads a LLM model into the Engine subprocess by sending a HTTP POST request. */ -function loadModelRequest( +async function loadModelRequest( settings: ModelLoadParams ): Promise<{ error: Error | undefined }> { debugLog(`Loading model with params ${JSON.stringify(settings)}`) @@ -107,23 +132,66 @@ function loadModelRequest( /** * Spawns engine subprocess. */ -function runEngine(): Promise { +async function runEngine(systemInfo: SystemInformation): Promise { debugLog(`Spawning engine subprocess...`) + if (systemInfo.gpuSetting == null) { + return Promise.reject( + 'No GPU information found. Please check your GPU setting.' + ) + } + + if (systemInfo.gpuSetting.gpus.length === 0) { + return Promise.reject('No GPU found. Please check your GPU setting.') + } + + if (systemInfo.osInfo == null) { + return Promise.reject( + 'No OS information found. Please check your OS setting.' + ) + } + const platform = systemInfo.osInfo.platform + if (platform == null || supportedPlatform().includes(platform) === false) { + return Promise.reject( + 'No OS architecture found. Please check your OS setting.' + ) + } + + const gpu = systemInfo.gpuSetting.gpus[0] + if (gpu.name.toLowerCase().includes('nvidia') === false) { + return Promise.reject('No Nvidia GPU found. Please check your GPU setting.') + } + const gpuArch = gpu.arch + if (gpuArch == null || supportedGpuArch().includes(gpuArch) === false) { + return Promise.reject( + `Your GPU: ${gpu.name} is not supported. Only ${supportedGpuArch().join( + ', ' + )} series are supported.` + ) + } + const janDataFolderPath = await getJanDataFolderPath() + const tensorRtVersion = TENSORRT_VERSION + const provider = PROVIDER return new Promise((resolve, reject) => { // Current directory by default - let binaryFolder = path.join(__dirname, '..', 'bin') - // Binary path - const binary = path.join( - binaryFolder, - process.platform === 'win32' ? 'nitro.exe' : 'nitro' + + const executableFolderPath = path.join( + janDataFolderPath, + 'engines', + provider, + tensorRtVersion, + gpuArch + ) + const nitroExecutablePath = path.join( + executableFolderPath, + platform === 'win32' ? 'nitro.exe' : 'nitro' ) const args: string[] = ['1', ENGINE_HOST, ENGINE_PORT] // Execute the binary - debugLog(`Spawn nitro at path: ${binary}, and args: ${args}`) - subprocess = spawn(binary, args, { - cwd: binaryFolder, + debugLog(`Spawn nitro at path: ${nitroExecutablePath}, and args: ${args}`) + subprocess = spawn(nitroExecutablePath, args, { + cwd: executableFolderPath, env: { ...process.env, }, @@ -155,12 +223,7 @@ function debugLog(message: string, level: string = 'Debug') { log(`[TENSORRT_LLM_NITRO]::${level}:${message}`) } -const binaryFolder = async (): Promise => { - return path.join(__dirname, '..', 'bin') -} - -const decompressRunner = async (zipPath: string) => { - const output = path.join(__dirname, '..', 'bin') +const decompressRunner = async (zipPath: string, output: string) => { console.debug(`Decompressing ${zipPath} to ${output}...`) try { const files = await decompress(zipPath, output) @@ -170,22 +233,57 @@ const decompressRunner = async (zipPath: string) => { } } -const isNitroExecutableAvailable = async (): Promise => { - const binary = path.join( - __dirname, - '..', - 'bin', - process.platform === 'win32' ? 'nitro.exe' : 'nitro' - ) +/** + * Parse prompt template into agrs settings + * @param promptTemplate Template as string + * @returns + */ +function promptTemplateConverter(promptTemplate: string): PromptTemplate { + // Split the string using the markers + const systemMarker = '{system_message}' + const promptMarker = '{prompt}' - return existsSync(binary) + if ( + promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker) + ) { + // Find the indices of the markers + const systemIndex = promptTemplate.indexOf(systemMarker) + const promptIndex = promptTemplate.indexOf(promptMarker) + + // Extract the parts of the string + const system_prompt = promptTemplate.substring(0, systemIndex) + const user_prompt = promptTemplate.substring( + systemIndex + systemMarker.length, + promptIndex + ) + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ) + + // Return the split parts + return { system_prompt, user_prompt, ai_prompt } + } else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + const promptIndex = promptTemplate.indexOf(promptMarker) + const user_prompt = promptTemplate.substring(0, promptIndex) + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ) + + // Return the split parts + return { user_prompt, ai_prompt } + } + + // Return an error if none of the conditions are met + return { error: 'Cannot split prompt template' } } export default { - binaryFolder, + supportedPlatform, + supportedGpuArch, decompressRunner, loadModel, unloadModel, dispose: unloadModel, - isNitroExecutableAvailable, } diff --git a/web/containers/Providers/index.tsx b/web/containers/Providers/index.tsx index e70a56ca8..10c6c7547 100644 --- a/web/containers/Providers/index.tsx +++ b/web/containers/Providers/index.tsx @@ -1,6 +1,6 @@ 'use client' -import { PropsWithChildren, useEffect, useState } from 'react' +import { PropsWithChildren, useCallback, useEffect, useState } from 'react' import { Toaster } from 'react-hot-toast' @@ -37,7 +37,7 @@ const Providers = (props: PropsWithChildren) => { const [activated, setActivated] = useState(false) const [settingUp, setSettingUp] = useState(false) - async function setupExtensions() { + const setupExtensions = useCallback(async () => { // Register all active extensions await extensionManager.registerActive() @@ -57,7 +57,7 @@ const Providers = (props: PropsWithChildren) => { setSettingUp(false) setActivated(true) }, 500) - } + }, [pathname]) // Services Setup useEffect(() => { @@ -78,7 +78,7 @@ const Providers = (props: PropsWithChildren) => { setActivated(true) } } - }, [setupCore]) + }, [setupCore, setupExtensions]) return ( diff --git a/web/hooks/useSendChatMessage.ts b/web/hooks/useSendChatMessage.ts index 11a57a598..0bbc779a6 100644 --- a/web/hooks/useSendChatMessage.ts +++ b/web/hooks/useSendChatMessage.ts @@ -102,7 +102,6 @@ export default function useSendChatMessage() { console.error('No active thread') return } - setIsGeneratingResponse(true) updateThreadWaiting(activeThreadRef.current.id, true) const messages: ChatCompletionMessage[] = [ activeThreadRef.current.assistants[0]?.instructions, @@ -148,7 +147,7 @@ export default function useSendChatMessage() { await waitForModelStarting(modelId) setQueuedMessage(false) } - + setIsGeneratingResponse(true) if (currentMessage.role !== ChatCompletionRole.User) { // Delete last response before regenerating deleteMessage(currentMessage.id ?? '') @@ -171,7 +170,6 @@ export default function useSendChatMessage() { console.error('No active thread') return } - setIsGeneratingResponse(true) if (engineParamsUpdate) setReloadModel(true) @@ -361,7 +359,7 @@ export default function useSendChatMessage() { await waitForModelStarting(modelId) setQueuedMessage(false) } - + setIsGeneratingResponse(true) events.emit(MessageEvent.OnMessageSent, messageRequest) setReloadModel(false) diff --git a/web/hooks/useSettings.ts b/web/hooks/useSettings.ts index 9ff89827e..378ca33fa 100644 --- a/web/hooks/useSettings.ts +++ b/web/hooks/useSettings.ts @@ -70,11 +70,6 @@ export const useSettings = () => { } } await fs.writeFileSync(settingsFile, JSON.stringify(settings)) - - // Relaunch to apply settings - if (vulkan != null) { - window.location.reload() - } } return { diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index 3cc43e744..67ebf81d5 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -90,12 +90,38 @@ const Advanced = () => { [setPartialProxy, setProxy] ) - const updateQuickAskEnabled = async (e: boolean) => { + const updateQuickAskEnabled = async ( + e: boolean, + relaunch: boolean = true + ) => { const appConfiguration: AppConfiguration = await window.core?.api?.getAppConfigurations() appConfiguration.quick_ask = e await window.core?.api?.updateAppConfiguration(appConfiguration) - window.core?.api?.relaunch() + if (relaunch) window.core?.api?.relaunch() + } + + const updateVulkanEnabled = async (e: boolean, relaunch: boolean = true) => { + toaster({ + title: 'Reload', + description: 'Vulkan settings updated. Reload now to apply the changes.', + }) + stopModel() + setVulkanEnabled(e) + await saveSettings({ vulkan: e, gpusInUse: [] }) + // Relaunch to apply settings + if (relaunch) window.location.reload() + } + + const updateExperimentalEnabled = async (e: boolean) => { + setExperimentalEnabled(e) + if (e) return + + // It affects other settings, so we need to reset them + const isRelaunch = quickAskEnabled || vulkanEnabled + if (quickAskEnabled) await updateQuickAskEnabled(false, false) + if (vulkanEnabled) await updateVulkanEnabled(false, false) + if (isRelaunch) window.core?.api?.relaunch() } useEffect(() => { @@ -179,7 +205,7 @@ const Advanced = () => { @@ -381,16 +407,7 @@ const Advanced = () => { { - toaster({ - title: 'Reload', - description: - 'Vulkan settings updated. Reload now to apply the changes.', - }) - stopModel() - saveSettings({ vulkan: e, gpusInUse: [] }) - setVulkanEnabled(e) - }} + onCheckedChange={(e) => updateVulkanEnabled(e)} /> )} diff --git a/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx index c11041ffe..fb0214536 100644 --- a/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx +++ b/web/screens/Settings/CoreExtensions/TensorRtExtensionItem.tsx @@ -5,7 +5,7 @@ import { GpuSetting, InstallationState, abortDownload, - systemInformations, + systemInformation, } from '@janhq/core' import { Badge, @@ -23,6 +23,8 @@ import { useAtomValue } from 'jotai' import { Marked, Renderer } from 'marked' +import UpdateExtensionModal from './UpdateExtensionModal' + import { extensionManager } from '@/extension' import Extension from '@/extension/Extension' import { installingExtensionAtom } from '@/helpers/atoms/Extension.atom' @@ -39,7 +41,7 @@ const TensorRtExtensionItem: React.FC = ({ item }) => { useState('NotRequired') const installingExtensions = useAtomValue(installingExtensionAtom) const [isGpuSupported, setIsGpuSupported] = useState(false) - + const [promptUpdateModal, setPromptUpdateModal] = useState(false) const isInstalling = installingExtensions.some( (e) => e.extensionId === item.name ) @@ -51,7 +53,7 @@ const TensorRtExtensionItem: React.FC = ({ item }) => { useEffect(() => { const getSystemInfos = async () => { - const info = await systemInformations() + const info = await systemInformation() if (!info) { setIsGpuSupported(false) return @@ -69,7 +71,7 @@ const TensorRtExtensionItem: React.FC = ({ item }) => { return } - const supportedGpuArch = ['turing', 'ampere', 'ada'] + const supportedGpuArch = ['ampere', 'ada'] setIsGpuSupported(supportedGpuArch.includes(arch)) } getSystemInfos() @@ -112,7 +114,7 @@ const TensorRtExtensionItem: React.FC = ({ item }) => { } const description = marked.parse(item.description ?? '', { async: false }) - console.log(description) + return (
@@ -138,6 +140,7 @@ const TensorRtExtensionItem: React.FC = ({ item }) => { installProgress={progress} installState={installState} onInstallClick={onInstallClick} + onUpdateClick={() => setPromptUpdateModal(true)} onCancelClick={onCancelInstallingClick} />
@@ -177,6 +180,9 @@ const TensorRtExtensionItem: React.FC = ({ item }) => {
)} + {promptUpdateModal && ( + + )} ) } @@ -185,6 +191,7 @@ type InstallStateProps = { installProgress: number installState: InstallationState onInstallClick: () => void + onUpdateClick: () => void onCancelClick: () => void } @@ -192,6 +199,7 @@ const InstallStateIndicator: React.FC = ({ installProgress, installState, onInstallClick, + onUpdateClick, onCancelClick, }) => { if (installProgress !== -1) { @@ -218,6 +226,12 @@ const InstallStateIndicator: React.FC = ({ Installed ) + case 'Updatable': + return ( + + ) case 'NotInstalled': return ( + + + + + + + + + ) +} + +export default React.memo(UpdateExtensionModal) diff --git a/web/services/appService.ts b/web/services/appService.ts index 9327d55c3..16060e2d9 100644 --- a/web/services/appService.ts +++ b/web/services/appService.ts @@ -1,20 +1,32 @@ -import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' +import { + ExtensionTypeEnum, + MonitoringExtension, + SystemInformation, +} from '@janhq/core' import { toaster } from '@/containers/Toast' import { extensionManager } from '@/extension' export const appService = { - systemInformations: async () => { - const gpuSetting = await extensionManager - ?.get(ExtensionTypeEnum.SystemMonitoring) - ?.getGpuSetting() + systemInformation: async (): Promise => { + const monitorExtension = extensionManager?.get( + ExtensionTypeEnum.SystemMonitoring + ) + if (!monitorExtension) { + console.warn('System monitoring extension not found') + return undefined + } + + const gpuSetting = await monitorExtension.getGpuSetting() + const osInfo = await monitorExtension.getOsInfo() return { gpuSetting, - // TODO: Other system information + osInfo, } }, + showToast: (title: string, description: string) => { toaster({ title, From 68572246e7ed9b9299045a9ee28a7b553307f4d8 Mon Sep 17 00:00:00 2001 From: Service Account Date: Tue, 19 Mar 2024 05:41:56 +0000 Subject: [PATCH 16/16] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6e5cc7c1e..ec05e93bb 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage