Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into Ssstars-patch-1

Browse Source
This commit is contained in:
Hieu 2024-01-12 12:31:30 +07:00 committed by GitHub
commit 0557a0bd57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 258 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/jan-electron-linter-and-test.yml vendored
View File

@ -63,7 +63,10 @@ jobs:
CSC_IDENTITY_AUTO_DISCOVERY: "false"
test-on-windows:
runs-on: [self-hosted, Windows, windows-desktop]
strategy:
matrix:
antivirus-tools: ['mcafee', 'default-windows-security','bit-defender']
runs-on: windows-desktop-${{ matrix.antivirus-tools }}
steps:
- name: Clean workspace
run: |

14
README.md
View File

@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute
<tr style="text-align:center">
<td style="text-align:center"><b>Experimental (Nightly Build)</b></td>
<td style="text-align:center">
<a href='https://delta.jan.ai/0.4.3-139/jan-win-x64-0.4.3-139.exe'>
<a href='https://delta.jan.ai/0.4.3-140/jan-win-x64-0.4.3-140.exe'>
<img src='./docs/static/img/windows.png' style="height:14px; width: 14px" />
<b>jan.exe</b>
</a>
</td>
<td style="text-align:center">
<a href='https://delta.jan.ai/0.4.3-139/jan-mac-x64-0.4.3-139.dmg'>
<a href='https://delta.jan.ai/0.4.3-140/jan-mac-x64-0.4.3-140.dmg'>
<img src='./docs/static/img/mac.png' style="height:15px; width: 15px" />
<b>Intel</b>
</a>
</td>
<td style="text-align:center">
<a href='https://delta.jan.ai/0.4.3-139/jan-mac-arm64-0.4.3-139.dmg'>
<a href='https://delta.jan.ai/0.4.3-140/jan-mac-arm64-0.4.3-140.dmg'>
<img src='./docs/static/img/mac.png' style="height:15px; width: 15px" />
<b>M1/M2</b>
</a>
</td>
<td style="text-align:center">
<a href='https://delta.jan.ai/0.4.3-139/jan-linux-amd64-0.4.3-139.deb'>
<a href='https://delta.jan.ai/0.4.3-140/jan-linux-amd64-0.4.3-140.deb'>
<img src='./docs/static/img/linux.png' style="height:14px; width: 14px" />
<b>jan.deb</b>
</a>
</td>
<td style="text-align:center">
<a href='https://delta.jan.ai/0.4.3-139/jan-linux-x86_64-0.4.3-139.AppImage'>
<a href='https://delta.jan.ai/0.4.3-140/jan-linux-x86_64-0.4.3-140.AppImage'>
<img src='./docs/static/img/linux.png' style="height:14px; width: 14px" />
<b>jan.AppImage</b>
</a>
@ -171,13 +171,13 @@ To reset your installation:
- Windows:
- Windows 10 or higher
- To enable GPU support:
- Nvidia GPU with CUDA Toolkit 11.4 or higher
- Nvidia GPU with CUDA Toolkit 11.7 or higher
- Nvidia driver 470.63.01 or higher
- Linux:
- glibc 2.27 or higher (check with `ldd --version`)
- gcc 11, g++ 11, cpp 11 or higher, refer to this [link](https://jan.ai/guides/troubleshooting/gpu-not-used/#specific-requirements-for-linux) for more information
- To enable GPU support:
- Nvidia GPU with CUDA Toolkit 11.4 or higher
- Nvidia GPU with CUDA Toolkit 11.7 or higher
- Nvidia driver 470.63.01 or higher
## Contributing

80
docs/docs/guides/07-integrations/02-integrate-openrouter.mdx Normal file
View File

@ -0,0 +1,80 @@
---
title: Integrate OpenRouter with Jan
slug: /guides/integrations/openrouter
description: Guide to integrate OpenRouter with Jan
keywords:
[
Jan AI,
Jan,
ChatGPT alternative,
local AI,
private AI,
conversational AI,
no-subscription fee,
large language model,
OpenRouter integration,
]
---
## Quick Introduction
[OpenRouter](https://openrouter.ai/docs#quick-start) is an AI model aggregator. The API can be used by developers to interact with a variety of large language models, generative image models, and generative 3D object models.
In this guide, we will show you how to integrate OpenRouter with Jan, enabling you to leverage remote Large Language Models (LLM) that are available at OpenRouter.
## Steps to Integrate OpenRouter with Jan
### 1. Configure OpenRouter API key
You can find your API keys in the [OpenRouter API Key](https://openrouter.ai/keys) and set the OpenRouter API key in `~/jan/engines/openai.json` file.
```json title="~/jan/engines/openai.json"
{
// highlight-start
"full_url": "https://openrouter.ai/api/v1/chat/completions",
"api_key": "sk-or-v1<your-openrouter-api-key-here>"
// highlight-end
}
```
### 2. Mofidy a Model JSON
Navigate to the `~/jan/models` folder. Create a folder named `<openrouter-modelname>`, for example, `openrouter-dolphin-mixtral-8x7b` and create a `model.json` file inside the folder including the following configurations:
- Ensure the filename must be `model.json`.
- Ensure the `id` property is set to the model id from OpenRouter.
- Ensure the `format` property is set to `api`.
- Ensure the `engine` property is set to `openai`.
- Ensure the `state` property is set to `ready`.
```json title="~/jan/models/openrouter-dolphin-mixtral-8x7b/model.json"
{
"source_url": "https://openrouter.ai/",
"id": "cognitivecomputations/dolphin-mixtral-8x7b",
"object": "model",
"name": "Dolphin 2.6 Mixtral 8x7B",
"version": "1.0",
"description": "This is a 16k context fine-tune of Mixtral-8x7b. It excels in coding tasks due to extensive training with coding data and is known for its obedience, although it lacks DPO tuning. The model is uncensored and is stripped of alignment and bias. It requires an external alignment layer for ethical use. Users are cautioned to use this highly compliant model responsibly, as detailed in a blog post about uncensored models at erichartford.com/uncensored-models.",
// highlight-next-line
"format": "api",
"settings": {},
"parameters": {},
"metadata": {
"tags": ["General", "Big Context Length"]
},
// highlight-start
"engine": "openai",
"state": "ready"
// highlight-end
}
```
### 3. Start the Model
Restart Jan and navigate to the Hub. Locate your model and click the Use button.
![Dolphin OpenRouter Model](assets/02-openrouter-dolphin.png)
### 4. Try Out the Integration of Jan and OpenRouter
![Dolphin OpenRouter Model Trial](assets/02-openrouter-dolphin-trial.gif)

BIN
docs/docs/guides/07-integrations/assets/02-openrouter-dolphin-trial.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 MiB

BIN
docs/docs/guides/07-integrations/assets/02-openrouter-dolphin.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.3 MiB

14
docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx
View File

@ -23,7 +23,7 @@ This guide provides steps to troubleshoot and resolve issues when Jan app does n
### NVIDIA Driver
Ensure that you have installed the NVIDIA driver that supports CUDA 11.4 or higher. For a detailed of CUDA compatibility, please refer [here](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver).
Ensure that you have installed the NVIDIA driver that supports CUDA 11.7 or higher. For a detailed of CUDA compatibility, please refer [here](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver).
To verify, open PowerShell or Terminal and enter the following command:
@ -35,7 +35,7 @@ If you see a result similar to the following, you have successfully installed th
```bash
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 470.57.02 Driver Version: 470.57.02 CUDA Version: 11.4 |
| NVIDIA-SMI 470.57.02 Driver Version: 470.57.02 CUDA Version: 11.7 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
@ -62,7 +62,7 @@ If you see a result similar to the following, you have successfully installed CU
```bash
nvcc: NVIDIA (R) Cuda compiler driver
Cuda compilation tools, release 11.4, V11.7.100
Cuda compilation tools, release 11.7, V11.7.100
Build cuda_11.7.r11.7/compiler.30033411_0
```
@ -93,11 +93,11 @@ g++ --version
**Post-Installation Actions**: You must add the `.so` libraries of CUDA to the `LD_LIBRARY_PATH` environment variable by following the [Post-installation Actions instruction](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#post-installation-actions).
```bash
# Example for ubuntu with CUDA 11.4
# Example for ubuntu with CUDA 11.7
sudo nano /etc/environment
# Add /usr/local/cuda-11.4/bin to the PATH environment variable - the first line
# Add /usr/local/cuda-11.7/bin to the PATH environment variable - the first line
# Add the following line to the end of the file
LD_LIBRARY_PATH=/usr/local/cuda-11.4/lib64
LD_LIBRARY_PATH=/usr/local/cuda-11.7/lib64
# Save and exit
# Restart your computer or log out and log in again, the changes will take effect
@ -113,7 +113,7 @@ If you find that GPU mode is available but not enabled by default, consider the
:::tip
1. Check if you have installed the NVIDIA driver that supports CUDA 11.4 or higher. For a detailed of CUDA compatibility, please refer [here](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver).
1. Check if you have installed the NVIDIA driver that supports CUDA 11.7 or higher. For a detailed of CUDA compatibility, please refer [here](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver).
2. Ensure that the CUDA toolkit is installed and compatible with your NVIDIA driver. For a detailed of CUDA compatibility, please refer [here](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#binary-compatibility__table-toolkit-driver).

2
docs/docusaurus.config.js
View File

@ -187,7 +187,7 @@ const config = {
},
{ property: "og:type", content: "website" },
{ property: "twitter:card", content: "summary_large_image" },
{ property: "twitter:site", content: "@janhq_" },
{ property: "twitter:site", content: "@janframework" },
{
property: "twitter:title",
content: "Jan | Open-source ChatGPT Alternative",

2
docs/src/containers/Banner/index.js
View File

@ -9,7 +9,7 @@ import { BiLogoDiscordAlt } from "react-icons/bi";
const socials = [
{
icon: <AiOutlineTwitter className="text-xl text-white" />,
href: "https://twitter.com/janhq_",
href: "https://twitter.com/janframework",
},
{
icon: <BiLogoDiscordAlt className="text-xl text-white" />,

4
docs/src/containers/Footer/index.js
View File

@ -6,7 +6,7 @@ import { BiLogoDiscordAlt } from "react-icons/bi";
const socials = [
{
icon: <AiOutlineTwitter className="text-xl text-black dark:text-white" />,
href: "https://twitter.com/janhq_",
href: "https://twitter.com/janframework",
},
{
icon: <BiLogoDiscordAlt className="text-xl text-black dark:text-white" />,
@ -56,7 +56,7 @@ const menus = [
},
{
menu: "Twitter",
path: "https://twitter.com/janhq_",
path: "https://twitter.com/janframework",
external: true,
},
],

2
extensions/inference-nitro-extension/bin/version.txt
View File

@ -1 +1 @@
0.2.6
0.2.7

2
extensions/inference-nitro-extension/download.bat
View File

@ -1,3 +1,3 @@
@echo off
set /p NITRO_VERSION=<./bin/version.txt
.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-4.tar.gz -e --strip 1 -o ./bin/win-cuda-11-4 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu
.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu

2
extensions/inference-nitro-extension/package.json
View File

@ -8,7 +8,7 @@
"license": "AGPL-3.0",
"scripts": {
"build": "tsc -b . && webpack --config webpack.config.js",
"downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-4.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-4 && chmod +x ./bin/linux-cuda-11-4/nitro",
"downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro",
"downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
"downloadnitro:win32": "download.bat",
"downloadnitro": "run-script-os",

138
extensions/inference-nitro-extension/src/helpers/nvidia.ts
View File

@ -1,138 +0,0 @@
/**
* Default GPU settings
**/
const DEFALT_SETTINGS = {
notify: true,
run_mode: "cpu",
nvidia_driver: {
exist: false,
version: "",
},
cuda: {
exist: false,
version: "",
},
gpus: [],
gpu_highest_vram: "",
};
/**
* Validate nvidia and cuda for linux and windows
*/
async function updateNvidiaDriverInfo(): Promise<void> {
exec(
"nvidia-smi --query-gpu=driver_version --format=csv,noheader",
(error, stdout) => {
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} catch (error) {
data = DEFALT_SETTINGS;
}
if (!error) {
const firstLine = stdout.split("\n")[0].trim();
data["nvidia_driver"].exist = true;
data["nvidia_driver"].version = firstLine;
} else {
data["nvidia_driver"].exist = false;
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
Promise.resolve();
}
);
}
function checkFileExistenceInPaths(file: string, paths: string[]): boolean {
return paths.some((p) => existsSync(path.join(p, file)));
}
function updateCudaExistence() {
let filesCuda12: string[];
let filesCuda11: string[];
let paths: string[];
let cudaVersion: string = "";
if (process.platform === "win32") {
filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"];
filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"];
paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [];
} else {
filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"];
filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"];
paths = process.env.LD_LIBRARY_PATH
? process.env.LD_LIBRARY_PATH.split(path.delimiter)
: [];
paths.push("/usr/lib/x86_64-linux-gnu/");
}
let cudaExists = filesCuda12.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
if (!cudaExists) {
cudaExists = filesCuda11.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
if (cudaExists) {
cudaVersion = "11";
}
} else {
cudaVersion = "12";
}
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} catch (error) {
data = DEFALT_SETTINGS;
}
data["cuda"].exist = cudaExists;
data["cuda"].version = cudaVersion;
if (cudaExists) {
data.run_mode = "gpu";
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
}
async function updateGpuInfo(): Promise<void> {
exec(
"nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits",
(error, stdout) => {
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} catch (error) {
data = DEFALT_SETTINGS;
}
if (!error) {
// Get GPU info and gpu has higher memory first
let highestVram = 0;
let highestVramId = "0";
let gpus = stdout
.trim()
.split("\n")
.map((line) => {
let [id, vram] = line.split(", ");
vram = vram.replace(/\r/g, "");
if (parseFloat(vram) > highestVram) {
highestVram = parseFloat(vram);
highestVramId = id;
}
return { id, vram };
});
data["gpus"] = gpus;
data["gpu_highest_vram"] = highestVramId;
} else {
data["gpus"] = [];
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
Promise.resolve();
}
);
}

152
extensions/inference-nitro-extension/src/module.ts
View File

@ -29,6 +29,24 @@ let currentSettings = undefined;
let nitroProcessInfo = undefined;
/**
* Default GPU settings
**/
const DEFALT_SETTINGS = {
notify: true,
run_mode: "cpu",
nvidia_driver: {
exist: false,
version: "",
},
cuda: {
exist: false,
version: "",
},
gpus: [],
gpu_highest_vram: "",
};
/**
* Stops a Nitro subprocess.
* @param wrapper - The model wrapper.
@ -248,7 +266,7 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise<any> {
if (nvidiaInfo["cuda"].version === "12") {
binaryFolder = path.join(binaryFolder, "win-cuda-12-0");
} else {
binaryFolder = path.join(binaryFolder, "win-cuda-11-4");
binaryFolder = path.join(binaryFolder, "win-cuda-11-7");
}
cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"];
}
@ -268,7 +286,7 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise<any> {
if (nvidiaInfo["cuda"].version === "12") {
binaryFolder = path.join(binaryFolder, "linux-cuda-12-0");
} else {
binaryFolder = path.join(binaryFolder, "linux-cuda-11-4");
binaryFolder = path.join(binaryFolder, "linux-cuda-11-7");
}
cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"];
}
@ -356,6 +374,136 @@ function dispose() {
killSubprocess();
}
/**
* Validate nvidia and cuda for linux and windows
*/
async function updateNvidiaDriverInfo(): Promise<void> {
exec(
"nvidia-smi --query-gpu=driver_version --format=csv,noheader",
(error, stdout) => {
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} catch (error) {
data = DEFALT_SETTINGS;
}
if (!error) {
const firstLine = stdout.split("\n")[0].trim();
data["nvidia_driver"].exist = true;
data["nvidia_driver"].version = firstLine;
} else {
data["nvidia_driver"].exist = false;
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
Promise.resolve();
}
);
}
/**
* Check if file exists in paths
*/
function checkFileExistenceInPaths(file: string, paths: string[]): boolean {
return paths.some((p) => existsSync(path.join(p, file)));
}
/**
* Validate cuda for linux and windows
*/
function updateCudaExistence() {
let filesCuda12: string[];
let filesCuda11: string[];
let paths: string[];
let cudaVersion: string = "";
if (process.platform === "win32") {
filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"];
filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"];
paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [];
} else {
filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"];
filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"];
paths = process.env.LD_LIBRARY_PATH
? process.env.LD_LIBRARY_PATH.split(path.delimiter)
: [];
paths.push("/usr/lib/x86_64-linux-gnu/");
}
let cudaExists = filesCuda12.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
if (!cudaExists) {
cudaExists = filesCuda11.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
if (cudaExists) {
cudaVersion = "11";
}
} else {
cudaVersion = "12";
}
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} catch (error) {
data = DEFALT_SETTINGS;
}
data["cuda"].exist = cudaExists;
data["cuda"].version = cudaVersion;
if (cudaExists) {
data.run_mode = "gpu";
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
}
/**
* Get GPU information
*/
async function updateGpuInfo(): Promise<void> {
exec(
"nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits",
(error, stdout) => {
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
} catch (error) {
data = DEFALT_SETTINGS;
}
if (!error) {
// Get GPU info and gpu has higher memory first
let highestVram = 0;
let highestVramId = "0";
let gpus = stdout
.trim()
.split("\n")
.map((line) => {
let [id, vram] = line.split(", ");
vram = vram.replace(/\r/g, "");
if (parseFloat(vram) > highestVram) {
highestVram = parseFloat(vram);
highestVramId = id;
}
return { id, vram };
});
data["gpus"] = gpus;
data["gpu_highest_vram"] = highestVramId;
} else {
data["gpus"] = [];
}
writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2));
Promise.resolve();
}
);
}
module.exports = {
initModel,
stopModel,

21
web/screens/ExploreModels/ExploreModelItemHeader/index.tsx
View File

@ -14,7 +14,6 @@ import ModalCancelDownload from '@/containers/ModalCancelDownload'
import { MainViewState } from '@/constants/screens'
import { useActiveModel } from '@/hooks/useActiveModel'
import { useCreateNewThread } from '@/hooks/useCreateNewThread'
import useDownloadModel from '@/hooks/useDownloadModel'
import { useDownloadState } from '@/hooks/useDownloadState'
@ -24,7 +23,7 @@ import { useMainViewState } from '@/hooks/useMainViewState'
import { toGibibytes } from '@/utils/converter'
import { totalRamAtom, usedRamAtom } from '@/helpers/atoms/SystemBar.atom'
import { totalRamAtom } from '@/helpers/atoms/SystemBar.atom'
type Props = {
model: Model
@ -38,7 +37,6 @@ const ExploreModelItemHeader: React.FC<Props> = ({ model, onClick, open }) => {
const { modelDownloadStateAtom, downloadStates } = useDownloadState()
const { requestCreateNewThread } = useCreateNewThread()
const totalRam = useAtomValue(totalRamAtom)
const usedRam = useAtomValue(usedRamAtom)
const downloadAtom = useMemo(
() => atom((get) => get(modelDownloadStateAtom)[model.id]),
@ -84,33 +82,20 @@ const ExploreModelItemHeader: React.FC<Props> = ({ model, onClick, open }) => {
downloadButton = <ModalCancelDownload model={model} />
}
const { activeModel } = useActiveModel()
const getLabel = (size: number) => {
const minimumRamModel = size * 1.25
const availableRam = totalRam - usedRam + (activeModel?.metadata.size ?? 0)
if (minimumRamModel > totalRam) {
if (size * 1.25 >= totalRam) {
return (
<Badge className="rounded-md" themes="danger">
Not enough RAM
</Badge>
)
}
if (minimumRamModel < availableRam) {
} else {
return (
<Badge className="rounded-md" themes="success">
Recommended
</Badge>
)
}
if (minimumRamModel < totalRam && minimumRamModel > availableRam) {
return (
<Badge className="rounded-md" themes="warning">
Slow on your device
</Badge>
)
}
}
return (