Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add detect cuda version (#1351)

Browse Source 
Co-authored-by: Hien To <tominhhien97@gmail.com>
This commit is contained in:
hiento09 2024-01-04 22:53:21 +07:00 committed by GitHub
parent f47bf36e70
commit f11a59bece
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 42 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/inference-nitro-extension/bin/version.txt
View File

@ -1 +1 @@
0.1.32
0.1.34

2
extensions/inference-nitro-extension/download.bat
View File

@ -1,3 +1,3 @@
@echo off
set /p NITRO_VERSION=<./bin/version.txt
.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda.tar.gz -e --strip 1 -o ./bin/win-cuda && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu
.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-4.tar.gz -e --strip 1 -o ./bin/win-cuda-11-4 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu

2
extensions/inference-nitro-extension/package.json
View File

@ -8,7 +8,7 @@
"license": "AGPL-3.0",
"scripts": {
"build": "tsc -b . && webpack --config webpack.config.js",
"downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda.tar.gz -e --strip 1 -o ./bin/linux-cuda && chmod +x ./bin/linux-cuda/nitro",
"downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-4.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-4 && chmod +x ./bin/linux-cuda-11-4/nitro",
"downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
"downloadnitro:win32": "download.bat",
"downloadnitro": "run-script-os",

55
extensions/inference-nitro-extension/src/module.ts
View File

@ -85,28 +85,40 @@ function checkFileExistenceInPaths(file: string, paths: string[]): boolean {
}
function updateCudaExistence() {
let files: string[];
let filesCuda12: string[];
let filesCuda11: string[];
let paths: string[];
let cudaVersion: string = "";
if (process.platform === "win32") {
files = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"];
filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"];
filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"];
paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [];
const nitro_cuda_path = path.join(__dirname, "bin", "win-cuda");
paths.push(nitro_cuda_path);
} else {
files = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"];
filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"];
filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"];
paths = process.env.LD_LIBRARY_PATH
? process.env.LD_LIBRARY_PATH.split(path.delimiter)
: [];
const nitro_cuda_path = path.join(__dirname, "bin", "linux-cuda");
paths.push(nitro_cuda_path);
paths.push("/usr/lib/x86_64-linux-gnu/");
}
let cudaExists = files.every(
let cudaExists = filesCuda12.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
if (!cudaExists) {
cudaExists = filesCuda11.every(
(file) => existsSync(file) || checkFileExistenceInPaths(file, paths)
);
if (cudaExists) {
cudaVersion = "11";
}
}
else {
cudaVersion = "12";
}
let data;
try {
data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
@ -115,6 +127,7 @@ function updateCudaExistence() {
}
data["cuda"].exist = cudaExists;
data["cuda"].version = cudaVersion;
if (cudaExists) {
data.run_mode = "gpu";
}
@ -376,12 +389,17 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise<any> {
let cudaVisibleDevices = "";
let binaryName;
if (process.platform === "win32") {
let nvida_info = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
if (nvida_info["run_mode"] === "cpu") {
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
if (nvidiaInfo["run_mode"] === "cpu") {
binaryFolder = path.join(binaryFolder, "win-cpu");
} else {
binaryFolder = path.join(binaryFolder, "win-cuda");
cudaVisibleDevices = nvida_info["gpu_highest_vram"];
if (nvidiaInfo["cuda"].version === "12") {
binaryFolder = path.join(binaryFolder, "win-cuda-12-0");
}
else {
binaryFolder = path.join(binaryFolder, "win-cuda-11-4");
}
cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"];
}
binaryName = "nitro.exe";
} else if (process.platform === "darwin") {
@ -392,12 +410,17 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise<any> {
}
binaryName = "nitro";
} else {
let nvida_info = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
if (nvida_info["run_mode"] === "cpu") {
let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8"));
if (nvidiaInfo["run_mode"] === "cpu") {
binaryFolder = path.join(binaryFolder, "linux-cpu");
} else {
binaryFolder = path.join(binaryFolder, "linux-cuda");
cudaVisibleDevices = nvida_info["gpu_highest_vram"];
if (nvidiaInfo["cuda"].version === "12") {
binaryFolder = path.join(binaryFolder, "linux-cuda-12-0");
}
else {
binaryFolder = path.join(binaryFolder, "linux-cuda-11-4");
}
cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"];
}
binaryName = "nitro";
}