From f11a59bece1d2bc348f00299ff955125c39c75da Mon Sep 17 00:00:00 2001 From: hiento09 <136591877+hiento09@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:53:21 +0700 Subject: [PATCH] Add detect cuda version (#1351) Co-authored-by: Hien To --- .../inference-nitro-extension/bin/version.txt | 2 +- .../inference-nitro-extension/download.bat | 2 +- .../inference-nitro-extension/package.json | 2 +- .../inference-nitro-extension/src/module.ts | 55 +++++++++++++------ 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/extensions/inference-nitro-extension/bin/version.txt b/extensions/inference-nitro-extension/bin/version.txt index 28d007539..9dd179330 100644 --- a/extensions/inference-nitro-extension/bin/version.txt +++ b/extensions/inference-nitro-extension/bin/version.txt @@ -1 +1 @@ -0.1.32 +0.1.34 diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat index 1776b7dfe..f82b49c2e 100644 --- a/extensions/inference-nitro-extension/download.bat +++ b/extensions/inference-nitro-extension/download.bat @@ -1,3 +1,3 @@ @echo off set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda.tar.gz -e --strip 1 -o ./bin/win-cuda && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-4.tar.gz -e --strip 1 -o ./bin/win-cuda-11-4 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json index a35c51e58..479cc51dc 100644 --- a/extensions/inference-nitro-extension/package.json +++ b/extensions/inference-nitro-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda.tar.gz -e --strip 1 -o ./bin/linux-cuda && chmod +x ./bin/linux-cuda/nitro", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-4.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-4 && chmod +x ./bin/linux-cuda-11-4/nitro", "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", "downloadnitro:win32": "download.bat", "downloadnitro": "run-script-os", diff --git a/extensions/inference-nitro-extension/src/module.ts b/extensions/inference-nitro-extension/src/module.ts index 7678bae19..44c6b9100 100644 --- a/extensions/inference-nitro-extension/src/module.ts +++ b/extensions/inference-nitro-extension/src/module.ts @@ -85,28 +85,40 @@ function checkFileExistenceInPaths(file: string, paths: string[]): boolean { } function updateCudaExistence() { - let files: string[]; + let filesCuda12: string[]; + let filesCuda11: string[]; let paths: string[]; + let cudaVersion: string = ""; if (process.platform === "win32") { - files = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; + filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; + filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; - const nitro_cuda_path = path.join(__dirname, "bin", "win-cuda"); - paths.push(nitro_cuda_path); } else { - files = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; + filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; + filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; paths = process.env.LD_LIBRARY_PATH ? process.env.LD_LIBRARY_PATH.split(path.delimiter) : []; - const nitro_cuda_path = path.join(__dirname, "bin", "linux-cuda"); - paths.push(nitro_cuda_path); paths.push("/usr/lib/x86_64-linux-gnu/"); } - let cudaExists = files.every( + let cudaExists = filesCuda12.every( (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) ); + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ); + if (cudaExists) { + cudaVersion = "11"; + } + } + else { + cudaVersion = "12"; + } + let data; try { data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); @@ -115,6 +127,7 @@ function updateCudaExistence() { } data["cuda"].exist = cudaExists; + data["cuda"].version = cudaVersion; if (cudaExists) { data.run_mode = "gpu"; } @@ -376,12 +389,17 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise { let cudaVisibleDevices = ""; let binaryName; if (process.platform === "win32") { - let nvida_info = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvida_info["run_mode"] === "cpu") { + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + if (nvidiaInfo["run_mode"] === "cpu") { binaryFolder = path.join(binaryFolder, "win-cpu"); } else { - binaryFolder = path.join(binaryFolder, "win-cuda"); - cudaVisibleDevices = nvida_info["gpu_highest_vram"]; + if (nvidiaInfo["cuda"].version === "12") { + binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); + } + else { + binaryFolder = path.join(binaryFolder, "win-cuda-11-4"); + } + cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; } binaryName = "nitro.exe"; } else if (process.platform === "darwin") { @@ -392,12 +410,17 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise { } binaryName = "nitro"; } else { - let nvida_info = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvida_info["run_mode"] === "cpu") { + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + if (nvidiaInfo["run_mode"] === "cpu") { binaryFolder = path.join(binaryFolder, "linux-cpu"); } else { - binaryFolder = path.join(binaryFolder, "linux-cuda"); - cudaVisibleDevices = nvida_info["gpu_highest_vram"]; + if (nvidiaInfo["cuda"].version === "12") { + binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); + } + else { + binaryFolder = path.join(binaryFolder, "linux-cuda-11-4"); + } + cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; } binaryName = "nitro"; }