Inference Plugin pull nitro binary from release (#524)

* refactor nitro folder structure inside inference plugin

* Change to flat folder structure for nitro binary

* Remove all exiting nitro binary file

* add download command for nitron linux cuda

* Add yarn script to download nitro

* Add build deps to data-plugins yarn script

* add chmod for nitro binary file

---------

Co-authored-by: Hien To <tominhhien97@gmail.com>
Co-authored-by: Service Account <service@jan.ai>

.gitignore

@@ -16,3 +16,9 @@ package-lock.json
 *.log
 plugin-core/lib
 core/lib/**
+
+# Nitro binary files
+plugins/inference-plugin/nitro/*/nitro
+plugins/inference-plugin/nitro/*/*.exe
+plugins/inference-plugin/nitro/*/*.dll
+plugins/inference-plugin/nitro/*/*.metal

package.json

@@ -28,7 +28,7 @@
     "build:electron": "yarn workspace jan build",
     "build:electron:test": "yarn workspace jan build:test",
     "build:pull-plugins": "rimraf ./electron/core/pre-install/*.tgz && cd ./electron/core/pre-install && npm pack @janhq/inference-plugin @janhq/data-plugin @janhq/model-management-plugin @janhq/monitoring-plugin",
-    "build:plugins": "rimraf ./electron/core/pre-install/*.tgz && concurrently --kill-others-on-fail \"cd ./plugins/data-plugin && npm install && npm run postinstall\" \"cd ./plugins/inference-plugin && npm install && npm run postinstall\" \"cd ./plugins/model-management-plugin && npm install && npm run postinstall\" \"cd ./plugins/monitoring-plugin && npm install && npm run postinstall\" && concurrently --kill-others-on-fail \"cd ./plugins/data-plugin && npm run build:publish\" \"cd ./plugins/inference-plugin && npm run build:publish\" \"cd ./plugins/model-management-plugin && npm run build:publish\" \"cd ./plugins/monitoring-plugin && npm run build:publish\"",
+    "build:plugins": "rimraf ./electron/core/pre-install/*.tgz && concurrently --kill-others-on-fail \"cd ./plugins/data-plugin && npm install && npm run build:deps && npm run postinstall\" \"cd ./plugins/inference-plugin && npm install && npm run postinstall\" \"cd ./plugins/model-management-plugin && npm install && npm run postinstall\" \"cd ./plugins/monitoring-plugin && npm install && npm run postinstall\" && concurrently --kill-others-on-fail \"cd ./plugins/data-plugin && npm run build:publish\" \"cd ./plugins/inference-plugin && npm run build:publish\" \"cd ./plugins/model-management-plugin && npm run build:publish\" \"cd ./plugins/monitoring-plugin && npm run build:publish\"",
     "build": "yarn build:web && yarn build:electron",
     "build:test": "yarn build:web && yarn build:electron:test",
     "build:test-darwin": "yarn build:web && yarn workspace jan build:test-darwin",

plugins/inference-plugin/module.ts

@@ -25,14 +25,19 @@ const initModel = (fileName) => {
 
         if (process.platform === "win32") {
           // Todo: Need to check for CUDA support to switch between CUDA and non-CUDA binaries
-          binaryName = "nitro_start_windows.bat";
+          binaryName = "win-start.bat";
         } else if (process.platform === "darwin") {
           // Mac OS platform
-          binaryName = process.arch === "arm64" ? "nitro_mac_arm64" : "nitro_mac_intel";
+          if (process.arch === "arm64") {
+            binaryFolder = path.join(binaryFolder, "mac-arm64")
+          } else {
+            binaryFolder = path.join(binaryFolder, "mac-x64")
+          }
+          binaryName = "nitro"
         } else {
           // Linux
           // Todo: Need to check for CUDA support to switch between CUDA and non-CUDA binaries
-          binaryName = "nitro_start_linux.sh"; // For other platforms
+          binaryName = "linux-start.sh"; // For other platforms
         }
 
             const binaryPath = path.join(binaryFolder, binaryName);

plugins/inference-plugin/nitro/linux-start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Attempt to run the nitro_linux_amd64_cuda file and if it fails, run nitro_linux_amd64
+cd linux-cuda
+./nitro "$@" || (echo "nitro_linux_amd64_cuda encountered an error, attempting to run nitro_linux_amd64..." && cd ../linux-cpu && ./nitro "$@")

plugins/inference-plugin/nitro/nitro_start_linux.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-#!/bin/bash
-
-# Attempt to run the nitro_linux_amd64_cuda file and if it fails, run nitro_linux_amd64
-./nitro_linux_amd64_cuda "$@" || (echo "nitro_linux_amd64_cuda encountered an error, attempting to run nitro_linux_amd64..." && ./nitro_linux_amd64 "$@")

plugins/inference-plugin/nitro/version.txt

@@ -0,0 +1 @@
+0.1.3

plugins/inference-plugin/nitro/win-start.bat

@@ -1,10 +1,12 @@
 @echo off
 
 rem Attempt to run nitro_windows_amd64_cuda.exe
-nitro_windows_amd64_cuda.exe
+cd win-cuda
+nitro.exe
 
 rem Check the exit code of the previous command
 if %errorlevel% neq 0 (
     echo nitro_windows_amd64_cuda.exe encountered an error, attempting to run nitro_windows_amd64.exe...
-    nitro_windows_amd64.exe
+    cd ..\win-cpu
+    nitro.exe
 )

plugins/inference-plugin/package.json

@@ -14,7 +14,13 @@
   ],
   "scripts": {
     "build": "tsc -b . && webpack --config webpack.config.js",
-    "postinstall": "rimraf *.tgz --glob && npm run build && rimraf dist/nitro/* && cpx \"nitro/**\" \"dist/nitro\"",
+    "downloadnitro:linux-cpu": "NITRO_VERSION=$(cat ./nitro/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.zip -e --strip 1 -o ./nitro/linux-cpu && chmod +x ./nitro/linux-cpu/nitro",
+    "downloadnitro:linux-cuda": "NITRO_VERSION=$(cat ./nitro/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda.zip -e --strip 1 -o ./nitro/linux-cuda && chmod +x ./nitro/linux-cuda/nitro",
+    "downloadnitro:mac-arm64": "NITRO_VERSION=$(cat ./nitro/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.zip -e --strip 1 -o ./nitro/mac-arm64 && chmod +x ./nitro/mac-arm64/nitro",
+    "downloadnitro:mac-x64": "NITRO_VERSION=$(cat ./nitro/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.zip -e --strip 1 -o ./nitro/mac-x64 && chmod +x ./nitro/mac-x64/nitro",
+    "downloadnitro:win-cpu": "NITRO_VERSION=$(cat ./nitro/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-win-amd64.zip -e --strip 1 -o ./nitro/win-cpu",
+    "downloadnitro:win-cuda": "NITRO_VERSION=$(cat ./nitro/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-win-amd64-cuda.zip -e --strip 1 -o ./nitro/win-cuda",
+    "postinstall": "rimraf *.tgz --glob && npm run build && npm run downloadnitro:linux-cpu && npm run downloadnitro:linux-cuda && npm run downloadnitro:mac-arm64 && npm run downloadnitro:mac-x64 && npm run downloadnitro:win-cpu && npm run downloadnitro:win-cuda && rimraf dist/nitro/* && cpx \"nitro/**\" \"dist/nitro\"",
     "build:publish": "npm pack && cpx *.tgz ../../electron/core/pre-install"
   },
   "exports": {
@@ -29,6 +35,9 @@
   },
   "dependencies": {
     "@janhq/core": "^0.1.6",
+    "cross-env": "^7.0.3",
+    "decompress-cli": "^2.0.0",
+    "download-cli": "^1.1.1",
     "kill-port": "^2.0.1",
     "rxjs": "^7.8.1",
     "tcp-port-used": "^1.0.2",