diff --git a/electron/core/plugins/inference-plugin/index.js b/electron/core/plugins/inference-plugin/index.js index 9f9ec36cc..16ee2ca4f 100644 --- a/electron/core/plugins/inference-plugin/index.js +++ b/electron/core/plugins/inference-plugin/index.js @@ -17,7 +17,7 @@ const dispose = async () => .then((res) => resolve(res)); } }); -const inferenceUrl = () => "http://localhost:8080/llama/chat_completion"; +const inferenceUrl = () => "http://localhost:3928/llama/chat_completion"; // Register all the above functions and objects with the relevant extension points export function init({ register }) { diff --git a/electron/core/plugins/inference-plugin/module.js b/electron/core/plugins/inference-plugin/module.js index 2d6b8ca5c..05906fa88 100644 --- a/electron/core/plugins/inference-plugin/module.js +++ b/electron/core/plugins/inference-plugin/module.js @@ -51,12 +51,12 @@ async function initModel(product) { let binaryName; if (process.platform === "win32") { - binaryName = "nitro.exe"; + binaryName = "nitro_windows_amd64.exe"; } else if (process.platform === "darwin") { // Mac OS platform - binaryName = process.arch === "arm64" ? "nitro" : "nitro_mac_intel"; + binaryName = process.arch === "arm64" ? "nitro_mac_arm64" : "nitro_mac_amd64"; } else { // Linux - binaryName = "nitro_linux"; // For other platforms + binaryName = "nitro_linux_amd64_cuda"; // For other platforms } const binaryPath = path.join(binaryFolder, binaryName); diff --git a/electron/core/plugins/inference-plugin/nitro/config/config.json b/electron/core/plugins/inference-plugin/nitro/config/config.json index 230918933..1ec0e4cf6 100644 --- a/electron/core/plugins/inference-plugin/nitro/config/config.json +++ b/electron/core/plugins/inference-plugin/nitro/config/config.json @@ -1 +1,13 @@ -{"custom_config": {"llama_model_path":"","ctx_len":2048,"ngl":100}} +{ + "listeners": [ + { + "address": "0.0.0.0", + "port": 3928 + } + ], + "custom_config": { + "llama_model_path": "", + "ctx_len": 2048, + "ngl": 100 + } +} diff --git a/electron/core/plugins/inference-plugin/nitro/nitro_linux b/electron/core/plugins/inference-plugin/nitro/nitro_linux deleted file mode 100755 index 380c8ab0d..000000000 Binary files a/electron/core/plugins/inference-plugin/nitro/nitro_linux and /dev/null differ diff --git a/electron/core/plugins/inference-plugin/nitro/nitro_linux_amd64_cuda b/electron/core/plugins/inference-plugin/nitro/nitro_linux_amd64_cuda new file mode 100755 index 000000000..8b6afc37a Binary files /dev/null and b/electron/core/plugins/inference-plugin/nitro/nitro_linux_amd64_cuda differ diff --git a/electron/core/plugins/inference-plugin/nitro/nitro_mac_amd64 b/electron/core/plugins/inference-plugin/nitro/nitro_mac_amd64 new file mode 100755 index 000000000..648801ae7 Binary files /dev/null and b/electron/core/plugins/inference-plugin/nitro/nitro_mac_amd64 differ diff --git a/electron/core/plugins/inference-plugin/nitro/nitro b/electron/core/plugins/inference-plugin/nitro/nitro_mac_arm64 similarity index 62% rename from electron/core/plugins/inference-plugin/nitro/nitro rename to electron/core/plugins/inference-plugin/nitro/nitro_mac_arm64 index 77ad3a521..803c7cbc2 100755 Binary files a/electron/core/plugins/inference-plugin/nitro/nitro and b/electron/core/plugins/inference-plugin/nitro/nitro_mac_arm64 differ diff --git a/electron/core/plugins/inference-plugin/nitro/nitro_mac_intel b/electron/core/plugins/inference-plugin/nitro/nitro_mac_intel deleted file mode 100755 index 2f55b4c86..000000000 Binary files a/electron/core/plugins/inference-plugin/nitro/nitro_mac_intel and /dev/null differ diff --git a/electron/core/plugins/inference-plugin/nitro/nitro.exe b/electron/core/plugins/inference-plugin/nitro/nitro_windows_amd64.exe similarity index 52% rename from electron/core/plugins/inference-plugin/nitro/nitro.exe rename to electron/core/plugins/inference-plugin/nitro/nitro_windows_amd64.exe index d0fbcb117..89a6551da 100644 Binary files a/electron/core/plugins/inference-plugin/nitro/nitro.exe and b/electron/core/plugins/inference-plugin/nitro/nitro_windows_amd64.exe differ