Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(Model): remove unsupported default model setting params (#1382)

Browse Source 
Signed-off-by: James <james@jan.ai>
Co-authored-by: James <james@jan.ai>
This commit is contained in:
NamH 2024-01-05 15:13:21 +07:00 committed by GitHub
parent e0a7d3cf58
commit 764e4c6fe3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
extensions/inference-nitro-extension/src/module.ts
View File

@ -24,19 +24,19 @@ const NVIDIA_INFO_FILE = path.join(
); );
const DEFALT_SETTINGS = { const DEFALT_SETTINGS = {
"notify": true, notify: true,
"run_mode": "cpu", run_mode: "cpu",
"nvidia_driver": { nvidia_driver: {
"exist": false, exist: false,
"version": "" version: "",
}, },
"cuda": { cuda: {
"exist": false, exist: false,
"version": "" version: "",
}, },
"gpus": [], gpus: [],
"gpu_highest_vram": "" gpu_highest_vram: "",
} };
// The subprocess instance for Nitro // The subprocess instance for Nitro
let subprocess = undefined; let subprocess = undefined;
@ -114,8 +114,7 @@ function updateCudaExistence() {
if (cudaExists) { if (cudaExists) {
cudaVersion = "11"; cudaVersion = "11";
} }
} } else {
else {
cudaVersion = "12"; cudaVersion = "12";
} }
@ -395,8 +394,7 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise<any> {
} else { } else {
if (nvidiaInfo["cuda"].version === "12") { if (nvidiaInfo["cuda"].version === "12") {
binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); binaryFolder = path.join(binaryFolder, "win-cuda-12-0");
} } else {
else {
binaryFolder = path.join(binaryFolder, "win-cuda-11-4"); binaryFolder = path.join(binaryFolder, "win-cuda-11-4");
} }
cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"];
@ -416,8 +414,7 @@ function spawnNitroProcess(nitroResourceProbe: any): Promise<any> {
} else { } else {
if (nvidiaInfo["cuda"].version === "12") { if (nvidiaInfo["cuda"].version === "12") {
binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); binaryFolder = path.join(binaryFolder, "linux-cuda-12-0");
} } else {
else {
binaryFolder = path.join(binaryFolder, "linux-cuda-11-4"); binaryFolder = path.join(binaryFolder, "linux-cuda-11-4");
} }
cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"];

16
models/config/default-model.json
View File

@ -11,18 +11,14 @@
"ctx_len": 4096, "ctx_len": 4096,
"ngl": 0, "ngl": 0,
"embedding": false, "embedding": false,
"n_parallel": 0, "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:"
"cpu_threads": 0,
"prompt_template": ""
}, },
"parameters": { "parameters": {
"temperature": 0, "temperature": 0.7,
"token_limit": 0, "top_p": 0.95,
"top_k": 0, "stream": true,
"top_p": 0, "max_tokens": 2048,
"stream": false, "stop": ["<endofstring>"],
"max_tokens": 4096,
"stop": [],
"frequency_penalty": 0, "frequency_penalty": 0,
"presence_penalty": 0 "presence_penalty": 0
}, },