From bc4bbd32ab1a7dcaa7349a871202510d4b8e900a Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Tue, 27 Aug 2024 18:26:36 +0700 Subject: [PATCH 1/8] fix: wrong classname icon (#3474) * fix: fix wrong classname icon * fix: button flashing on chat input --- web/containers/ModelDropdown/index.tsx | 5 ++++- web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx | 6 +++++- web/screens/Thread/ThreadLeftPanel/index.tsx | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/web/containers/ModelDropdown/index.tsx b/web/containers/ModelDropdown/index.tsx index 07c2cba3a..f8920a6dd 100644 --- a/web/containers/ModelDropdown/index.tsx +++ b/web/containers/ModelDropdown/index.tsx @@ -299,7 +299,10 @@ const ModelDropdown = ({ setOpen(!open)} > {selectedModel?.name} diff --git a/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx b/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx index ff3f19e16..e3d469a97 100644 --- a/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx @@ -388,7 +388,11 @@ const ChatInput = () => { { > - + Edit title From 8232a1e0c2bbc5a83ed8f35bfa4a924dba17c938 Mon Sep 17 00:00:00 2001 From: Van Pham <64197333+Van-QA@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:44:24 +0700 Subject: [PATCH 2/8] Update model.json --- .../resources/models/gemma-1.1-2b/model.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json index 56cd9c81c..837b10ce3 100644 --- a/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json +++ b/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json @@ -1,7 +1,7 @@ { "sources": [ { - "filename": "gemma-1.1-2b-it-q4_k_m.gguf", + "filename": "gemma-1.1-2b-it-Q4_K_M.gguf", "url": "https://huggingface.co/bartowski/gemma-1.1-2b-it-GGUF/resolve/main/gemma-1.1-2b-it-Q4_K_M.gguf" } ], From 47b074af471bbcb620b2e80c0c2ba2040a070370 Mon Sep 17 00:00:00 2001 From: Van Pham <64197333+Van-QA@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:46:20 +0700 Subject: [PATCH 3/8] Update model.json --- .../resources/models/gemma-1.1-7b/model.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json index 5bd89b478..a532c1dc3 100644 --- a/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json +++ b/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json @@ -1,7 +1,7 @@ { "sources": [ { - "filename": "gemma-1.1-7b-it-q4_K_M.gguf", + "filename": "gemma-1.1-7b-it-Q4_K_M.gguf", "url": "https://huggingface.co/bartowski/gemma-1.1-7b-it-GGUF/resolve/main/gemma-1.1-7b-it-Q4_K_M.gguf" } ], @@ -14,7 +14,7 @@ "settings": { "ctx_len": 8192, "prompt_template": "user\n{prompt}\nmodel", - "llama_model_path": "gemma-1.1-7b-it-q4_K_M.gguf", + "llama_model_path": "gemma-1.1-7b-it-Q4_K_M.gguf", "ngl": 29 }, "parameters": { From 20f0e4301e2cb22b4c5d3b26bef33657b2025a48 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 28 Aug 2024 11:34:35 +0700 Subject: [PATCH 4/8] fix: add engine in payload for nitro chat completion api --- core/src/node/api/restful/helper/builder.ts | 5 +++++ web/hooks/useImportModel.ts | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index cd121cdb7..08da0ff33 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -341,6 +341,11 @@ export const chatCompletions = async (request: any, reply: any) => { request.body.stop = request.body.stop.slice(0, 4) } + // add engine for new cortex cpp engine + if (requestedModel.engine === 'nitro') { + request.body.engine = 'cortex.llamacpp' + } + const fetch = require('node-fetch') const response = await fetch(apiUrl, { method: 'POST', diff --git a/web/hooks/useImportModel.ts b/web/hooks/useImportModel.ts index b23f5a6fb..effc64f86 100644 --- a/web/hooks/useImportModel.ts +++ b/web/hooks/useImportModel.ts @@ -6,9 +6,7 @@ import { Model, ModelExtension, OptionType, - baseName, fs, - joinPath, } from '@janhq/core' import { atom, useSetAtom } from 'jotai' From 46aa9d3a29d93d402573c08112e5939a369c0094 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Wed, 28 Aug 2024 13:57:50 +0700 Subject: [PATCH 5/8] fix: responsive modal import model gguf --- .../ModelDownloadList/index.tsx | 2 +- .../ModelDownloadRow/index.tsx | 30 ++++++++++++------- .../ModelSegmentInfo/index.tsx | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx index e6285565f..e59ee3416 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx @@ -29,7 +29,7 @@ const ModelDownloadList = () => { return (

Available Versions

- + {ggufModels.map((model, index) => { if (!model.downloadUrl) return null return ( diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx index 98914c94e..951a11d59 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx @@ -10,6 +10,8 @@ import { Badge, Button, Progress } from '@janhq/joi' import { useAtomValue, useSetAtom } from 'jotai' +import { twMerge } from 'tailwind-merge' + import { MainViewState } from '@/constants/screens' import { useCreateNewThread } from '@/hooks/useCreateNewThread' @@ -114,16 +116,24 @@ const ModelDownloadRow: React.FC = ({ } return ( -
-
- {quantization && ( - - {quantization} - - )} -

- {fileName} -

+
+
+
+ {quantization && ( + + {quantization} + + )} +

+ {fileName} +

+
{toGibibytes(fileSize)} diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx index ba17e9b57..10f4515c8 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx @@ -33,7 +33,7 @@ const ModelSegmentInfo = () => { if (!importingHuggingFaceRepoData) return null return ( -
+

{modelName} From 31ae14c1f1aa7069ac822f801237d90518f19553 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Wed, 28 Aug 2024 17:04:47 +0700 Subject: [PATCH 6/8] chore: fix responsive and add tensorrt_llm nitro to model utils --- .../HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx | 2 +- web/utils/modelEngine.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx index e59ee3416..a1d153d10 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx @@ -29,7 +29,7 @@ const ModelDownloadList = () => { return (

Available Versions

- + {ggufModels.map((model, index) => { if (!model.downloadUrl) return null return ( diff --git a/web/utils/modelEngine.ts b/web/utils/modelEngine.ts index a12e9bb0e..3d132c5d5 100644 --- a/web/utils/modelEngine.ts +++ b/web/utils/modelEngine.ts @@ -4,6 +4,7 @@ export const getLogoEngine = (engine: InferenceEngine) => { switch (engine) { case InferenceEngine.anthropic: return 'images/ModelProvider/anthropic.svg' + case InferenceEngine.nitro_tensorrt_llm: case InferenceEngine.nitro: return 'images/ModelProvider/nitro.svg' case InferenceEngine.cortex_llamacpp: @@ -43,6 +44,8 @@ export const getTitleByEngine = (engine: InferenceEngine) => { switch (engine) { case InferenceEngine.nitro: return 'Llama.cpp (Nitro)' + case InferenceEngine.nitro_tensorrt_llm: + return 'TensorRT-LLM (Nitro)' case InferenceEngine.cortex_llamacpp: return 'Llama.cpp (Cortex)' case InferenceEngine.cortex_onnx: From e41f816fc8122c69458c5f4c291ae6de012addec Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Wed, 28 Aug 2024 17:24:57 +0700 Subject: [PATCH 7/8] chore: fix system monitor gpu list --- .../BottomPanel/SystemMonitor/index.tsx | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx index 7fb20b0a4..4fd902075 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx @@ -159,35 +159,40 @@ const SystemMonitor = () => { {gpus.length > 0 && (
- {gpus.map((gpu, index) => ( -
-
- - {gpu.name} - -
-
- - {gpu.memoryTotal - gpu.memoryFree}/ - {gpu.memoryTotal} - - MB + {gpus.map((gpu, index) => { + const gpuUtilization = Math.round( + (gpu.memoryFree / Math.max(gpu.memoryTotal, 1)) * 100 + ) + return ( +
+
+ + {gpu.name} + +
+
+ + {gpu.memoryTotal - gpu.memoryFree}/ + {gpu.memoryTotal} + + MB +
-
-
- - - {gpu.utilization}% - +
+ + + {gpuUtilization}% + +
-
- ))} + ) + })}
)}
From c10695be7c4a4eedf5b94fd4be2f26584bd2b206 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 29 Aug 2024 12:55:07 +0700 Subject: [PATCH 8/8] fix: utilized memory --- web/.eslintrc.js | 3 +++ .../BottomPanel/SystemMonitor/index.tsx | 7 +++++-- web/jest.config.js | 5 +++++ web/package.json | 12 ++++++++---- web/testRunner.js | 19 +++++++++++++++++++ web/tsconfig.json | 8 ++++++-- web/utils/memory.test.ts | 13 +++++++++++++ web/utils/memory.ts | 9 +++++++++ 8 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 web/jest.config.js create mode 100644 web/testRunner.js create mode 100644 web/utils/memory.test.ts create mode 100644 web/utils/memory.ts diff --git a/web/.eslintrc.js b/web/.eslintrc.js index 0ed19d001..73fb1bcf5 100644 --- a/web/.eslintrc.js +++ b/web/.eslintrc.js @@ -11,6 +11,9 @@ module.exports = { '_next', '*.md', 'out', + '**/*.test.tsx', + '**/*.test.ts', + 'testRunner.js', ], extends: [ 'next/core-web-vitals', diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx index 4fd902075..a69e34d57 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx @@ -19,6 +19,8 @@ import { usePath } from '@/hooks/usePath' import { toGibibytes } from '@/utils/converter' +import { utilizedMemory } from '@/utils/memory' + import TableActiveModel from './TableActiveModel' import { showSystemMonitorPanelAtom } from '@/helpers/atoms/App.atom' @@ -160,8 +162,9 @@ const SystemMonitor = () => { {gpus.length > 0 && (
{gpus.map((gpu, index) => { - const gpuUtilization = Math.round( - (gpu.memoryFree / Math.max(gpu.memoryTotal, 1)) * 100 + const gpuUtilization = utilizedMemory( + gpu.memoryFree, + gpu.memoryTotal ) return (
diff --git a/web/jest.config.js b/web/jest.config.js new file mode 100644 index 000000000..b34e410d2 --- /dev/null +++ b/web/jest.config.js @@ -0,0 +1,5 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + runner: './testRunner.js', +} diff --git a/web/package.json b/web/package.json index b24163bb7..3ba02c3b9 100644 --- a/web/package.json +++ b/web/package.json @@ -10,7 +10,8 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write \"**/*.{js,jsx,ts,tsx}\"", - "compile": "tsc --noEmit -p . --pretty" + "compile": "tsc --noEmit -p . --pretty", + "test": "jest" }, "dependencies": { "@heroicons/react": "^2.0.18", @@ -19,9 +20,9 @@ "@janhq/joi": "link:./joi", "autoprefixer": "10.4.16", "class-variance-authority": "^0.7.0", + "csstype": "^3.0.10", "framer-motion": "^10.16.4", "highlight.js": "^11.9.0", - "postcss-url": "10.1.3", "jotai": "^2.6.0", "katex": "^0.16.10", "lodash": "^4.17.21", @@ -32,6 +33,7 @@ "next": "14.2.3", "next-themes": "^0.2.1", "postcss": "8.4.31", + "postcss-url": "10.1.3", "posthog-js": "^1.95.1", "react": "18.2.0", "react-circular-progressbar": "^2.1.0", @@ -39,7 +41,6 @@ "react-dropzone": "^14.2.3", "react-hook-form": "^7.47.0", "react-hot-toast": "^2.4.1", - "csstype": "^3.0.10", "react-icons": "^4.12.0", "react-scroll-to-bottom": "^4.2.0", "react-toastify": "^9.1.3", @@ -47,12 +48,13 @@ "tailwind-merge": "^2.0.0", "tailwindcss": "3.3.5", "ulidx": "^2.3.0", - "uuid": "^9.0.1", "use-debounce": "^10.0.0", + "uuid": "^9.0.1", "zod": "^3.22.4" }, "devDependencies": { "@next/eslint-plugin-next": "^14.0.1", + "@types/jest": "^29.5.12", "@types/lodash": "^4.14.200", "@types/node": "20.8.10", "@types/react": "18.2.34", @@ -72,9 +74,11 @@ "eslint-plugin-prettier": "^5.0.1", "eslint-plugin-react": "^7.34.0", "eslint-plugin-react-hooks": "^4.6.0", + "jest-runner": "^29.7.0", "prettier": "^3.0.3", "prettier-plugin-tailwindcss": "^0.5.6", "rimraf": "^5.0.5", + "ts-jest": "^29.2.5", "typescript": "^5.3.3" } } diff --git a/web/testRunner.js b/web/testRunner.js new file mode 100644 index 000000000..1067f05a3 --- /dev/null +++ b/web/testRunner.js @@ -0,0 +1,19 @@ +const jestRunner = require('jest-runner') + +class EmptyTestFileRunner extends jestRunner.default { + async runTests(tests, watcher, onStart, onResult, onFailure, options) { + const nonEmptyTests = tests.filter( + (test) => test.context.hasteFS.getSize(test.path) > 0 + ) + return super.runTests( + nonEmptyTests, + watcher, + onStart, + onResult, + onFailure, + options + ) + } +} + +module.exports = EmptyTestFileRunner diff --git a/web/tsconfig.json b/web/tsconfig.json index 1729c971f..caa244645 100644 --- a/web/tsconfig.json +++ b/web/tsconfig.json @@ -2,7 +2,11 @@ "compilerOptions": { "target": "ES2015", "lib": ["dom", "dom.iterable", "esnext"], - "typeRoots": ["node_modules/@types", "./src/types"], + "typeRoots": [ + "./node_modules/@types", + "./src/types", + "../node_modules/@types/jest" + ], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -25,5 +29,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "**/*.test.ts"] } diff --git a/web/utils/memory.test.ts b/web/utils/memory.test.ts new file mode 100644 index 000000000..e7420957d --- /dev/null +++ b/web/utils/memory.test.ts @@ -0,0 +1,13 @@ +// @auto-generated + +import { utilizedMemory } from './memory' + +test('test_utilizedMemory_arbitraryValues', () => { + const result = utilizedMemory(30, 100) + expect(result).toBe(70) +}) + +test('test_utilizedMemory_freeEqualsTotal', () => { + const result = utilizedMemory(100, 100) + expect(result).toBe(0) +}) diff --git a/web/utils/memory.ts b/web/utils/memory.ts new file mode 100644 index 000000000..6e5aa79ad --- /dev/null +++ b/web/utils/memory.ts @@ -0,0 +1,9 @@ +/** + * Calculate the percentage of memory used + * @param free + * @param total + * @returns + */ +export const utilizedMemory = (free: number, total: number) => { + return Math.round(((total - free) / Math.max(total, 1)) * 100) +}