diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index cd121cdb7..08da0ff33 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -341,6 +341,11 @@ export const chatCompletions = async (request: any, reply: any) => { request.body.stop = request.body.stop.slice(0, 4) } + // add engine for new cortex cpp engine + if (requestedModel.engine === 'nitro') { + request.body.engine = 'cortex.llamacpp' + } + const fetch = require('node-fetch') const response = await fetch(apiUrl, { method: 'POST', diff --git a/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json index 56cd9c81c..837b10ce3 100644 --- a/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json +++ b/extensions/inference-nitro-extension/resources/models/gemma-1.1-2b/model.json @@ -1,7 +1,7 @@ { "sources": [ { - "filename": "gemma-1.1-2b-it-q4_k_m.gguf", + "filename": "gemma-1.1-2b-it-Q4_K_M.gguf", "url": "https://huggingface.co/bartowski/gemma-1.1-2b-it-GGUF/resolve/main/gemma-1.1-2b-it-Q4_K_M.gguf" } ], diff --git a/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json index 5bd89b478..a532c1dc3 100644 --- a/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json +++ b/extensions/inference-nitro-extension/resources/models/gemma-1.1-7b/model.json @@ -1,7 +1,7 @@ { "sources": [ { - "filename": "gemma-1.1-7b-it-q4_K_M.gguf", + "filename": "gemma-1.1-7b-it-Q4_K_M.gguf", "url": "https://huggingface.co/bartowski/gemma-1.1-7b-it-GGUF/resolve/main/gemma-1.1-7b-it-Q4_K_M.gguf" } ], @@ -14,7 +14,7 @@ "settings": { "ctx_len": 8192, "prompt_template": "user\n{prompt}\nmodel", - "llama_model_path": "gemma-1.1-7b-it-q4_K_M.gguf", + "llama_model_path": "gemma-1.1-7b-it-Q4_K_M.gguf", "ngl": 29 }, "parameters": { diff --git a/web/.eslintrc.js b/web/.eslintrc.js index 0ed19d001..73fb1bcf5 100644 --- a/web/.eslintrc.js +++ b/web/.eslintrc.js @@ -11,6 +11,9 @@ module.exports = { '_next', '*.md', 'out', + '**/*.test.tsx', + '**/*.test.ts', + 'testRunner.js', ], extends: [ 'next/core-web-vitals', diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx index 7fb20b0a4..a69e34d57 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx @@ -19,6 +19,8 @@ import { usePath } from '@/hooks/usePath' import { toGibibytes } from '@/utils/converter' +import { utilizedMemory } from '@/utils/memory' + import TableActiveModel from './TableActiveModel' import { showSystemMonitorPanelAtom } from '@/helpers/atoms/App.atom' @@ -159,35 +161,41 @@ const SystemMonitor = () => { {gpus.length > 0 && (
- {gpus.map((gpu, index) => ( -
-
- - {gpu.name} - -
-
- - {gpu.memoryTotal - gpu.memoryFree}/ - {gpu.memoryTotal} - - MB + {gpus.map((gpu, index) => { + const gpuUtilization = utilizedMemory( + gpu.memoryFree, + gpu.memoryTotal + ) + return ( +
+
+ + {gpu.name} + +
+
+ + {gpu.memoryTotal - gpu.memoryFree}/ + {gpu.memoryTotal} + + MB +
-
-
- - - {gpu.utilization}% - +
+ + + {gpuUtilization}% + +
-
- ))} + ) + })}
)}
diff --git a/web/containers/ModelDropdown/index.tsx b/web/containers/ModelDropdown/index.tsx index 07c2cba3a..f8920a6dd 100644 --- a/web/containers/ModelDropdown/index.tsx +++ b/web/containers/ModelDropdown/index.tsx @@ -299,7 +299,10 @@ const ModelDropdown = ({ setOpen(!open)} > {selectedModel?.name} diff --git a/web/hooks/useImportModel.ts b/web/hooks/useImportModel.ts index b23f5a6fb..effc64f86 100644 --- a/web/hooks/useImportModel.ts +++ b/web/hooks/useImportModel.ts @@ -6,9 +6,7 @@ import { Model, ModelExtension, OptionType, - baseName, fs, - joinPath, } from '@janhq/core' import { atom, useSetAtom } from 'jotai' diff --git a/web/jest.config.js b/web/jest.config.js new file mode 100644 index 000000000..b34e410d2 --- /dev/null +++ b/web/jest.config.js @@ -0,0 +1,5 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + runner: './testRunner.js', +} diff --git a/web/package.json b/web/package.json index b24163bb7..3ba02c3b9 100644 --- a/web/package.json +++ b/web/package.json @@ -10,7 +10,8 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write \"**/*.{js,jsx,ts,tsx}\"", - "compile": "tsc --noEmit -p . --pretty" + "compile": "tsc --noEmit -p . --pretty", + "test": "jest" }, "dependencies": { "@heroicons/react": "^2.0.18", @@ -19,9 +20,9 @@ "@janhq/joi": "link:./joi", "autoprefixer": "10.4.16", "class-variance-authority": "^0.7.0", + "csstype": "^3.0.10", "framer-motion": "^10.16.4", "highlight.js": "^11.9.0", - "postcss-url": "10.1.3", "jotai": "^2.6.0", "katex": "^0.16.10", "lodash": "^4.17.21", @@ -32,6 +33,7 @@ "next": "14.2.3", "next-themes": "^0.2.1", "postcss": "8.4.31", + "postcss-url": "10.1.3", "posthog-js": "^1.95.1", "react": "18.2.0", "react-circular-progressbar": "^2.1.0", @@ -39,7 +41,6 @@ "react-dropzone": "^14.2.3", "react-hook-form": "^7.47.0", "react-hot-toast": "^2.4.1", - "csstype": "^3.0.10", "react-icons": "^4.12.0", "react-scroll-to-bottom": "^4.2.0", "react-toastify": "^9.1.3", @@ -47,12 +48,13 @@ "tailwind-merge": "^2.0.0", "tailwindcss": "3.3.5", "ulidx": "^2.3.0", - "uuid": "^9.0.1", "use-debounce": "^10.0.0", + "uuid": "^9.0.1", "zod": "^3.22.4" }, "devDependencies": { "@next/eslint-plugin-next": "^14.0.1", + "@types/jest": "^29.5.12", "@types/lodash": "^4.14.200", "@types/node": "20.8.10", "@types/react": "18.2.34", @@ -72,9 +74,11 @@ "eslint-plugin-prettier": "^5.0.1", "eslint-plugin-react": "^7.34.0", "eslint-plugin-react-hooks": "^4.6.0", + "jest-runner": "^29.7.0", "prettier": "^3.0.3", "prettier-plugin-tailwindcss": "^0.5.6", "rimraf": "^5.0.5", + "ts-jest": "^29.2.5", "typescript": "^5.3.3" } } diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx index e6285565f..a1d153d10 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadList/index.tsx @@ -29,7 +29,7 @@ const ModelDownloadList = () => { return (

Available Versions

- + {ggufModels.map((model, index) => { if (!model.downloadUrl) return null return ( diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx index 98914c94e..951a11d59 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx @@ -10,6 +10,8 @@ import { Badge, Button, Progress } from '@janhq/joi' import { useAtomValue, useSetAtom } from 'jotai' +import { twMerge } from 'tailwind-merge' + import { MainViewState } from '@/constants/screens' import { useCreateNewThread } from '@/hooks/useCreateNewThread' @@ -114,16 +116,24 @@ const ModelDownloadRow: React.FC = ({ } return ( -
-
- {quantization && ( - - {quantization} - - )} -

- {fileName} -

+
+
+
+ {quantization && ( + + {quantization} + + )} +

+ {fileName} +

+
{toGibibytes(fileSize)} diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx index ba17e9b57..10f4515c8 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelSegmentInfo/index.tsx @@ -33,7 +33,7 @@ const ModelSegmentInfo = () => { if (!importingHuggingFaceRepoData) return null return ( -
+

{modelName} diff --git a/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx b/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx index ff3f19e16..e3d469a97 100644 --- a/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx @@ -388,7 +388,11 @@ const ChatInput = () => { { > - + Edit title

diff --git a/web/testRunner.js b/web/testRunner.js new file mode 100644 index 000000000..1067f05a3 --- /dev/null +++ b/web/testRunner.js @@ -0,0 +1,19 @@ +const jestRunner = require('jest-runner') + +class EmptyTestFileRunner extends jestRunner.default { + async runTests(tests, watcher, onStart, onResult, onFailure, options) { + const nonEmptyTests = tests.filter( + (test) => test.context.hasteFS.getSize(test.path) > 0 + ) + return super.runTests( + nonEmptyTests, + watcher, + onStart, + onResult, + onFailure, + options + ) + } +} + +module.exports = EmptyTestFileRunner diff --git a/web/tsconfig.json b/web/tsconfig.json index 1729c971f..caa244645 100644 --- a/web/tsconfig.json +++ b/web/tsconfig.json @@ -2,7 +2,11 @@ "compilerOptions": { "target": "ES2015", "lib": ["dom", "dom.iterable", "esnext"], - "typeRoots": ["node_modules/@types", "./src/types"], + "typeRoots": [ + "./node_modules/@types", + "./src/types", + "../node_modules/@types/jest" + ], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -25,5 +29,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "**/*.test.ts"] } diff --git a/web/utils/memory.test.ts b/web/utils/memory.test.ts new file mode 100644 index 000000000..e7420957d --- /dev/null +++ b/web/utils/memory.test.ts @@ -0,0 +1,13 @@ +// @auto-generated + +import { utilizedMemory } from './memory' + +test('test_utilizedMemory_arbitraryValues', () => { + const result = utilizedMemory(30, 100) + expect(result).toBe(70) +}) + +test('test_utilizedMemory_freeEqualsTotal', () => { + const result = utilizedMemory(100, 100) + expect(result).toBe(0) +}) diff --git a/web/utils/memory.ts b/web/utils/memory.ts new file mode 100644 index 000000000..6e5aa79ad --- /dev/null +++ b/web/utils/memory.ts @@ -0,0 +1,9 @@ +/** + * Calculate the percentage of memory used + * @param free + * @param total + * @returns + */ +export const utilizedMemory = (free: number, total: number) => { + return Math.round(((total - free) / Math.max(total, 1)) * 100) +} diff --git a/web/utils/modelEngine.ts b/web/utils/modelEngine.ts index a12e9bb0e..3d132c5d5 100644 --- a/web/utils/modelEngine.ts +++ b/web/utils/modelEngine.ts @@ -4,6 +4,7 @@ export const getLogoEngine = (engine: InferenceEngine) => { switch (engine) { case InferenceEngine.anthropic: return 'images/ModelProvider/anthropic.svg' + case InferenceEngine.nitro_tensorrt_llm: case InferenceEngine.nitro: return 'images/ModelProvider/nitro.svg' case InferenceEngine.cortex_llamacpp: @@ -43,6 +44,8 @@ export const getTitleByEngine = (engine: InferenceEngine) => { switch (engine) { case InferenceEngine.nitro: return 'Llama.cpp (Nitro)' + case InferenceEngine.nitro_tensorrt_llm: + return 'TensorRT-LLM (Nitro)' case InferenceEngine.cortex_llamacpp: return 'Llama.cpp (Cortex)' case InferenceEngine.cortex_onnx: