From ef3c09036049a523957b1987c902430be050d6f9 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 21 Nov 2024 16:14:01 +0700 Subject: [PATCH 1/3] fix: Phi 3 Mini Instruct model download url replacement --- extensions/inference-cortex-extension/package.json | 2 +- .../resources/models/phi3-3.8b/model.json | 6 +++--- .../resources/models/phi3-medium/model.json | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extensions/inference-cortex-extension/package.json b/extensions/inference-cortex-extension/package.json index d262ad5ec..c6d3f70b6 100644 --- a/extensions/inference-cortex-extension/package.json +++ b/extensions/inference-cortex-extension/package.json @@ -1,7 +1,7 @@ { "name": "@janhq/inference-cortex-extension", "productName": "Cortex Inference Engine", - "version": "1.0.21", + "version": "1.0.22", "description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.", "main": "dist/index.js", "node": "dist/node/index.cjs.js", diff --git a/extensions/inference-cortex-extension/resources/models/phi3-3.8b/model.json b/extensions/inference-cortex-extension/resources/models/phi3-3.8b/model.json index a2197dab2..90aa50117 100644 --- a/extensions/inference-cortex-extension/resources/models/phi3-3.8b/model.json +++ b/extensions/inference-cortex-extension/resources/models/phi3-3.8b/model.json @@ -1,8 +1,8 @@ { "sources": [ { - "url": "https://huggingface.co/cortexso/phi3/resolve/main/model.gguf", - "filename": "model.gguf" + "url": "https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF/resolve/main/Phi-3-mini-4k-instruct-Q4_K_M.gguf", + "filename": "Phi-3-mini-4k-instruct-Q4_K_M.gguf" } ], "id": "phi3-3.8b", @@ -14,7 +14,7 @@ "settings": { "ctx_len": 4096, "prompt_template": "<|user|>\n{prompt}<|end|>\n<|assistant|>\n", - "llama_model_path": "model.gguf", + "llama_model_path": "Phi-3-mini-4k-instruct-Q4_K_M.gguf", "ngl": 33 }, "parameters": { diff --git a/extensions/inference-cortex-extension/resources/models/phi3-medium/model.json b/extensions/inference-cortex-extension/resources/models/phi3-medium/model.json index f7131ee98..afce04952 100644 --- a/extensions/inference-cortex-extension/resources/models/phi3-medium/model.json +++ b/extensions/inference-cortex-extension/resources/models/phi3-medium/model.json @@ -1,8 +1,8 @@ { "sources": [ { - "url": "https://huggingface.co/bartowski/Phi-3-medium-128k-instruct-GGUF/resolve/main/Phi-3-medium-128k-instruct-Q4_K_M.gguf", - "filename": "Phi-3-medium-128k-instruct-Q4_K_M.gguf" + "url": "https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF/resolve/main/Phi-3-mini-4k-instruct-Q4_K_M.gguf", + "filename": "Phi-3-mini-4k-instruct-Q4_K_M.gguf" } ], "id": "phi3-medium", @@ -14,7 +14,7 @@ "settings": { "ctx_len": 128000, "prompt_template": "<|user|> {prompt}<|end|><|assistant|>", - "llama_model_path": "Phi-3-medium-128k-instruct-Q4_K_M.gguf", + "llama_model_path": "Phi-3-mini-4k-instruct-Q4_K_M.gguf", "ngl": 33 }, "parameters": { From f74994a735801cdf208efc79354aa41f5886c3fe Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 21 Nov 2024 16:58:15 +0700 Subject: [PATCH 2/3] fix: tensorrt-llm model list --- web/hooks/useModels.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/hooks/useModels.ts b/web/hooks/useModels.ts index 75c86035a..309cc4782 100644 --- a/web/hooks/useModels.ts +++ b/web/hooks/useModels.ts @@ -89,13 +89,13 @@ const useModels = () => { const cachedModels = ModelManager.instance().models.values().toArray() const toUpdate = [ ...downloadedModels, - ...cachedModels.filter( - (e: Model) => !downloadedModels.some((g: Model) => g.id === e.id) + ...cachedModels.filter((e) => !isLocalEngine(e.engine) && !downloadedModels.some((g: Model) => g.id === e.id) ), ] setDownloadedModels(toUpdate) - }, [downloadedModels, setDownloadedModels]) + setExtensionModels(cachedModels) + }, [downloadedModels, setDownloadedModels, setExtensionModels]) const getModels = async (): Promise => extensionManager From 73007b49fa3c2f7dd022c23a29c4c8e65ac248e5 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 21 Nov 2024 17:14:31 +0700 Subject: [PATCH 3/3] chore: lint fix --- web/hooks/useModels.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/hooks/useModels.ts b/web/hooks/useModels.ts index 309cc4782..ef15de763 100644 --- a/web/hooks/useModels.ts +++ b/web/hooks/useModels.ts @@ -89,7 +89,10 @@ const useModels = () => { const cachedModels = ModelManager.instance().models.values().toArray() const toUpdate = [ ...downloadedModels, - ...cachedModels.filter((e) => !isLocalEngine(e.engine) && !downloadedModels.some((g: Model) => g.id === e.id) + ...cachedModels.filter( + (e) => + !isLocalEngine(e.engine) && + !downloadedModels.some((g: Model) => g.id === e.id) ), ]