From 0bad1a479f642c8c5183017641656d52c4b33c61 Mon Sep 17 00:00:00 2001
From: NamH <NamNh0122@gmail.com>
Date: Thu, 25 Apr 2024 17:24:29 +0700
Subject: [PATCH 01/26] fix: remove scroll animation chat screen (#2819)

Signed-off-by: James <james@jan.ai>
Co-authored-by: James <james@jan.ai>
---
 web/containers/ListContainer/index.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/containers/ListContainer/index.tsx b/web/containers/ListContainer/index.tsx
index fdf34b9ec..0d3e6de61 100644
--- a/web/containers/ListContainer/index.tsx
+++ b/web/containers/ListContainer/index.tsx
@@ -12,7 +12,7 @@ const ListContainer: React.FC<Props> = ({ children }) => {
 
     listRef.current?.scrollTo({
       top: scrollHeight,
-      behavior: 'smooth',
+      behavior: 'instant',
     })
   })
 

From ce2d8e540582a73423efed02dbe484e213cdf0fe Mon Sep 17 00:00:00 2001
From: NamH <NamNh0122@gmail.com>
Date: Thu, 2 May 2024 21:02:05 +0700
Subject: [PATCH 02/26] chore: remove nutjs (#2860)

Signed-off-by: James <james@jan.ai>
Co-authored-by: James <james@jan.ai>
---
 electron/package.json          |  3 +--
 electron/utils/selectedText.ts | 27 ++++++++++++++-------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/electron/package.json b/electron/package.json
index 7f1c978e4..f012055e2 100644
--- a/electron/package.json
+++ b/electron/package.json
@@ -95,8 +95,7 @@
     "pacote": "^17.0.4",
     "request": "^2.88.2",
     "request-progress": "^3.0.0",
-    "ulidx": "^2.3.0",
-    "@nut-tree/nut-js": "^4.0.0"
+    "ulidx": "^2.3.0"
   },
   "devDependencies": {
     "@electron/notarize": "^2.1.0",
diff --git a/electron/utils/selectedText.ts b/electron/utils/selectedText.ts
index 25d31fab5..f76146d13 100644
--- a/electron/utils/selectedText.ts
+++ b/electron/utils/selectedText.ts
@@ -1,23 +1,24 @@
 import { clipboard, globalShortcut } from 'electron'
-import { keyboard, Key } from '@nut-tree/nut-js'
 
 /**
  * Gets selected text by synthesizing the keyboard shortcut
  * "CommandOrControl+c" then reading text from the clipboard
  */
 export const getSelectedText = async () => {
-  const currentClipboardContent = clipboard.readText() // preserve clipboard content
-  clipboard.clear()
-  const hotkeys: Key[] = [
-    process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl,
-    Key.C,
-  ]
-  await keyboard.pressKey(...hotkeys)
-  await keyboard.releaseKey(...hotkeys)
-  await new Promise((resolve) => setTimeout(resolve, 200)) // add a delay before checking clipboard
-  const selectedText = clipboard.readText()
-  clipboard.writeText(currentClipboardContent)
-  return selectedText
+  // TODO: Implement this function
+  // const currentClipboardContent = clipboard.readText() // preserve clipboard content
+  // clipboard.clear()
+  // const hotkeys: Key[] = [
+  //   process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl,
+  //   Key.C,
+  // ]
+  // await keyboard.pressKey(...hotkeys)
+  // await keyboard.releaseKey(...hotkeys)
+  // await new Promise((resolve) => setTimeout(resolve, 200)) // add a delay before checking clipboard
+  // const selectedText = clipboard.readText()
+  // clipboard.writeText(currentClipboardContent)
+  // return selectedText
+  return ''
 }
 
 /**

From c6182ab59c0bb54ce476b47a05f423fa77cc00d8 Mon Sep 17 00:00:00 2001
From: Quentin Macheda <69113642+QuentinMacheda@users.noreply.github.com>
Date: Fri, 3 May 2024 04:17:05 +0100
Subject: [PATCH 03/26] Customize scroll-bar style (#2857)

---
 uikit/src/main.scss               |  1 +
 uikit/src/scroll-area/styles.scss | 35 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/uikit/src/main.scss b/uikit/src/main.scss
index f3294e12e..4aa044034 100644
--- a/uikit/src/main.scss
+++ b/uikit/src/main.scss
@@ -45,6 +45,7 @@
   --border: 20 5.9% 90%;
   --input: 20 5.9% 90%;
   --ring: 20 14.3% 4.1%;
+  --scroll-bar: 60, 3%, 86%;
 
   .primary-blue {
     --primary: 221 83% 53%;
diff --git a/uikit/src/scroll-area/styles.scss b/uikit/src/scroll-area/styles.scss
index a6e1caf3a..3b076ffc2 100644
--- a/uikit/src/scroll-area/styles.scss
+++ b/uikit/src/scroll-area/styles.scss
@@ -21,3 +21,38 @@
     @apply bg-border relative z-50 w-[10px] rounded-full;
   }
 }
+
+// Customized scroll bar
+::-webkit-scrollbar {
+  width: 7px;
+}
+
+::-webkit-scrollbar-thumb {
+  background-color: hsl(var(--scroll-bar));
+  border-radius: 4px;
+}
+
+::-webkit-scrollbar-track {
+  background-color: hsl(var(--background));
+}
+
+::-webkit-scrollbar-corner {
+  background-color: hsl(var(--background));
+}
+
+::-moz-scrollbar {
+  width: 7px;
+}
+
+::-moz-scrollbar-thumb {
+  background-color: hsl(var(--scroll-bar));
+  border-radius: 4px;
+}
+
+::-moz-scrollbar-track {
+  background-color: hsl(var(--background));
+}
+
+::-moz-scrollbar-corner {
+  background-color: hsl(var(--background));
+}

From 2016eae0c9e62a841df25011a86886a1b37ea572 Mon Sep 17 00:00:00 2001
From: Quentin Macheda <69113642+QuentinMacheda@users.noreply.github.com>
Date: Fri, 3 May 2024 09:11:29 +0100
Subject: [PATCH 04/26] Remove hidden overflow property of tailwind + Update
 buttons position (under textarea) (#2856)

Please be aware of the lint prettier failed
---
 web/screens/Chat/EditChatInput/index.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/screens/Chat/EditChatInput/index.tsx b/web/screens/Chat/EditChatInput/index.tsx
index fba4b0b8a..e40e1effd 100644
--- a/web/screens/Chat/EditChatInput/index.tsx
+++ b/web/screens/Chat/EditChatInput/index.tsx
@@ -129,11 +129,11 @@ const EditChatInput: React.FC<Props> = ({ message }) => {
   }
 
   return (
-    <div className="mx-auto flex w-full flex-shrink-0 items-end justify-center space-x-4 pb-0 pt-1">
+    <div className="mx-auto flex w-full flex-col flex-shrink-0 items-start justify-center space-y-4 pb-0 pt-1">
       <div className="relative flex w-full flex-col">
         <Textarea
           className={twMerge(
-            'max-h-[400px] resize-none overflow-y-hidden pr-20'
+            'max-h-[400px] resize-none pr-20'
           )}
           style={{ height: '40px' }}
           ref={textareaRef}

From c21bc0879371940a4452925cc2a019dfca1be5c0 Mon Sep 17 00:00:00 2001
From: Quentin Macheda <69113642+QuentinMacheda@users.noreply.github.com>
Date: Fri, 3 May 2024 13:16:06 +0100
Subject: [PATCH 05/26] Fix eslint issue in EditChatInput (#2864)

---
 web/screens/Chat/EditChatInput/index.tsx | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/web/screens/Chat/EditChatInput/index.tsx b/web/screens/Chat/EditChatInput/index.tsx
index e40e1effd..e87a4e680 100644
--- a/web/screens/Chat/EditChatInput/index.tsx
+++ b/web/screens/Chat/EditChatInput/index.tsx
@@ -129,12 +129,10 @@ const EditChatInput: React.FC<Props> = ({ message }) => {
   }
 
   return (
-    <div className="mx-auto flex w-full flex-col flex-shrink-0 items-start justify-center space-y-4 pb-0 pt-1">
+    <div className="mx-auto flex w-full flex-shrink-0 flex-col items-start justify-center space-y-4 pb-0 pt-1">
       <div className="relative flex w-full flex-col">
         <Textarea
-          className={twMerge(
-            'max-h-[400px] resize-none pr-20'
-          )}
+          className={twMerge('max-h-[400px] resize-none pr-20')}
           style={{ height: '40px' }}
           ref={textareaRef}
           onKeyDown={onKeyDown}

From 092a57268453f36a832e893e09a76ad9fecd2eb6 Mon Sep 17 00:00:00 2001
From: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
Date: Sat, 4 May 2024 15:44:19 +0700
Subject: [PATCH 06/26] Feat: Remote API Parameters Correction (#2802)

* fix: change to gpt4 turbo

* add: params

* fix: change to gpt 3.5 turbo

* delete: redundant

* fix: correct description

* version bump

* add: params

* fix: version bump

* delete: deprecated

* add: params

* add: new model

* chore: version bump

* fix: version correct

* add: params

* fix: version bump

* fix: change to gpt4 turbo

* add: params

* fix: change to gpt 3.5 turbo

* delete: redundant

* fix: correct description

* version bump

* add: params

* fix: version bump

* delete: deprecated

* add: params

* add: new model

* chore: version bump

* fix: version correct

* add: params

* fix: version bump

* fix: llama2 no longer supported

* fix: reverse mistral api

* fix: add params

* fix: mistral api redundant params

* fix: typo

* fix: typo

* fix: correct context length

* fix: remove stop

---------

Co-authored-by: Van Pham <64197333+Van-QA@users.noreply.github.com>
---
 .../inference-groq-extension/package.json     |  2 +-
 .../resources/models.json                     | 88 +++++++------------
 .../inference-mistral-extension/package.json  |  2 +-
 .../resources/models.json                     | 84 +++++++++---------
 .../inference-openai-extension/package.json   |  2 +-
 .../resources/models.json                     | 59 +++++--------
 6 files changed, 98 insertions(+), 139 deletions(-)

diff --git a/extensions/inference-groq-extension/package.json b/extensions/inference-groq-extension/package.json
index 4fc08927b..509cb7611 100644
--- a/extensions/inference-groq-extension/package.json
+++ b/extensions/inference-groq-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-groq-extension",
   "productName": "Groq Inference Engine",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "This extension enables fast Groq chat completion API calls",
   "main": "dist/index.js",
   "module": "dist/module.js",
diff --git a/extensions/inference-groq-extension/resources/models.json b/extensions/inference-groq-extension/resources/models.json
index ad721ba62..32ec60add 100644
--- a/extensions/inference-groq-extension/resources/models.json
+++ b/extensions/inference-groq-extension/resources/models.json
@@ -8,18 +8,18 @@
     "id": "llama3-70b-8192",
     "object": "model",
     "name": "Groq Llama 3 70b",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Groq Llama 3 70b with supercharged speed!",
     "format": "api",
-    "settings": {
-      "text_model": false
-    },
+    "settings": {},
     "parameters": {
       "max_tokens": 8192,
       "temperature": 0.7,
-      "top_p": 1,
-      "stop": null,
-      "stream": true
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
     },
     "metadata": {
       "author": "Meta",
@@ -36,18 +36,18 @@
     "id": "llama3-8b-8192",
     "object": "model",
     "name": "Groq Llama 3 8b",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Groq Llama 3 8b with supercharged speed!",
     "format": "api",
-    "settings": {
-      "text_model": false
-    },
+    "settings": {},
     "parameters": {
       "max_tokens": 8192,
       "temperature": 0.7,
-      "top_p": 1,
-      "stop": null,
-      "stream": true
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
     },
     "metadata": {
       "author": "Meta",
@@ -64,18 +64,18 @@
     "id": "gemma-7b-it",
     "object": "model",
     "name": "Groq Gemma 7b Instruct",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Groq Gemma 7b Instruct with supercharged speed!",
     "format": "api",
-    "settings": {
-      "text_model": false
-    },
+    "settings": {},
     "parameters": {
-      "max_tokens": 4096,
+      "max_tokens": 8192,
       "temperature": 0.7,
-      "top_p": 1,
-      "stop": null,
-      "stream": true
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
     },
     "metadata": {
       "author": "Google",
@@ -83,34 +83,6 @@
     },
     "engine": "groq"
   },
-  {
-    "sources": [
-      {
-        "url": "https://groq.com"
-      }
-    ],
-    "id": "llama2-70b-4096",
-    "object": "model",
-    "name": "Groq Llama 2 70b",
-    "version": "1.0",
-    "description": "Groq Llama 2 70b with supercharged speed!",
-    "format": "api",
-    "settings": {
-      "text_model": false
-    },
-    "parameters": {
-      "max_tokens": 4096,
-      "temperature": 0.7,
-      "top_p": 1,
-      "stop": null,
-      "stream": true
-    },
-    "metadata": {
-      "author": "Meta",
-      "tags": ["General", "Big Context Length"]
-    },
-    "engine": "groq"
-  },
   {
     "sources": [
       {
@@ -120,18 +92,18 @@
     "id": "mixtral-8x7b-32768",
     "object": "model",
     "name": "Groq Mixtral 8x7b Instruct",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Groq Mixtral 8x7b Instruct is Mixtral with supercharged speed!",
     "format": "api",
-    "settings": {
-      "text_model": false
-    },
+    "settings": {},
     "parameters": {
-      "max_tokens": 4096,
+      "max_tokens": 32768,
       "temperature": 0.7,
-      "top_p": 1,
-      "stop": null,
-      "stream": true
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
     },
     "metadata": {
       "author": "Mistral",
diff --git a/extensions/inference-mistral-extension/package.json b/extensions/inference-mistral-extension/package.json
index 86fa8bc77..7cdb61253 100644
--- a/extensions/inference-mistral-extension/package.json
+++ b/extensions/inference-mistral-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-mistral-extension",
   "productName": "MistralAI Inference Engine",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "This extension enables Mistral chat completion API calls",
   "main": "dist/index.js",
   "module": "dist/module.js",
diff --git a/extensions/inference-mistral-extension/resources/models.json b/extensions/inference-mistral-extension/resources/models.json
index a5189311b..23ecd6fdd 100644
--- a/extensions/inference-mistral-extension/resources/models.json
+++ b/extensions/inference-mistral-extension/resources/models.json
@@ -8,48 +8,20 @@
     "id": "mistral-small-latest",
     "object": "model",
     "name": "Mistral Small",
-    "version": "1.0",
-    "description": "Mistral Small is the ideal choice for simpe tasks that one can do in builk - like Classification, Customer Support, or Text Generation. It offers excellent performance at an affordable price point.",
+    "version": "1.1",
+    "description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.",
     "format": "api",
     "settings": {},
     "parameters": {
-      "max_tokens": 4096,
-      "temperature": 0.7
+      "max_tokens": 32000,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true
     },
     "metadata": {
       "author": "Mistral",
       "tags": [
-        "Classification",
-        "Customer Support",
-        "Text Generation"
-      ]
-    },
-    "engine": "mistral"
-  },
-  {
-    "sources": [
-      {
-        "url": "https://docs.mistral.ai/api/"
-      }
-    ],
-    "id": "mistral-medium-latest",
-    "object": "model",
-    "name": "Mistral Medium",
-    "version": "1.0",
-    "description": "Mistral Medium is the ideal for intermediate tasks that require moderate reasoning - like Data extraction, Summarizing a Document, Writing a Job Description, or Writing Product Descriptions. Mistral Medium strikes a balance between performance and capability, making it suitable for a wide range of tasks that only require language transformaion",
-    "format": "api",
-    "settings": {},
-    "parameters": {
-      "max_tokens": 4096,
-      "temperature": 0.7
-    },
-    "metadata": {
-      "author": "Mistral",
-      "tags": [
-        "Data extraction",
-        "Summarizing a Document",
-        "Writing a Job Description",
-        "Writing Product Descriptions"
+        "General"
       ]
     },
     "engine": "mistral"
@@ -63,21 +35,47 @@
     "id": "mistral-large-latest",
     "object": "model",
     "name": "Mistral Large",
-    "version": "1.0",
-    "description": "Mistral Large is ideal for complex tasks that require large reasoning capabilities or are highly specialized - like Synthetic Text Generation, Code Generation, RAG, or Agents.",
+    "version": "1.1",
+    "description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).",
     "format": "api",
     "settings": {},
     "parameters": {
-      "max_tokens": 4096,
-      "temperature": 0.7
+      "max_tokens": 32000,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true
     },
     "metadata": {
       "author": "Mistral",
       "tags": [
-        "Text Generation",
-        "Code Generation",
-        "RAG",
-        "Agents"
+        "General"
+      ]
+    },
+    "engine": "mistral"
+  },
+  {
+    "sources": [
+      {
+        "url": "https://docs.mistral.ai/api/"
+      }
+    ],
+    "id": "open-mixtral-8x22b",
+    "object": "model",
+    "name": "Mixtral 8x22B",
+    "version": "1.1",
+    "description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 32000,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true
+    },
+    "metadata": {
+      "author": "Mistral",
+      "tags": [
+        "General"
       ]
     },
     "engine": "mistral"
diff --git a/extensions/inference-openai-extension/package.json b/extensions/inference-openai-extension/package.json
index d1a751069..713989e75 100644
--- a/extensions/inference-openai-extension/package.json
+++ b/extensions/inference-openai-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-openai-extension",
   "productName": "OpenAI Inference Engine",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "This extension enables OpenAI chat completion API calls",
   "main": "dist/index.js",
   "module": "dist/module.js",
diff --git a/extensions/inference-openai-extension/resources/models.json b/extensions/inference-openai-extension/resources/models.json
index b7c5885c6..096c96390 100644
--- a/extensions/inference-openai-extension/resources/models.json
+++ b/extensions/inference-openai-extension/resources/models.json
@@ -5,20 +5,25 @@
         "url": "https://openai.com"
       }
     ],
-    "id": "gpt-4",
+    "id": "gpt-4-turbo",
     "object": "model",
     "name": "OpenAI GPT 4",
-    "version": "1.0",
+    "version": "1.1",
     "description": "OpenAI GPT 4 model is extremely good",
     "format": "api",
     "settings": {},
     "parameters": {
       "max_tokens": 4096,
-      "temperature": 0.7
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
     },
     "metadata": {
       "author": "OpenAI",
-      "tags": ["General", "Big Context Length"]
+      "tags": ["General"]
     },
     "engine": "openai"
   },
@@ -31,8 +36,8 @@
     "id": "gpt-4-vision-preview",
     "object": "model",
     "name": "OpenAI GPT 4 with Vision (Preview)",
-    "version": "1.0",
-    "description": "OpenAI GPT 4 with Vision model is extremely good in preview",
+    "version": "1.1",
+    "description": "OpenAI GPT-4 Vision model features vision understanding capabilities",
     "format": "api",
     "settings": {
       "vision_model": true,
@@ -40,34 +45,13 @@
     },
     "parameters": {
       "max_tokens": 4096,
-      "temperature": 0.7
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true
     },
     "metadata": {
       "author": "OpenAI",
-      "tags": ["General", "Big Context Length", "Vision"]
-    },
-    "engine": "openai"
-  },
-  {
-    "sources": [
-      {
-        "url": "https://openai.com"
-      }
-    ],
-    "id": "gpt-3.5-turbo-16k-0613",
-    "object": "model",
-    "name": "OpenAI GPT 3.5 Turbo 16k 0613",
-    "version": "1.0",
-    "description": "OpenAI GPT 3.5 Turbo 16k 0613 model is extremely good",
-    "format": "api",
-    "settings": {},
-    "parameters": {
-      "max_tokens": 4096,
-      "temperature": 0.7
-    },
-    "metadata": {
-      "author": "OpenAI",
-      "tags": ["General", "Big Context Length"]
+      "tags": ["General", "Vision"]
     },
     "engine": "openai"
   },
@@ -80,17 +64,22 @@
     "id": "gpt-3.5-turbo",
     "object": "model",
     "name": "OpenAI GPT 3.5 Turbo",
-    "version": "1.0",
-    "description": "OpenAI GPT 3.5 Turbo model is extremely good",
+    "version": "1.1",
+    "description": "OpenAI GPT 3.5 Turbo model is extremely fast",
     "format": "api",
     "settings": {},
     "parameters": {
       "max_tokens": 4096,
-      "temperature": 0.7
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
     },
     "metadata": {
       "author": "OpenAI",
-      "tags": ["General", "Big Context Length"]
+      "tags": ["General"]
     },
     "engine": "openai"
   }

From 4c88d03aa59ea19e62cf864728eb7bbd06eac86f Mon Sep 17 00:00:00 2001
From: Henry <150573299+henryh0x1@users.noreply.github.com>
Date: Mon, 6 May 2024 12:37:57 +0900
Subject: [PATCH 07/26] feat: add remote model command-r (#2868)

---
 .../resources/models.json                     | 34 +++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/extensions/inference-cohere-extension/resources/models.json b/extensions/inference-cohere-extension/resources/models.json
index 0fce51fb7..2b4cc3e8e 100644
--- a/extensions/inference-cohere-extension/resources/models.json
+++ b/extensions/inference-cohere-extension/resources/models.json
@@ -1,4 +1,4 @@
-  [
+[
   {
     "sources": [
       {
@@ -19,7 +19,37 @@
     },
     "metadata": {
       "author": "Cohere",
-      "tags": ["General", "Big Context Length"]
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
+    },
+    "engine": "cohere"
+  },
+  {
+    "sources": [
+      {
+        "url": "https://cohere.com"
+      }
+    ],
+    "id": "command-r",
+    "object": "model",
+    "name": "Command R",
+    "version": "1.0",
+    "description": "Command R is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It can be used for complex workflows like code generation, retrieval augmented generation (RAG), tool use, and agents.",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 128000,
+      "temperature": 0.7,
+      "stream": false
+    },
+    "metadata": {
+      "author": "Cohere",
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
     },
     "engine": "cohere"
   }

From 86fda1cf6c866eb8a399946fb66e39b23e4d711c Mon Sep 17 00:00:00 2001
From: Henry <150573299+henryh0x1@users.noreply.github.com>
Date: Mon, 6 May 2024 12:43:15 +0900
Subject: [PATCH 08/26] feat: add model gpt-4 turbo (#2836)

* feat: add model gpt-4 turbo

* fix: correct naming

---------

Co-authored-by: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
---
 extensions/inference-openai-extension/resources/models.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/extensions/inference-openai-extension/resources/models.json b/extensions/inference-openai-extension/resources/models.json
index 096c96390..d8aa787d9 100644
--- a/extensions/inference-openai-extension/resources/models.json
+++ b/extensions/inference-openai-extension/resources/models.json
@@ -7,9 +7,9 @@
     ],
     "id": "gpt-4-turbo",
     "object": "model",
-    "name": "OpenAI GPT 4",
-    "version": "1.1",
-    "description": "OpenAI GPT 4 model is extremely good",
+    "name": "OpenAI GPT 4 Turbo",
+    "version": "1.2",
+    "description": "OpenAI GPT 4 Turbo model is extremely good",
     "format": "api",
     "settings": {},
     "parameters": {

From a6ccd672c5cdafb29f363b65655da2b7d7ce16fc Mon Sep 17 00:00:00 2001
From: Faisal Amir <urmauur@gmail.com>
Date: Mon, 6 May 2024 15:10:37 +0700
Subject: [PATCH 09/26] fix: validate max_token from context_length value
 (#2870)

---
 web/hooks/useCreateNewThread.ts               |  8 +++++-
 .../Chat/ModelSetting/SettingComponent.tsx    | 14 +++++++++-
 web/screens/Chat/Sidebar/index.tsx            | 26 +++++++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/web/hooks/useCreateNewThread.ts b/web/hooks/useCreateNewThread.ts
index e42bc1d4c..6939b1af6 100644
--- a/web/hooks/useCreateNewThread.ts
+++ b/web/hooks/useCreateNewThread.ts
@@ -99,6 +99,11 @@ export const useCreateNewThread = () => {
         ? { ctx_len: 2048 }
         : {}
 
+    const overriddenParameters =
+      defaultModel?.parameters.max_tokens && defaultModel.parameters.max_tokens
+        ? { max_tokens: 2048 }
+        : {}
+
     const createdAt = Date.now()
     const assistantInfo: ThreadAssistantInfo = {
       assistant_id: assistant.id,
@@ -107,7 +112,8 @@ export const useCreateNewThread = () => {
       model: {
         id: defaultModel?.id ?? '*',
         settings: { ...defaultModel?.settings, ...overriddenSettings } ?? {},
-        parameters: defaultModel?.parameters ?? {},
+        parameters:
+          { ...defaultModel?.parameters, ...overriddenParameters } ?? {},
         engine: defaultModel?.engine,
       },
       instructions: assistant.instructions,
diff --git a/web/screens/Chat/ModelSetting/SettingComponent.tsx b/web/screens/Chat/ModelSetting/SettingComponent.tsx
index 43df16430..2fde17276 100644
--- a/web/screens/Chat/ModelSetting/SettingComponent.tsx
+++ b/web/screens/Chat/ModelSetting/SettingComponent.tsx
@@ -5,10 +5,14 @@ import {
   SliderComponentProps,
 } from '@janhq/core'
 
+import { useAtomValue } from 'jotai/react'
+
 import Checkbox from '@/containers/Checkbox'
 import ModelConfigInput from '@/containers/ModelConfigInput'
 import SliderRightPanel from '@/containers/SliderRightPanel'
 
+import { activeThreadAtom } from '@/helpers/atoms/Thread.atom'
+
 type Props = {
   componentProps: SettingComponentProps[]
   disabled?: boolean
@@ -20,6 +24,7 @@ const SettingComponent: React.FC<Props> = ({
   disabled = false,
   onValueUpdated,
 }) => {
+  const activeThread = useAtomValue(activeThreadAtom)
   const components = componentProps.map((data) => {
     switch (data.controllerType) {
       case 'slider': {
@@ -31,7 +36,14 @@ const SettingComponent: React.FC<Props> = ({
             title={data.title}
             description={data.description}
             min={min}
-            max={max}
+            max={
+              data.key === 'max_tokens'
+                ? Number(
+                    activeThread &&
+                      activeThread.assistants[0].model.settings.ctx_len
+                  )
+                : max
+            }
             step={step}
             value={value}
             name={data.key}
diff --git a/web/screens/Chat/Sidebar/index.tsx b/web/screens/Chat/Sidebar/index.tsx
index 6829ac9ff..ba4fdb5a1 100644
--- a/web/screens/Chat/Sidebar/index.tsx
+++ b/web/screens/Chat/Sidebar/index.tsx
@@ -118,6 +118,32 @@ const Sidebar: React.FC = () => {
       updateModelParameter(activeThread, {
         params: { [key]: value },
       })
+
+      if (
+        activeThread.assistants[0].model.parameters.max_tokens &&
+        activeThread.assistants[0].model.settings.ctx_len
+      ) {
+        if (
+          key === 'max_tokens' &&
+          Number(value) > activeThread.assistants[0].model.settings.ctx_len
+        ) {
+          updateModelParameter(activeThread, {
+            params: {
+              max_tokens: activeThread.assistants[0].model.settings.ctx_len,
+            },
+          })
+        }
+        if (
+          key === 'ctx_len' &&
+          Number(value) < activeThread.assistants[0].model.parameters.max_tokens
+        ) {
+          updateModelParameter(activeThread, {
+            params: {
+              max_tokens: activeThread.assistants[0].model.settings.ctx_len,
+            },
+          })
+        }
+      }
     },
     [activeThread, setEngineParamsUpdate, stopModel, updateModelParameter]
   )

From 1e3e5a83f480baefce789380c360036bec12b370 Mon Sep 17 00:00:00 2001
From: Henry <150573299+henryh0x1@users.noreply.github.com>
Date: Mon, 6 May 2024 17:24:07 +0900
Subject: [PATCH 10/26] feat/implement-inference-martian-extension (#2869)

---
 .../inference-martian-extension/README.md     | 79 +++++++++++++++++++
 .../inference-martian-extension/package.json  | 42 ++++++++++
 .../resources/models.json                     | 32 ++++++++
 .../resources/settings.json                   | 23 ++++++
 .../inference-martian-extension/src/index.ts  | 66 ++++++++++++++++
 .../inference-martian-extension/tsconfig.json | 14 ++++
 .../webpack.config.js                         | 37 +++++++++
 7 files changed, 293 insertions(+)
 create mode 100644 extensions/inference-martian-extension/README.md
 create mode 100644 extensions/inference-martian-extension/package.json
 create mode 100644 extensions/inference-martian-extension/resources/models.json
 create mode 100644 extensions/inference-martian-extension/resources/settings.json
 create mode 100644 extensions/inference-martian-extension/src/index.ts
 create mode 100644 extensions/inference-martian-extension/tsconfig.json
 create mode 100644 extensions/inference-martian-extension/webpack.config.js

diff --git a/extensions/inference-martian-extension/README.md b/extensions/inference-martian-extension/README.md
new file mode 100644
index 000000000..5b8e898d7
--- /dev/null
+++ b/extensions/inference-martian-extension/README.md
@@ -0,0 +1,79 @@
+# Martian Engine Extension
+
+Created using Jan extension example
+
+# Create a Jan Extension using Typescript
+
+Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
+
+## Create Your Own Extension
+
+To create your own extension, you can use this repository as a template! Just follow the below instructions:
+
+1. Click the Use this template button at the top of the repository
+2. Select Create a new repository
+3. Select an owner and name for your new repository
+4. Click Create repository
+5. Clone your new repository
+
+## Initial Setup
+
+After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
+
+> [!NOTE]
+>
+> You'll need to have a reasonably modern version of
+> [Node.js](https://nodejs.org) handy. If you are using a version manager like
+> [`nodenv`](https://github.com/nodenv/nodenv) or
+> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
+> root of your repository to install the version specified in
+> [`package.json`](./package.json). Otherwise, 20.x or later should work!
+
+1. :hammer_and_wrench: Install the dependencies
+
+   ```bash
+   npm install
+   ```
+
+1. :building_construction: Package the TypeScript for distribution
+
+   ```bash
+   npm run bundle
+   ```
+
+1. :white_check_mark: Check your artifact
+
+   There will be a tgz file in your extension directory now
+
+## Update the Extension Metadata
+
+The [`package.json`](package.json) file defines metadata about your extension, such as
+extension name, main entry, description and version.
+
+When you copy this repository, update `package.json` with the name, description for your extension.
+
+## Update the Extension Code
+
+The [`src/`](./src/) directory is the heart of your extension! This contains the
+source code that will be run when your extension functions are invoked. You can replace the
+contents of this directory with your own code.
+
+There are a few things to keep in mind when writing your extension code:
+
+- Most Jan Extension functions are processed asynchronously.
+  In `index.ts`, you will see that the extension function will return a `Promise<any>`.
+
+  ```typescript
+  import { events, MessageEvent, MessageRequest } from '@janhq/core'
+
+  function onStart(): Promise<any> {
+    return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
+      this.inference(data)
+    )
+  }
+  ```
+
+  For more information about the Jan Extension Core module, see the
+  [documentation](https://github.com/janhq/jan/blob/main/core/README.md).
+
+So, what are you waiting for? Go ahead and start customizing your extension!
diff --git a/extensions/inference-martian-extension/package.json b/extensions/inference-martian-extension/package.json
new file mode 100644
index 000000000..15d392b9c
--- /dev/null
+++ b/extensions/inference-martian-extension/package.json
@@ -0,0 +1,42 @@
+{
+  "name": "@janhq/inference-martian-extension",
+  "productName": "Martian Inference Engine",
+  "version": "1.0.1",
+  "description": "This extension enables Martian chat completion API calls",
+  "main": "dist/index.js",
+  "module": "dist/module.js",
+  "engine": "martian",
+  "author": "Jan <service@jan.ai>",
+  "license": "AGPL-3.0",
+  "scripts": {
+    "build": "tsc -b . && webpack --config webpack.config.js",
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./main": "./dist/module.js"
+  },
+  "devDependencies": {
+    "cpx": "^1.5.0",
+    "rimraf": "^3.0.2",
+    "webpack": "^5.88.2",
+    "webpack-cli": "^5.1.4",
+    "ts-loader": "^9.5.0"
+  },
+  "dependencies": {
+    "@janhq/core": "file:../../core",
+    "fetch-retry": "^5.0.6",
+    "ulidx": "^2.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist/*",
+    "package.json",
+    "README.md"
+  ],
+  "bundleDependencies": [
+    "fetch-retry"
+  ]
+}
diff --git a/extensions/inference-martian-extension/resources/models.json b/extensions/inference-martian-extension/resources/models.json
new file mode 100644
index 000000000..cf59e958e
--- /dev/null
+++ b/extensions/inference-martian-extension/resources/models.json
@@ -0,0 +1,32 @@
+[
+  {
+    "sources": [
+      {
+        "url": "https://withmartian.com/"
+      }
+    ],
+    "id": "router",
+    "object": "model",
+    "name": "Martian Model Router",
+    "version": "1.0",
+    "description": "Martian Model Router dynamically routes requests to the best LLM in real-time",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 4096,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
+    },
+    "metadata": {
+      "author": "Martian",
+      "tags": [
+        "General"
+      ]
+    },
+    "engine": "martian"
+  }
+]
\ No newline at end of file
diff --git a/extensions/inference-martian-extension/resources/settings.json b/extensions/inference-martian-extension/resources/settings.json
new file mode 100644
index 000000000..bc83d76d4
--- /dev/null
+++ b/extensions/inference-martian-extension/resources/settings.json
@@ -0,0 +1,23 @@
+[
+  {
+    "key": "chat-completions-endpoint",
+    "title": "Chat Completions Endpoint",
+    "description": "The endpoint to use for chat completions. See the [Martian API documentation](https://docs.withmartian.com/martian-model-router/api-reference/get-chat-completions) for more information.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "https://withmartian.com/api/openai/v1/chat/completions",
+      "value": "https://withmartian.com/api/openai/v1/chat/completions"
+    }
+  },
+  {
+    "key": "martian-api-key",
+    "title": "API Key",
+    "description": "The Martian API uses API keys for authentication. Visit your [API Keys](https://withmartian.com/dashboard) page to retrieve the API key you'll use in your requests.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+      "value": "",
+      "type": "password"
+    }
+  }
+]
diff --git a/extensions/inference-martian-extension/src/index.ts b/extensions/inference-martian-extension/src/index.ts
new file mode 100644
index 000000000..f59a6b7fc
--- /dev/null
+++ b/extensions/inference-martian-extension/src/index.ts
@@ -0,0 +1,66 @@
+/**
+ * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
+ * The class provides methods for initializing and stopping a model, and for making inference requests.
+ * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
+ * @version 1.0.0
+ * @module inference-martian-extension/src/index
+ */
+
+import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
+
+declare const SETTINGS: Array<any>
+declare const MODELS: Array<any>
+
+enum Settings {
+  apiKey = 'martian-api-key',
+  chatCompletionsEndPoint = 'chat-completions-endpoint',
+}
+
+/**
+ * A class that implements the InferenceExtension interface from the @janhq/core package.
+ * The class provides methods for initializing and stopping a model, and for making inference requests.
+ * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
+ */
+export default class JanInferenceMartianExtension extends RemoteOAIEngine {
+  inferenceUrl: string = ''
+  provider: string = 'martian'
+
+  override async onLoad(): Promise<void> {
+    super.onLoad()
+
+    // Register Settings
+    this.registerSettings(SETTINGS)
+    this.registerModels(MODELS)
+
+    this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
+    this.inferenceUrl = await this.getSetting<string>(
+      Settings.chatCompletionsEndPoint,
+      ''
+    )
+    if (this.inferenceUrl.length === 0) {
+      SETTINGS.forEach((setting) => {
+        if (setting.key === Settings.chatCompletionsEndPoint) {
+          this.inferenceUrl = setting.controllerProps.value as string
+        }
+      })
+    }
+  }
+
+  onSettingUpdate<T>(key: string, value: T): void {
+    if (key === Settings.apiKey) {
+      this.apiKey = value as string
+    } else if (key === Settings.chatCompletionsEndPoint) {
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
+    }
+  }
+}
diff --git a/extensions/inference-martian-extension/tsconfig.json b/extensions/inference-martian-extension/tsconfig.json
new file mode 100644
index 000000000..2477d58ce
--- /dev/null
+++ b/extensions/inference-martian-extension/tsconfig.json
@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "ES6",
+    "moduleResolution": "node",
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": false,
+    "skipLibCheck": true,
+    "rootDir": "./src"
+  },
+  "include": ["./src"]
+}
diff --git a/extensions/inference-martian-extension/webpack.config.js b/extensions/inference-martian-extension/webpack.config.js
new file mode 100644
index 000000000..cd5e65c72
--- /dev/null
+++ b/extensions/inference-martian-extension/webpack.config.js
@@ -0,0 +1,37 @@
+const webpack = require('webpack')
+const packageJson = require('./package.json')
+const settingJson = require('./resources/settings.json')
+const modelsJson = require('./resources/models.json')
+
+module.exports = {
+  experiments: { outputModule: true },
+  entry: './src/index.ts', // Adjust the entry point to match your project's main file
+  mode: 'production',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  plugins: [
+    new webpack.DefinePlugin({
+      MODELS: JSON.stringify(modelsJson),
+      SETTINGS: JSON.stringify(settingJson),
+      ENGINE: JSON.stringify(packageJson.engine),
+    }),
+  ],
+  output: {
+    filename: 'index.js', // Adjust the output file name as needed
+    library: { type: 'module' }, // Specify ESM output format
+  },
+  resolve: {
+    extensions: ['.ts', '.js'],
+  },
+  optimization: {
+    minimize: false,
+  },
+  // Add loaders and other configuration as needed for your project
+}

From 9effb6a3a6b9e63f68ac877748855b343a31b983 Mon Sep 17 00:00:00 2001
From: Faisal Amir <urmauur@gmail.com>
Date: Mon, 6 May 2024 16:48:53 +0700
Subject: [PATCH 11/26] fix: validate context length (#2871)

* fix: adjust context_length min and conditional for remote model

* fix: remove console
---
 web/screens/Chat/ModelSetting/SettingComponent.tsx   | 5 ++++-
 web/screens/Chat/ModelSetting/predefinedComponent.ts | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/web/screens/Chat/ModelSetting/SettingComponent.tsx b/web/screens/Chat/ModelSetting/SettingComponent.tsx
index 2fde17276..396043f77 100644
--- a/web/screens/Chat/ModelSetting/SettingComponent.tsx
+++ b/web/screens/Chat/ModelSetting/SettingComponent.tsx
@@ -3,6 +3,7 @@ import {
   InputComponentProps,
   CheckboxComponentProps,
   SliderComponentProps,
+  InferenceEngine,
 } from '@janhq/core'
 
 import { useAtomValue } from 'jotai/react'
@@ -37,7 +38,9 @@ const SettingComponent: React.FC<Props> = ({
             description={data.description}
             min={min}
             max={
-              data.key === 'max_tokens'
+              data.key === 'max_tokens' &&
+              activeThread &&
+              activeThread.assistants[0].model.engine === InferenceEngine.nitro
                 ? Number(
                     activeThread &&
                       activeThread.assistants[0].model.settings.ctx_len
diff --git a/web/screens/Chat/ModelSetting/predefinedComponent.ts b/web/screens/Chat/ModelSetting/predefinedComponent.ts
index 652389d4a..91c3f71e1 100644
--- a/web/screens/Chat/ModelSetting/predefinedComponent.ts
+++ b/web/screens/Chat/ModelSetting/predefinedComponent.ts
@@ -33,7 +33,7 @@ export const presetConfiguration: Record<string, SettingComponentProps> = {
       'The context length for model operations varies; the maximum depends on the specific model used.',
     controllerType: 'slider',
     controllerProps: {
-      min: 0,
+      min: 128,
       max: 4096,
       step: 128,
       value: 2048,

From d2266405cc56e1c584b022d416f2e4b644114e08 Mon Sep 17 00:00:00 2001
From: Inchoker <54902491+Inchoker@users.noreply.github.com>
Date: Mon, 6 May 2024 17:36:52 +0700
Subject: [PATCH 12/26] Add OpenRouter (#2826)

* Add OpenRouter

* fix cohere setting description

* fix: update to auto router

* fix: auto router

* add: config parameters

* fix: correct max tokens

---------

Co-authored-by: Jack Tri Le <Jack>
Co-authored-by: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
---
 .../resources/settings.json                   |  2 +-
 .../inference-openrouter-extension/README.md  | 79 +++++++++++++++++++
 .../package.json                              | 43 ++++++++++
 .../resources/models.json                     | 28 +++++++
 .../resources/settings.json                   | 23 ++++++
 .../src/index.ts                              | 76 ++++++++++++++++++
 .../tsconfig.json                             | 14 ++++
 .../webpack.config.js                         | 37 +++++++++
 8 files changed, 301 insertions(+), 1 deletion(-)
 create mode 100644 extensions/inference-openrouter-extension/README.md
 create mode 100644 extensions/inference-openrouter-extension/package.json
 create mode 100644 extensions/inference-openrouter-extension/resources/models.json
 create mode 100644 extensions/inference-openrouter-extension/resources/settings.json
 create mode 100644 extensions/inference-openrouter-extension/src/index.ts
 create mode 100644 extensions/inference-openrouter-extension/tsconfig.json
 create mode 100644 extensions/inference-openrouter-extension/webpack.config.js

diff --git a/extensions/inference-cohere-extension/resources/settings.json b/extensions/inference-cohere-extension/resources/settings.json
index 9d9fb60de..2a32b57f8 100644
--- a/extensions/inference-cohere-extension/resources/settings.json
+++ b/extensions/inference-cohere-extension/resources/settings.json
@@ -12,7 +12,7 @@
   {
     "key": "cohere-api-key",
     "title": "API Key",
-    "description": "The Cohere API uses API keys for authentication. Visit your [API Keys](https://platform.openai.com/account/api-keys) page to retrieve the API key you'll use in your requests.",
+    "description": "The Cohere API uses API keys for authentication. Visit your [API Keys](https://dashboard.cohere.com/api-keys) page to retrieve the API key you'll use in your requests.",
     "controllerType": "input",
     "controllerProps": {
       "placeholder": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
diff --git a/extensions/inference-openrouter-extension/README.md b/extensions/inference-openrouter-extension/README.md
new file mode 100644
index 000000000..aab10755d
--- /dev/null
+++ b/extensions/inference-openrouter-extension/README.md
@@ -0,0 +1,79 @@
+# Open Router Engine Extension
+
+Created using Jan extension example
+
+# Create a Jan Extension using Typescript
+
+Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
+
+## Create Your Own Extension
+
+To create your own extension, you can use this repository as a template! Just follow the below instructions:
+
+1. Click the Use this template button at the top of the repository
+2. Select Create a new repository
+3. Select an owner and name for your new repository
+4. Click Create repository
+5. Clone your new repository
+
+## Initial Setup
+
+After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
+
+> [!NOTE]
+>
+> You'll need to have a reasonably modern version of
+> [Node.js](https://nodejs.org) handy. If you are using a version manager like
+> [`nodenv`](https://github.com/nodenv/nodenv) or
+> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
+> root of your repository to install the version specified in
+> [`package.json`](./package.json). Otherwise, 20.x or later should work!
+
+1. :hammer_and_wrench: Install the dependencies
+
+   ```bash
+   npm install
+   ```
+
+1. :building_construction: Package the TypeScript for distribution
+
+   ```bash
+   npm run bundle
+   ```
+
+1. :white_check_mark: Check your artifact
+
+   There will be a tgz file in your extension directory now
+
+## Update the Extension Metadata
+
+The [`package.json`](package.json) file defines metadata about your extension, such as
+extension name, main entry, description and version.
+
+When you copy this repository, update `package.json` with the name, description for your extension.
+
+## Update the Extension Code
+
+The [`src/`](./src/) directory is the heart of your extension! This contains the
+source code that will be run when your extension functions are invoked. You can replace the
+contents of this directory with your own code.
+
+There are a few things to keep in mind when writing your extension code:
+
+- Most Jan Extension functions are processed asynchronously.
+  In `index.ts`, you will see that the extension function will return a `Promise<any>`.
+
+  ```typescript
+  import { events, MessageEvent, MessageRequest } from '@janhq/core'
+
+  function onStart(): Promise<any> {
+    return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
+      this.inference(data)
+    )
+  }
+  ```
+
+  For more information about the Jan Extension Core module, see the
+  [documentation](https://github.com/janhq/jan/blob/main/core/README.md).
+
+So, what are you waiting for? Go ahead and start customizing your extension!
diff --git a/extensions/inference-openrouter-extension/package.json b/extensions/inference-openrouter-extension/package.json
new file mode 100644
index 000000000..9d3d68d47
--- /dev/null
+++ b/extensions/inference-openrouter-extension/package.json
@@ -0,0 +1,43 @@
+{
+  "name": "@janhq/inference-openrouter-extension",
+  "productName": "OpenRouter Inference Engine",
+  "version": "1.0.0",
+  "description": "This extension enables Open Router chat completion API calls",
+  "main": "dist/index.js",
+  "module": "dist/module.js",
+  "engine": "openrouter",
+  "author": "Jan <service@jan.ai>",
+  "license": "AGPL-3.0",
+  "scripts": {
+    "build": "tsc -b . && webpack --config webpack.config.js",
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install",
+    "sync:core": "cd ../.. && yarn build:core && cd extensions && rm yarn.lock &&  cd inference-openrouter-extension && yarn && yarn build:publish"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./main": "./dist/module.js"
+  },
+  "devDependencies": {
+    "cpx": "^1.5.0",
+    "rimraf": "^3.0.2",
+    "webpack": "^5.88.2",
+    "webpack-cli": "^5.1.4",
+    "ts-loader": "^9.5.0"
+  },
+  "dependencies": {
+    "@janhq/core": "file:../../core",
+    "fetch-retry": "^5.0.6",
+    "ulidx": "^2.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist/*",
+    "package.json",
+    "README.md"
+  ],
+  "bundleDependencies": [
+    "fetch-retry"
+  ]
+}
diff --git a/extensions/inference-openrouter-extension/resources/models.json b/extensions/inference-openrouter-extension/resources/models.json
new file mode 100644
index 000000000..d89c07e5a
--- /dev/null
+++ b/extensions/inference-openrouter-extension/resources/models.json
@@ -0,0 +1,28 @@
+  [
+  {
+    "sources": [
+      {
+        "url": "https://openrouter.ai"
+      }
+    ],
+    "id": "open-router-auto",
+    "object": "model",
+    "name": "OpenRouter",
+    "version": "1.0",
+    "description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 1024,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "frequency_penalty": 0,
+      "presence_penalty": 0
+    },
+    "metadata": {
+      "author": "OpenRouter",
+      "tags": ["General", "Big Context Length"]
+    },
+    "engine": "openrouter"
+  }
+]
diff --git a/extensions/inference-openrouter-extension/resources/settings.json b/extensions/inference-openrouter-extension/resources/settings.json
new file mode 100644
index 000000000..85040e96b
--- /dev/null
+++ b/extensions/inference-openrouter-extension/resources/settings.json
@@ -0,0 +1,23 @@
+[
+  {
+    "key": "chat-completions-endpoint",
+    "title": "Chat Completions Endpoint",
+    "description": "The endpoint to use for chat completions. See the [OpenRouter API documentation](https://openrouter.ai/docs) for more information.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "https://openrouter.ai/api/v1/chat/completions",
+      "value": "https://openrouter.ai/api/v1/chat/completions"
+    }
+  },
+  {
+    "key": "openrouter-api-key",
+    "title": "API Key",
+    "description": "The OpenRouter API uses API keys for authentication. Visit your [API Keys](https://openrouter.ai/keys) page to retrieve the API key you'll use in your requests.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+      "value": "",
+      "type": "password"
+    }
+  }
+]
diff --git a/extensions/inference-openrouter-extension/src/index.ts b/extensions/inference-openrouter-extension/src/index.ts
new file mode 100644
index 000000000..5417503e5
--- /dev/null
+++ b/extensions/inference-openrouter-extension/src/index.ts
@@ -0,0 +1,76 @@
+/**
+ * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
+ * The class provides methods for initializing and stopping a model, and for making inference requests.
+ * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
+ * @version 1.0.0
+ * @module inference-openai-extension/src/index
+ */
+
+import { RemoteOAIEngine } from '@janhq/core'
+import { PayloadType } from '@janhq/core'
+import { ChatCompletionRole } from '@janhq/core'
+
+declare const SETTINGS: Array<any>
+declare const MODELS: Array<any>
+
+enum Settings {
+  apiKey = 'openrouter-api-key',
+  chatCompletionsEndPoint = 'chat-completions-endpoint',
+}
+
+enum RoleType {
+  user = 'USER',
+  chatbot = 'CHATBOT',
+  system = 'SYSTEM',
+}
+
+/**
+ * A class that implements the InferenceExtension interface from the @janhq/core package.
+ * The class provides methods for initializing and stopping a model, and for making inference requests.
+ * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
+ */
+export default class JanInferenceOpenRouterExtension extends RemoteOAIEngine {
+  inferenceUrl: string = ''
+  provider: string = 'openrouter'
+
+  override async onLoad(): Promise<void> {
+    super.onLoad()
+
+    // Register Settings
+    this.registerSettings(SETTINGS)
+    this.registerModels(MODELS)
+
+    this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
+    this.inferenceUrl = await this.getSetting<string>(
+      Settings.chatCompletionsEndPoint,
+      ''
+    )
+    if (this.inferenceUrl.length === 0) {
+      SETTINGS.forEach((setting) => {
+        if (setting.key === Settings.chatCompletionsEndPoint) {
+          this.inferenceUrl = setting.controllerProps.value as string
+        }
+      })
+    }
+  }
+
+  onSettingUpdate<T>(key: string, value: T): void {
+    if (key === Settings.apiKey) {
+      this.apiKey = value as string
+    } else if (key === Settings.chatCompletionsEndPoint) {
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
+    }
+  }
+
+  transformPayload = (payload: PayloadType)=>({...payload,model:"openrouter/auto"})
+}
diff --git a/extensions/inference-openrouter-extension/tsconfig.json b/extensions/inference-openrouter-extension/tsconfig.json
new file mode 100644
index 000000000..2477d58ce
--- /dev/null
+++ b/extensions/inference-openrouter-extension/tsconfig.json
@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "ES6",
+    "moduleResolution": "node",
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": false,
+    "skipLibCheck": true,
+    "rootDir": "./src"
+  },
+  "include": ["./src"]
+}
diff --git a/extensions/inference-openrouter-extension/webpack.config.js b/extensions/inference-openrouter-extension/webpack.config.js
new file mode 100644
index 000000000..cd5e65c72
--- /dev/null
+++ b/extensions/inference-openrouter-extension/webpack.config.js
@@ -0,0 +1,37 @@
+const webpack = require('webpack')
+const packageJson = require('./package.json')
+const settingJson = require('./resources/settings.json')
+const modelsJson = require('./resources/models.json')
+
+module.exports = {
+  experiments: { outputModule: true },
+  entry: './src/index.ts', // Adjust the entry point to match your project's main file
+  mode: 'production',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  plugins: [
+    new webpack.DefinePlugin({
+      MODELS: JSON.stringify(modelsJson),
+      SETTINGS: JSON.stringify(settingJson),
+      ENGINE: JSON.stringify(packageJson.engine),
+    }),
+  ],
+  output: {
+    filename: 'index.js', // Adjust the output file name as needed
+    library: { type: 'module' }, // Specify ESM output format
+  },
+  resolve: {
+    extensions: ['.ts', '.js'],
+  },
+  optimization: {
+    minimize: false,
+  },
+  // Add loaders and other configuration as needed for your project
+}

From 2008aae100eaeffe4c1614ca895e195364b4cd1a Mon Sep 17 00:00:00 2001
From: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
Date: Mon, 6 May 2024 18:04:51 +0700
Subject: [PATCH 13/26] Feat: Correct context length for models (#2867)

* fix: correct ctx

* version bump

* fix: correct ctxlen

* fix: correct ctxlen

* version bump

* fix: correct ctx + q4

* fix: correct ctxlen

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx len

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx

* fix: correct ctx

* version bump
---
 extensions/inference-nitro-extension/package.json  |  2 +-
 .../resources/models/codeninja-1.0-7b/model.json   |  6 +++---
 .../resources/models/command-r-34b/model.json      |  6 +++---
 .../models/deepseek-coder-1.3b/model.json          |  6 +++---
 .../resources/models/deepseek-coder-34b/model.json | 14 +++++++-------
 .../resources/models/gemma-2b/model.json           |  6 +++---
 .../resources/models/gemma-7b/model.json           |  6 +++---
 .../resources/models/mistral-ins-7b-q4/model.json  |  8 ++++----
 .../models/mixtral-8x7b-instruct/model.json        |  6 +++---
 .../resources/models/noromaid-7b/model.json        |  6 +++---
 .../resources/models/openchat-3.5-7b/model.json    |  6 +++---
 .../resources/models/phind-34b/model.json          |  6 +++---
 .../resources/models/qwen-7b/model.json            |  6 +++---
 .../resources/models/stealth-v1.2-7b/model.json    |  4 ++--
 .../resources/models/trinity-v1.2-7b/model.json    |  4 ++--
 .../resources/models/vistral-7b/model.json         |  6 +++---
 .../resources/models/wizardcoder-13b/model.json    |  4 ++--
 17 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
index 3cfdd3338..e15b09ec0 100644
--- a/extensions/inference-nitro-extension/package.json
+++ b/extensions/inference-nitro-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-nitro-extension",
   "productName": "Nitro Inference Engine",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",
diff --git a/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
index 4ffe355d1..b82a430a1 100644
--- a/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
@@ -8,11 +8,11 @@
   "id": "codeninja-1.0-7b",
   "object": "model",
   "name": "CodeNinja 7B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "CodeNinja is good for coding tasks and can handle various languages including Python, C, C++, Rust, Java, JavaScript, and more.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 8192,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
     "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 8192,
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
diff --git a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
index 2f4b5e0dc..031dc88d4 100644
--- a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
@@ -8,11 +8,11 @@
     "id": "command-r-34b",
     "object": "model",
     "name": "Command-R v01 34B Q4",
-    "version": "1.3",
+    "version": "1.4",
     "description": "C4AI Command-R developed by CohereAI is optimized for a variety of use cases including reasoning, summarization, and question answering.",
     "format": "gguf",
     "settings": {
-      "ctx_len": 4096,
+      "ctx_len": 131072,
       "prompt_template": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
       "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf"
     },
@@ -20,7 +20,7 @@
       "temperature": 0.7,
       "top_p": 0.95,
       "stream": true,
-      "max_tokens": 4096,
+      "max_tokens": 131072,
       "stop": [],
       "frequency_penalty": 0,
       "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
index 365dbfd2f..49814adf2 100644
--- a/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
@@ -8,11 +8,11 @@
   "id": "deepseek-coder-1.3b",
   "object": "model",
   "name": "Deepseek Coder 1.3B Q8",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Deepseek Coder excelled in project-level code completion with advanced capabilities across multiple programming languages.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
     "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 16384,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
index 8e17b9563..9d3f6a4b2 100644
--- a/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
@@ -1,26 +1,26 @@
 {
   "sources": [
     {
-      "filename": "deepseek-coder-33b-instruct.Q5_K_M.gguf",
-      "url": "https://huggingface.co/TheBloke/deepseek-coder-33B-instruct-GGUF/resolve/main/deepseek-coder-33b-instruct.Q5_K_M.gguf"
+      "filename": "deepseek-coder-33b-instruct.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/deepseek-coder-33B-instruct-GGUF/resolve/main/deepseek-coder-33b-instruct.Q4_K_M.gguf"
     }
   ],
   "id": "deepseek-coder-34b",
   "object": "model",
-  "name": "Deepseek Coder 33B Q5",
-  "version": "1.0",
+  "name": "Deepseek Coder 33B Q4",
+  "version": "1.1",
   "description": "Deepseek Coder excelled in project-level code completion with advanced capabilities across multiple programming languages.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
-    "llama_model_path": "deepseek-coder-33b-instruct.Q5_K_M.gguf"
+    "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 16384,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
index 5615d3358..9673b62dd 100644
--- a/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
@@ -8,11 +8,11 @@
   "id": "gemma-2b",
   "object": "model",
   "name": "Gemma 2B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Gemma is built from the same technology with Google's Gemini.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 8192,
     "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
     "llama_model_path": "gemma-2b-it-q4_k_m.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 8192,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
index 043c85b4a..d7384a8b9 100644
--- a/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
@@ -8,11 +8,11 @@
   "id": "gemma-7b",
   "object": "model",
   "name": "Gemma 7B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Google's Gemma is built for multilingual purpose",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 8192,
     "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
     "llama_model_path": "gemma-7b-it-q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 8192,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
index 3f9cab127..73a908e17 100644
--- a/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
+++ b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
@@ -8,11 +8,11 @@
   "id": "mistral-ins-7b-q4",
   "object": "model",
   "name": "Mistral Instruct 7B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Mistral Instruct 7b model, specifically designed for a comprehensive understanding of the world.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 32768,
     "prompt_template": "[INST] {prompt} [/INST]",
     "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
   },
@@ -20,8 +20,8 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
-    "stop": [],
+    "max_tokens": 32768,
+    "stop": ["[/INST]"],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
diff --git a/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
index e0a0ee040..0935a25a3 100644
--- a/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
+++ b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
@@ -8,11 +8,11 @@
   "id": "mixtral-8x7b-instruct",
   "object": "model",
   "name": "Mixtral 8x7B Instruct Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "The Mixtral-8x7B is a pretrained generative Sparse Mixture of Experts. The Mixtral-8x7B outperforms 70B models on most benchmarks.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 32768,
     "prompt_template": "[INST] {prompt} [/INST]",
     "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 32768,
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
diff --git a/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
index 516bc62a9..ab0884936 100644
--- a/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
@@ -8,11 +8,11 @@
   "id": "noromaid-7b",
   "object": "model",
   "name": "Noromaid 7B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "The Noromaid 7b model is designed for role-playing with human-like behavior.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
     "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 32768,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
index 1b4dbae19..b37a470d3 100644
--- a/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
@@ -8,11 +8,11 @@
   "id": "openchat-3.5-7b",
   "object": "model",
   "name": "Openchat-3.5 7B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "The performance of Openchat surpasses ChatGPT-3.5 and Grok-1 across various benchmarks.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 8192,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
     "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 8192,
     "stop": ["<|end_of_turn|>"],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
index 6b0abe2a1..f79f4e2d0 100644
--- a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
@@ -8,11 +8,11 @@
   "id": "phind-34b",
   "object": "model",
   "name": "Phind 34B Q4",
-  "version": "1.1",
+  "version": "1.2",
   "description": "Phind 34B is the best Open-source coding model.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 16384,
     "prompt_template": "### System Prompt\n{system_message}\n### User Message\n{prompt}\n### Assistant",
     "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 16384,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
index 16def5b29..6c0f642e3 100644
--- a/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
@@ -8,11 +8,11 @@
   "id": "qwen-7b",
   "object": "model",
   "name": "Qwen Chat 7B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Qwen is optimized at Chinese, ideal for everyday tasks.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
     "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 32768,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
index 93fa6b610..574b4d893 100644
--- a/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
@@ -12,7 +12,7 @@
   "description": "This is a new experimental family designed to enhance Mathematical and Logical abilities.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
     "llama_model_path": "stealth-v1.3.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 32768,
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
diff --git a/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json
index 14444fbd4..9c988e19a 100644
--- a/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json
@@ -12,7 +12,7 @@
   "description": "Trinity is an experimental model merge using the Slerp method. Recommended for daily assistance purposes.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
     "llama_model_path": "trinity-v1.2.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 32768,
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
diff --git a/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json b/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json
index 83e0294c4..d435fb784 100644
--- a/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json
@@ -8,11 +8,11 @@
     "id": "vistral-7b",
     "object": "model",
     "name": "Vistral 7B Q4",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Vistral 7B has a deep understanding of Vietnamese.",
     "format": "gguf",
     "settings": {
-      "ctx_len": 4096,
+      "ctx_len": 32768,
       "prompt_template": "[INST] <<SYS>>\n{system_message}\n<</SYS>>\n{prompt} [/INST]",
       "llama_model_path": "vistral-7b-chat-dpo.Q4_K_M.gguf"
     },
@@ -20,7 +20,7 @@
       "temperature": 0.7,
       "top_p": 0.95,
       "stream": true,
-      "max_tokens": 4096,
+      "max_tokens": 32768,
       "stop": [],
       "frequency_penalty": 0,
       "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json b/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json
index cae96c26b..b7e84bf88 100644
--- a/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json
@@ -12,7 +12,7 @@
   "description": "WizardCoder 13B is a Python coding model. This model demonstrate high proficiency in specific domains like coding and mathematics.",
   "format": "gguf",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
     "llama_model_path": "wizardcoder-python-13b-v1.0.Q4_K_M.gguf"
   },
@@ -20,7 +20,7 @@
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
-    "max_tokens": 4096,
+    "max_tokens": 16384,
     "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0

From 0406b516156e6a50f8e4e7060675c51a288cfc14 Mon Sep 17 00:00:00 2001
From: NamH <NamNh0122@gmail.com>
Date: Mon, 6 May 2024 20:03:25 +0700
Subject: [PATCH 14/26] fix: stop auto scroll if user manually scrolling up
 (#2874)

Co-authored-by: James <james@jan.ai>
---
 web/containers/ListContainer/index.tsx | 30 +++++++++++++++++++++++---
 web/screens/Chat/ChatBody/index.tsx    |  4 ++--
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/web/containers/ListContainer/index.tsx b/web/containers/ListContainer/index.tsx
index 0d3e6de61..a9205e1bd 100644
--- a/web/containers/ListContainer/index.tsx
+++ b/web/containers/ListContainer/index.tsx
@@ -1,4 +1,4 @@
-import { ReactNode, useEffect, useRef } from 'react'
+import { ReactNode, useCallback, useEffect, useRef } from 'react'
 
 type Props = {
   children: ReactNode
@@ -6,20 +6,44 @@ type Props = {
 
 const ListContainer: React.FC<Props> = ({ children }) => {
   const listRef = useRef<HTMLDivElement>(null)
+  const prevScrollTop = useRef(0)
+  const isUserManuallyScrollingUp = useRef(false)
+
+  const handleScroll = useCallback((event: React.UIEvent<HTMLElement>) => {
+    const currentScrollTop = event.currentTarget.scrollTop
+
+    if (prevScrollTop.current > currentScrollTop) {
+      console.debug('User is manually scrolling up')
+      isUserManuallyScrollingUp.current = true
+    } else {
+      const currentScrollTop = event.currentTarget.scrollTop
+      const scrollHeight = event.currentTarget.scrollHeight
+      const clientHeight = event.currentTarget.clientHeight
+
+      if (currentScrollTop + clientHeight >= scrollHeight) {
+        console.debug('Scrolled to the bottom')
+        isUserManuallyScrollingUp.current = false
+      }
+    }
+
+    prevScrollTop.current = currentScrollTop
+  }, [])
 
   useEffect(() => {
-    const scrollHeight = listRef.current?.scrollHeight ?? 0
+    if (isUserManuallyScrollingUp.current === true) return
 
+    const scrollHeight = listRef.current?.scrollHeight ?? 0
     listRef.current?.scrollTo({
       top: scrollHeight,
       behavior: 'instant',
     })
-  })
+  }, [listRef.current?.scrollHeight, isUserManuallyScrollingUp])
 
   return (
     <div
       ref={listRef}
       className="flex h-full w-full flex-col overflow-y-scroll"
+      onScroll={handleScroll}
     >
       {children}
     </div>
diff --git a/web/screens/Chat/ChatBody/index.tsx b/web/screens/Chat/ChatBody/index.tsx
index 5f89b76cd..7ab36de9d 100644
--- a/web/screens/Chat/ChatBody/index.tsx
+++ b/web/screens/Chat/ChatBody/index.tsx
@@ -22,8 +22,8 @@ const ChatBody: React.FC = () => {
   const downloadedModels = useAtomValue(downloadedModelsAtom)
   const loadModelError = useAtomValue(loadModelErrorAtom)
 
-  if (downloadedModels.length === 0) return <EmptyModel />
-  if (messages.length === 0) return <EmptyThread />
+  if (!downloadedModels.length) return <EmptyModel />
+  if (!messages.length) return <EmptyThread />
 
   return (
     <ListContainer>

From efbc96dad94b62d192533d31176e61bd84920971 Mon Sep 17 00:00:00 2001
From: Henry <150573299+henryh0x1@users.noreply.github.com>
Date: Sat, 11 May 2024 21:22:05 +0900
Subject: [PATCH 15/26] feat: inference anthropic extension (#2885)

* feat: implement inference anthropic extension

* chore: format style and correct typo of other extensions
---
 .../inference-anthropic-extension/README.md   |  79 +++++++++++
 .../package.json                              |  43 ++++++
 .../resources/models.json                     |  83 ++++++++++++
 .../resources/settings.json                   |  23 ++++
 .../src/index.ts                              | 124 ++++++++++++++++++
 .../tsconfig.json                             |  14 ++
 .../webpack.config.js                         |  37 ++++++
 .../inference-cohere-extension/src/index.ts   |   6 +-
 .../resources/models.json                     |  21 ++-
 .../resources/models.json                     |  13 +-
 10 files changed, 432 insertions(+), 11 deletions(-)
 create mode 100644 extensions/inference-anthropic-extension/README.md
 create mode 100644 extensions/inference-anthropic-extension/package.json
 create mode 100644 extensions/inference-anthropic-extension/resources/models.json
 create mode 100644 extensions/inference-anthropic-extension/resources/settings.json
 create mode 100644 extensions/inference-anthropic-extension/src/index.ts
 create mode 100644 extensions/inference-anthropic-extension/tsconfig.json
 create mode 100644 extensions/inference-anthropic-extension/webpack.config.js

diff --git a/extensions/inference-anthropic-extension/README.md b/extensions/inference-anthropic-extension/README.md
new file mode 100644
index 000000000..1c0dcbd3d
--- /dev/null
+++ b/extensions/inference-anthropic-extension/README.md
@@ -0,0 +1,79 @@
+# Anthropic Engine Extension
+
+Created using Jan extension example
+
+# Create a Jan Extension using Typescript
+
+Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
+
+## Create Your Own Extension
+
+To create your own extension, you can use this repository as a template! Just follow the below instructions:
+
+1. Click the Use this template button at the top of the repository
+2. Select Create a new repository
+3. Select an owner and name for your new repository
+4. Click Create repository
+5. Clone your new repository
+
+## Initial Setup
+
+After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
+
+> [!NOTE]
+>
+> You'll need to have a reasonably modern version of
+> [Node.js](https://nodejs.org) handy. If you are using a version manager like
+> [`nodenv`](https://github.com/nodenv/nodenv) or
+> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
+> root of your repository to install the version specified in
+> [`package.json`](./package.json). Otherwise, 20.x or later should work!
+
+1. :hammer_and_wrench: Install the dependencies
+
+   ```bash
+   npm install
+   ```
+
+1. :building_construction: Package the TypeScript for distribution
+
+   ```bash
+   npm run bundle
+   ```
+
+1. :white_check_mark: Check your artifact
+
+   There will be a tgz file in your extension directory now
+
+## Update the Extension Metadata
+
+The [`package.json`](package.json) file defines metadata about your extension, such as
+extension name, main entry, description and version.
+
+When you copy this repository, update `package.json` with the name, description for your extension.
+
+## Update the Extension Code
+
+The [`src/`](./src/) directory is the heart of your extension! This contains the
+source code that will be run when your extension functions are invoked. You can replace the
+contents of this directory with your own code.
+
+There are a few things to keep in mind when writing your extension code:
+
+- Most Jan Extension functions are processed asynchronously.
+  In `index.ts`, you will see that the extension function will return a `Promise<any>`.
+
+  ```typescript
+  import { events, MessageEvent, MessageRequest } from '@janhq/core'
+
+  function onStart(): Promise<any> {
+    return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
+      this.inference(data)
+    )
+  }
+  ```
+
+  For more information about the Jan Extension Core module, see the
+  [documentation](https://github.com/janhq/jan/blob/main/core/README.md).
+
+So, what are you waiting for? Go ahead and start customizing your extension!
diff --git a/extensions/inference-anthropic-extension/package.json b/extensions/inference-anthropic-extension/package.json
new file mode 100644
index 000000000..aa3ff8b2a
--- /dev/null
+++ b/extensions/inference-anthropic-extension/package.json
@@ -0,0 +1,43 @@
+{
+  "name": "@janhq/inference-anthropic-extension",
+  "productName": "Anthropic Inference Engine",
+  "version": "1.0.0",
+  "description": "This extension enables Anthropic chat completion API calls",
+  "main": "dist/index.js",
+  "module": "dist/module.js",
+  "engine": "anthropic",
+  "author": "Jan <service@jan.ai>",
+  "license": "AGPL-3.0",
+  "scripts": {
+    "build": "tsc -b . && webpack --config webpack.config.js",
+    "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install",
+    "sync:core": "cd ../.. && yarn build:core && cd extensions && rm yarn.lock &&  cd inference-anthropic-extension && yarn && yarn build:publish"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./main": "./dist/module.js"
+  },
+  "devDependencies": {
+    "cpx": "^1.5.0",
+    "rimraf": "^3.0.2",
+    "webpack": "^5.88.2",
+    "webpack-cli": "^5.1.4",
+    "ts-loader": "^9.5.0"
+  },
+  "dependencies": {
+    "@janhq/core": "file:../../core",
+    "fetch-retry": "^5.0.6",
+    "ulidx": "^2.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist/*",
+    "package.json",
+    "README.md"
+  ],
+  "bundleDependencies": [
+    "fetch-retry"
+  ]
+}
diff --git a/extensions/inference-anthropic-extension/resources/models.json b/extensions/inference-anthropic-extension/resources/models.json
new file mode 100644
index 000000000..363e0bd38
--- /dev/null
+++ b/extensions/inference-anthropic-extension/resources/models.json
@@ -0,0 +1,83 @@
+[
+  {
+    "sources": [
+      {
+        "url": "https://www.anthropic.com/"
+      }
+    ],
+    "id": "claude-3-opus-20240229",
+    "object": "model",
+    "name": "Claude 3 Opus",
+    "version": "1.0",
+    "description": "Claude 3 Opus is a powerful model suitables for highly complex task.",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 4096,
+      "temperature": 0.7,
+      "stream": false
+    },
+    "metadata": {
+      "author": "Anthropic",
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
+    },
+    "engine": "anthropic"
+  },
+  {
+    "sources": [
+      {
+        "url": "https://www.anthropic.com/"
+      }
+    ],
+    "id": "claude-3-sonnet-20240229",
+    "object": "model",
+    "name": "Claude 3 Sonnet",
+    "version": "1.0",
+    "description": "Claude 3 Sonnet is an ideal model balance of intelligence and speed for enterprise workloads.",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 4096,
+      "temperature": 0.7,
+      "stream": false
+    },
+    "metadata": {
+      "author": "Anthropic",
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
+    },
+    "engine": "anthropic"
+  },
+  {
+    "sources": [
+      {
+        "url": "https://www.anthropic.com/"
+      }
+    ],
+    "id": "claude-3-haiku-20240307",
+    "object": "model",
+    "name": "Claude 3 Haiku",
+    "version": "1.0",
+    "description": "Claude 3 Haiku is the fastest model provides near-instant responsiveness.",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 4096,
+      "temperature": 0.7,
+      "stream": false
+    },
+    "metadata": {
+      "author": "Anthropic",
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
+    },
+    "engine": "anthropic"
+  }
+]
\ No newline at end of file
diff --git a/extensions/inference-anthropic-extension/resources/settings.json b/extensions/inference-anthropic-extension/resources/settings.json
new file mode 100644
index 000000000..bb35e6b3d
--- /dev/null
+++ b/extensions/inference-anthropic-extension/resources/settings.json
@@ -0,0 +1,23 @@
+[
+  {
+    "key": "chat-completions-endpoint",
+    "title": "Chat Completions Endpoint",
+    "description": "The endpoint to use for chat completions. See the [Anthropic API documentation](https://docs.anthropic.com/claude/docs/intro-to-claude) for more information.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "https://api.anthropic.com/v1/messages",
+      "value": "https://api.anthropic.com/v1/messages"
+    }
+  },
+  {
+    "key": "anthropic-api-key",
+    "title": "API Key",
+    "description": "The Anthropic API uses API keys for authentication. Visit your [API Keys](https://console.anthropic.com/settings/keys) page to retrieve the API key you'll use in your requests.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+      "value": "",
+      "type": "password"
+    }
+  }
+]
\ No newline at end of file
diff --git a/extensions/inference-anthropic-extension/src/index.ts b/extensions/inference-anthropic-extension/src/index.ts
new file mode 100644
index 000000000..c625d775a
--- /dev/null
+++ b/extensions/inference-anthropic-extension/src/index.ts
@@ -0,0 +1,124 @@
+/**
+ * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
+ * The class provides methods for initializing and stopping a model, and for making inference requests.
+ * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
+ * @version 1.0.0
+ * @module inference-anthropic-extension/src/index
+ */
+
+import { RemoteOAIEngine } from '@janhq/core'
+import { PayloadType } from '@janhq/core'
+import { ChatCompletionRole } from '@janhq/core'
+
+declare const SETTINGS: Array<any>
+declare const MODELS: Array<any>
+
+enum Settings {
+  apiKey = 'anthropic-api-key',
+  chatCompletionsEndPoint = 'chat-completions-endpoint',
+}
+
+type AnthropicPayloadType = {
+  model?: string
+  max_tokens?: number
+  messages?: Array<{ role: string; content: string }>
+}
+
+/**
+ * A class that implements the InferenceExtension interface from the @janhq/core package.
+ * The class provides methods for initializing and stopping a model, and for making inference requests.
+ * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
+ */
+export default class JanInferenceAnthropicExtension extends RemoteOAIEngine {
+  inferenceUrl: string = ''
+  provider: string = 'anthropic'
+  maxTokens: number = 4096
+
+  override async onLoad(): Promise<void> {
+    super.onLoad()
+
+    // Register Settings
+    this.registerSettings(SETTINGS)
+    this.registerModels(MODELS)
+
+    this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
+    this.inferenceUrl = await this.getSetting<string>(
+      Settings.chatCompletionsEndPoint,
+      ''
+    )
+
+    if (this.inferenceUrl.length === 0) {
+      SETTINGS.forEach((setting) => {
+        if (setting.key === Settings.chatCompletionsEndPoint) {
+          this.inferenceUrl = setting.controllerProps.value as string
+        }
+      })
+    }
+  }
+
+  // Override the headers method to include the x-API-key in the request headers
+  override async headers(): Promise<HeadersInit> {
+    return {
+      'Content-Type': 'application/json',
+      'x-api-key': this.apiKey,
+      'anthropic-version': '2023-06-01',
+    }
+  }
+
+  onSettingUpdate<T>(key: string, value: T): void {
+    if (key === Settings.apiKey) {
+      this.apiKey = value as string
+    } else if (key === Settings.chatCompletionsEndPoint) {
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
+    }
+  }
+
+  // Override the transformPayload method to convert the payload to the required format
+  transformPayload = (payload: PayloadType): AnthropicPayloadType => {
+    if (!payload.messages || payload.messages.length === 0) {
+      return { max_tokens: this.maxTokens, messages: [], model: payload.model }
+    }
+
+    const convertedData: AnthropicPayloadType = {
+      max_tokens: this.maxTokens,
+      messages: [],
+      model: payload.model,
+    }
+
+    payload.messages.forEach((item, index) => {
+      if (item.role === ChatCompletionRole.User) {
+        convertedData.messages.push({
+          role: 'user',
+          content: item.content as string,
+        })
+      } else if (item.role === ChatCompletionRole.Assistant) {
+        convertedData.messages.push({
+          role: 'assistant',
+          content: item.content as string,
+        })
+      }
+    })
+
+    return convertedData
+  }
+
+  // Override the transformResponse method to convert the response to the required format
+  transformResponse = (data: any): string => {
+    if (data.content && data.content.length > 0 && data.content[0].text) {
+      return data.content[0].text
+    } else {
+      console.error('Invalid response format:', data)
+      return ''
+    }
+  }
+}
diff --git a/extensions/inference-anthropic-extension/tsconfig.json b/extensions/inference-anthropic-extension/tsconfig.json
new file mode 100644
index 000000000..2477d58ce
--- /dev/null
+++ b/extensions/inference-anthropic-extension/tsconfig.json
@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "ES6",
+    "moduleResolution": "node",
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": false,
+    "skipLibCheck": true,
+    "rootDir": "./src"
+  },
+  "include": ["./src"]
+}
diff --git a/extensions/inference-anthropic-extension/webpack.config.js b/extensions/inference-anthropic-extension/webpack.config.js
new file mode 100644
index 000000000..cd5e65c72
--- /dev/null
+++ b/extensions/inference-anthropic-extension/webpack.config.js
@@ -0,0 +1,37 @@
+const webpack = require('webpack')
+const packageJson = require('./package.json')
+const settingJson = require('./resources/settings.json')
+const modelsJson = require('./resources/models.json')
+
+module.exports = {
+  experiments: { outputModule: true },
+  entry: './src/index.ts', // Adjust the entry point to match your project's main file
+  mode: 'production',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  plugins: [
+    new webpack.DefinePlugin({
+      MODELS: JSON.stringify(modelsJson),
+      SETTINGS: JSON.stringify(settingJson),
+      ENGINE: JSON.stringify(packageJson.engine),
+    }),
+  ],
+  output: {
+    filename: 'index.js', // Adjust the output file name as needed
+    library: { type: 'module' }, // Specify ESM output format
+  },
+  resolve: {
+    extensions: ['.ts', '.js'],
+  },
+  optimization: {
+    minimize: false,
+  },
+  // Add loaders and other configuration as needed for your project
+}
diff --git a/extensions/inference-cohere-extension/src/index.ts b/extensions/inference-cohere-extension/src/index.ts
index b986a25eb..24cc5935b 100644
--- a/extensions/inference-cohere-extension/src/index.ts
+++ b/extensions/inference-cohere-extension/src/index.ts
@@ -3,7 +3,7 @@
  * The class provides methods for initializing and stopping a model, and for making inference requests.
  * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
  * @version 1.0.0
- * @module inference-openai-extension/src/index
+ * @module inference-cohere-extension/src/index
  */
 
 import { RemoteOAIEngine } from '@janhq/core'
@@ -82,7 +82,7 @@ export default class JanInferenceCohereExtension extends RemoteOAIEngine {
     if (payload.messages.length === 0) {
       return {}
     }
-    const convertedData:CoherePayloadType = {
+    const convertedData: CoherePayloadType = {
       chat_history: [],
       message: '',
     }
@@ -93,7 +93,7 @@ export default class JanInferenceCohereExtension extends RemoteOAIEngine {
         return
       }
       if (item.role === ChatCompletionRole.User) {
-        convertedData.chat_history.push({ role: RoleType.user, message: item.content as string})
+        convertedData.chat_history.push({ role: RoleType.user, message: item.content as string })
       } else if (item.role === ChatCompletionRole.Assistant) {
         convertedData.chat_history.push({
           role: RoleType.chatbot,
diff --git a/extensions/inference-groq-extension/resources/models.json b/extensions/inference-groq-extension/resources/models.json
index 32ec60add..81275f47c 100644
--- a/extensions/inference-groq-extension/resources/models.json
+++ b/extensions/inference-groq-extension/resources/models.json
@@ -23,7 +23,10 @@
     },
     "metadata": {
       "author": "Meta",
-      "tags": ["General", "Big Context Length"]
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
     },
     "engine": "groq"
   },
@@ -51,7 +54,10 @@
     },
     "metadata": {
       "author": "Meta",
-      "tags": ["General", "Big Context Length"]
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
     },
     "engine": "groq"
   },
@@ -79,7 +85,9 @@
     },
     "metadata": {
       "author": "Google",
-      "tags": ["General"]
+      "tags": [
+        "General"
+      ]
     },
     "engine": "groq"
   },
@@ -107,8 +115,11 @@
     },
     "metadata": {
       "author": "Mistral",
-      "tags": ["General", "Big Context Length"]
+      "tags": [
+        "General",
+        "Big Context Length"
+      ]
     },
     "engine": "groq"
   }
-]
+]
\ No newline at end of file
diff --git a/extensions/inference-openai-extension/resources/models.json b/extensions/inference-openai-extension/resources/models.json
index d8aa787d9..eb2fc662c 100644
--- a/extensions/inference-openai-extension/resources/models.json
+++ b/extensions/inference-openai-extension/resources/models.json
@@ -23,7 +23,9 @@
     },
     "metadata": {
       "author": "OpenAI",
-      "tags": ["General"]
+      "tags": [
+        "General"
+      ]
     },
     "engine": "openai"
   },
@@ -51,7 +53,10 @@
     },
     "metadata": {
       "author": "OpenAI",
-      "tags": ["General", "Vision"]
+      "tags": [
+        "General",
+        "Vision"
+      ]
     },
     "engine": "openai"
   },
@@ -79,7 +84,9 @@
     },
     "metadata": {
       "author": "OpenAI",
-      "tags": ["General"]
+      "tags": [
+        "General"
+      ]
     },
     "engine": "openai"
   }

From 6af4a2d4849a60b96b6c123356b6ed6e91946471 Mon Sep 17 00:00:00 2001
From: NamH <NamNh0122@gmail.com>
Date: Mon, 13 May 2024 09:37:05 +0700
Subject: [PATCH 16/26] feat: add deeplink support (#2883)

* feat: add deeplink support

* fix windows not receive deeplink when not running

---------

Co-authored-by: James <james@jan.ai>
---
 core/src/types/api/index.ts                   |  3 +
 electron/handlers/native.ts                   |  4 ++
 electron/main.ts                              | 45 +++++++++---
 electron/managers/window.ts                   | 36 +++++++++-
 electron/package.json                         |  8 +++
 electron/utils/dev.ts                         | 24 +++----
 web/containers/Layout/index.tsx               |  3 +
 web/containers/LoadingModal/index.tsx         | 28 ++++++++
 web/containers/Providers/DeepLinkListener.tsx | 72 +++++++++++++++++++
 web/containers/Providers/index.tsx            |  5 +-
 10 files changed, 202 insertions(+), 26 deletions(-)
 create mode 100644 web/containers/LoadingModal/index.tsx
 create mode 100644 web/containers/Providers/DeepLinkListener.tsx

diff --git a/core/src/types/api/index.ts b/core/src/types/api/index.ts
index 1a95ad9c9..fb0dc5b93 100644
--- a/core/src/types/api/index.ts
+++ b/core/src/types/api/index.ts
@@ -19,6 +19,7 @@ export enum NativeRoute {
   showMainWindow = 'showMainWindow',
 
   quickAskSizeUpdated = 'quickAskSizeUpdated',
+  ackDeepLink = 'ackDeepLink',
 }
 
 /**
@@ -45,6 +46,8 @@ export enum AppEvent {
 
   onUserSubmitQuickAsk = 'onUserSubmitQuickAsk',
   onSelectedText = 'onSelectedText',
+
+  onDeepLink = 'onDeepLink',
 }
 
 export enum DownloadRoute {
diff --git a/electron/handlers/native.ts b/electron/handlers/native.ts
index 556b66e66..89bce15df 100644
--- a/electron/handlers/native.ts
+++ b/electron/handlers/native.ts
@@ -151,4 +151,8 @@ export function handleAppIPCs() {
     async (_event, heightOffset: number): Promise<void> =>
       windowManager.expandQuickAskWindow(heightOffset)
   )
+
+  ipcMain.handle(NativeRoute.ackDeepLink, async (_event): Promise<void> => {
+    windowManager.ackDeepLink()
+  })
 }
diff --git a/electron/main.ts b/electron/main.ts
index 1f4719e8d..9f0bd8393 100644
--- a/electron/main.ts
+++ b/electron/main.ts
@@ -1,6 +1,6 @@
 import { app, BrowserWindow } from 'electron'
 
-import { join } from 'path'
+import { join, resolve } from 'path'
 /**
  * Managers
  **/
@@ -39,15 +39,44 @@ const quickAskUrl = `${mainUrl}/search`
 
 const gotTheLock = app.requestSingleInstanceLock()
 
+if (process.defaultApp) {
+  if (process.argv.length >= 2) {
+    app.setAsDefaultProtocolClient('jan', process.execPath, [
+      resolve(process.argv[1]),
+    ])
+  }
+} else {
+  app.setAsDefaultProtocolClient('jan')
+}
+
+const createMainWindow = () => {
+  const startUrl = app.isPackaged ? `file://${mainPath}` : mainUrl
+  windowManager.createMainWindow(preloadPath, startUrl)
+}
+
 app
   .whenReady()
   .then(() => {
     if (!gotTheLock) {
       app.quit()
       throw new Error('Another instance of the app is already running')
+    } else {
+      app.on(
+        'second-instance',
+        (_event, commandLine, _workingDirectory): void => {
+          if (process.platform === 'win32' || process.platform === 'linux') {
+            // this is for handling deeplink on windows and linux
+            // since those OS will emit second-instance instead of open-url
+            const url = commandLine.pop()
+            if (url) {
+              windowManager.sendMainAppDeepLink(url)
+            }
+          }
+          windowManager.showMainWindow()
+        }
+      )
     }
   })
-  .then(setupReactDevTool)
   .then(setupCore)
   .then(createUserSpace)
   .then(migrateExtensions)
@@ -60,6 +89,7 @@ app
   .then(registerGlobalShortcuts)
   .then(() => {
     if (!app.isPackaged) {
+      setupReactDevTool()
       windowManager.mainWindow?.webContents.openDevTools()
     }
   })
@@ -75,11 +105,11 @@ app
     })
   })
 
-app.on('second-instance', (_event, _commandLine, _workingDirectory) => {
-  windowManager.showMainWindow()
+app.on('open-url', (_event, url) => {
+  windowManager.sendMainAppDeepLink(url)
 })
 
-app.on('before-quit', function (evt) {
+app.on('before-quit', function (_event) {
   trayManager.destroyCurrentTray()
 })
 
@@ -104,11 +134,6 @@ function createQuickAskWindow() {
   windowManager.createQuickAskWindow(preloadPath, startUrl)
 }
 
-function createMainWindow() {
-  const startUrl = app.isPackaged ? `file://${mainPath}` : mainUrl
-  windowManager.createMainWindow(preloadPath, startUrl)
-}
-
 /**
  * Handles various IPC messages from the renderer process.
  */
diff --git a/electron/managers/window.ts b/electron/managers/window.ts
index 8c7348651..6052f332a 100644
--- a/electron/managers/window.ts
+++ b/electron/managers/window.ts
@@ -14,9 +14,9 @@ class WindowManager {
   private _quickAskWindowVisible = false
   private _mainWindowVisible = false
 
+  private deeplink: string | undefined
   /**
    * Creates a new window instance.
-   * @param {Electron.BrowserWindowConstructorOptions} options - The options to create the window with.
    * @returns The created window instance.
    */
   createMainWindow(preloadPath: string, startUrl: string) {
@@ -29,6 +29,17 @@ class WindowManager {
       },
     })
 
+    if (process.platform === 'win32') {
+      /// This is work around for windows deeplink.
+      /// second-instance event is not fired when app is not open, so the app
+      /// does not received the deeplink.
+      const commandLine = process.argv.slice(1)
+      if (commandLine.length > 0) {
+        const url = commandLine[0]
+        this.sendMainAppDeepLink(url)
+      }
+    }
+
     /* Load frontend app to the window */
     this.mainWindow.loadURL(startUrl)
 
@@ -123,6 +134,22 @@ class WindowManager {
     )
   }
 
+  /**
+   * Try to send the deep link to the main app.
+   */
+  sendMainAppDeepLink(url: string): void {
+    this.deeplink = url
+    const interval = setInterval(() => {
+      if (!this.deeplink) clearInterval(interval)
+      const mainWindow = this.mainWindow
+      if (mainWindow) {
+        mainWindow.webContents.send(AppEvent.onDeepLink, this.deeplink)
+        if (mainWindow.isMinimized()) mainWindow.restore()
+        mainWindow.focus()
+      }
+    }, 500)
+  }
+
   cleanUp(): void {
     if (!this.mainWindow?.isDestroyed()) {
       this.mainWindow?.close()
@@ -137,6 +164,13 @@ class WindowManager {
       this._quickAskWindowVisible = false
     }
   }
+
+  /**
+   * Acknowledges that the window has received a deep link. We can remove it.
+   */
+  ackDeepLink() {
+    this.deeplink = undefined
+  }
 }
 
 export const windowManager = new WindowManager()
diff --git a/electron/package.json b/electron/package.json
index f012055e2..fc81fcb19 100644
--- a/electron/package.json
+++ b/electron/package.json
@@ -61,6 +61,14 @@
       "include": "scripts/uninstaller.nsh",
       "deleteAppDataOnUninstall": true
     },
+    "protocols": [
+      {
+        "name": "Jan",
+        "schemes": [
+          "jan"
+        ]
+      }
+    ],
     "artifactName": "jan-${os}-${arch}-${version}.${ext}"
   },
   "scripts": {
diff --git a/electron/utils/dev.ts b/electron/utils/dev.ts
index 16e5241b6..bd510096b 100644
--- a/electron/utils/dev.ts
+++ b/electron/utils/dev.ts
@@ -1,17 +1,13 @@
-import { app } from 'electron'
-
 export const setupReactDevTool = async () => {
-  if (!app.isPackaged) {
-    // Which means you're running from source code
-    const { default: installExtension, REACT_DEVELOPER_TOOLS } = await import(
-      'electron-devtools-installer'
-    ) // Don't use import on top level, since the installer package is dev-only
-    try {
-      const name = await installExtension(REACT_DEVELOPER_TOOLS)
-      console.debug(`Added Extension: ${name}`)
-    } catch (err) {
-      console.error('An error occurred while installing devtools:', err)
-      // Only log the error and don't throw it because it's not critical
-    }
+  // Which means you're running from source code
+  const { default: installExtension, REACT_DEVELOPER_TOOLS } = await import(
+    'electron-devtools-installer'
+  ) // Don't use import on top level, since the installer package is dev-only
+  try {
+    const name = await installExtension(REACT_DEVELOPER_TOOLS)
+    console.debug(`Added Extension: ${name}`)
+  } catch (err) {
+    console.error('An error occurred while installing devtools:', err)
+    // Only log the error and don't throw it because it's not critical
   }
 }
diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx
index 6e3c78a94..2e7db1610 100644
--- a/web/containers/Layout/index.tsx
+++ b/web/containers/Layout/index.tsx
@@ -25,6 +25,8 @@ import ImportModelOptionModal from '@/screens/Settings/ImportModelOptionModal'
 import ImportingModelModal from '@/screens/Settings/ImportingModelModal'
 import SelectingModelModal from '@/screens/Settings/SelectingModelModal'
 
+import LoadingModal from '../LoadingModal'
+
 import MainViewContainer from '../MainViewContainer'
 
 import InstallingExtensionModal from './BottomBar/InstallingExtension/InstallingExtensionModal'
@@ -69,6 +71,7 @@ const BaseLayout = () => {
           <BottomBar />
         </div>
       </div>
+      <LoadingModal />
       {importModelStage === 'SELECTING_MODEL' && <SelectingModelModal />}
       {importModelStage === 'MODEL_SELECTED' && <ImportModelOptionModal />}
       {importModelStage === 'IMPORTING_MODEL' && <ImportingModelModal />}
diff --git a/web/containers/LoadingModal/index.tsx b/web/containers/LoadingModal/index.tsx
new file mode 100644
index 000000000..cfaf05c7e
--- /dev/null
+++ b/web/containers/LoadingModal/index.tsx
@@ -0,0 +1,28 @@
+import { Modal, ModalContent, ModalHeader, ModalTitle } from '@janhq/uikit'
+import { atom, useAtomValue } from 'jotai'
+
+export type LoadingInfo = {
+  title: string
+  message: string
+}
+
+export const loadingModalVisibilityAtom = atom<LoadingInfo | undefined>(
+  undefined
+)
+
+const ResettingModal: React.FC = () => {
+  const loadingInfo = useAtomValue(loadingModalVisibilityAtom)
+
+  return (
+    <Modal open={loadingInfo != null}>
+      <ModalContent>
+        <ModalHeader>
+          <ModalTitle>{loadingInfo?.title}</ModalTitle>
+        </ModalHeader>
+        <p className="text-muted-foreground">{loadingInfo?.message}</p>
+      </ModalContent>
+    </Modal>
+  )
+}
+
+export default ResettingModal
diff --git a/web/containers/Providers/DeepLinkListener.tsx b/web/containers/Providers/DeepLinkListener.tsx
new file mode 100644
index 000000000..ca275e52c
--- /dev/null
+++ b/web/containers/Providers/DeepLinkListener.tsx
@@ -0,0 +1,72 @@
+import { Fragment, ReactNode } from 'react'
+
+import { useSetAtom } from 'jotai'
+
+import { useDebouncedCallback } from 'use-debounce'
+
+import { useGetHFRepoData } from '@/hooks/useGetHFRepoData'
+
+import { loadingModalVisibilityAtom as loadingModalInfoAtom } from '../LoadingModal'
+import { toaster } from '../Toast'
+
+import {
+  importHuggingFaceModelStageAtom,
+  importingHuggingFaceRepoDataAtom,
+} from '@/helpers/atoms/HuggingFace.atom'
+type Props = {
+  children: ReactNode
+}
+
+const DeepLinkListener: React.FC<Props> = ({ children }) => {
+  const { getHfRepoData } = useGetHFRepoData()
+  const setLoadingInfo = useSetAtom(loadingModalInfoAtom)
+  const setImportingHuggingFaceRepoData = useSetAtom(
+    importingHuggingFaceRepoDataAtom
+  )
+  const setImportHuggingFaceModelStage = useSetAtom(
+    importHuggingFaceModelStageAtom
+  )
+
+  const debounced = useDebouncedCallback(async (searchText) => {
+    if (searchText.indexOf('/') === -1) {
+      toaster({
+        title: 'Failed to get Hugging Face models',
+        description: 'Invalid Hugging Face model URL',
+        type: 'error',
+      })
+      return
+    }
+
+    try {
+      setLoadingInfo({
+        title: 'Getting Hugging Face models',
+        message: 'Please wait..',
+      })
+      const data = await getHfRepoData(searchText)
+      setImportingHuggingFaceRepoData(data)
+      setImportHuggingFaceModelStage('REPO_DETAIL')
+      setLoadingInfo(undefined)
+    } catch (err) {
+      setLoadingInfo(undefined)
+      let errMessage = 'Unexpected Error'
+      if (err instanceof Error) {
+        errMessage = err.message
+      }
+      toaster({
+        title: 'Failed to get Hugging Face models',
+        description: errMessage,
+        type: 'error',
+      })
+      console.error(err)
+    }
+  }, 300)
+  window.electronAPI?.onDeepLink((_event: string, input: string) => {
+    window.core?.api?.ackDeepLink()
+    const url = input.replaceAll('jan://', '')
+    debounced(url)
+  })
+
+  return <Fragment>{children}</Fragment>
+}
+
+export default DeepLinkListener
diff --git a/web/containers/Providers/index.tsx b/web/containers/Providers/index.tsx
index 66ba42a7d..0b5e236e0 100644
--- a/web/containers/Providers/index.tsx
+++ b/web/containers/Providers/index.tsx
@@ -22,6 +22,7 @@ import Loader from '../Loader'
 
 import DataLoader from './DataLoader'
 
+import DeepLinkListener from './DeepLinkListener'
 import KeyListener from './KeyListener'
 
 import { extensionManager } from '@/extension'
@@ -78,7 +79,9 @@ const Providers = ({ children }: PropsWithChildren) => {
           <KeyListener>
             <EventListenerWrapper>
               <TooltipProvider delayDuration={0}>
-                <DataLoader>{children}</DataLoader>
+                <DataLoader>
+                  <DeepLinkListener>{children}</DeepLinkListener>
+                </DataLoader>
               </TooltipProvider>
             </EventListenerWrapper>
             <Toaster />

From 1e0d4f37538f21c556084db8c9b76d51052b9940 Mon Sep 17 00:00:00 2001
From: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
Date: Mon, 13 May 2024 11:48:03 +0700
Subject: [PATCH 17/26] Feat: Adjust model hub v0.4.13 (#2879)

* fix: correct phi3

* redundant phi2 dolphin

* add: hermes llama3

* add: ngl settings

* correct ctx len

* correct ngl

* correct maxlen + ngl

* disable phi3

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* remove redundant  hermes pro

* add ngl

* add ngl

* add ngl

* remove miqu

* add ngl

* add ngl

* add ngl

* add ngl

* remove redundant

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* add ngl

* version package bump

* feat: resolve issue of cannot found model in the extensions due to the removal

* feat: completely remove hermes-pro-7b

* feat: completely remove openhermes-neural-7b and miqu-70b, and add llama3-hermes-8b via renaming from Rex

* fix: correct description

---------

Co-authored-by: Van-QA <van@jan.ai>
---
 .../inference-nitro-extension/package.json    |  2 +-
 .../models/codeninja-1.0-7b/model.json        |  3 +-
 .../resources/models/command-r-34b/model.json |  3 +-
 .../models/deepseek-coder-1.3b/model.json     |  3 +-
 .../models/deepseek-coder-34b/model.json      |  3 +-
 .../resources/models/dolphin-phi-2/model.json | 32 -----------------
 .../resources/models/gemma-2b/model.json      |  3 +-
 .../resources/models/gemma-7b/model.json      |  3 +-
 .../models/llama2-chat-70b/model.json         |  3 +-
 .../models/llama2-chat-7b/model.json          |  3 +-
 .../models/llama3-8b-instruct/model.json      |  7 ++--
 .../model.json                                | 25 ++++++++------
 .../models/llamacorn-1.1b/model.json          |  3 +-
 .../resources/models/miqu-70b/model.json      | 34 -------------------
 .../models/mistral-ins-7b-q4/model.json       |  3 +-
 .../models/mixtral-8x7b-instruct/model.json   |  3 +-
 .../resources/models/noromaid-7b/model.json   |  3 +-
 .../models/openchat-3.5-7b/model.json         |  3 +-
 .../models/openhermes-neural-7b/model.json    | 34 -------------------
 .../resources/models/phi3-3.8b/model.json     | 32 -----------------
 .../resources/models/phind-34b/model.json     |  3 +-
 .../resources/models/qwen-7b/model.json       |  3 +-
 .../models/stable-zephyr-3b/model.json        |  3 +-
 .../models/stealth-v1.2-7b/model.json         |  3 +-
 .../models/tinyllama-1.1b/model.json          |  3 +-
 .../models/trinity-v1.2-7b/model.json         |  3 +-
 .../resources/models/vistral-7b/model.json    |  3 +-
 .../models/wizardcoder-13b/model.json         |  3 +-
 .../resources/models/yi-34b/model.json        |  3 +-
 .../rollup.config.ts                          | 12 ++-----
 30 files changed, 66 insertions(+), 178 deletions(-)
 delete mode 100644 extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json
 rename extensions/inference-nitro-extension/resources/models/{hermes-pro-7b => llama3-hermes-8b}/model.json (54%)
 delete mode 100644 extensions/inference-nitro-extension/resources/models/miqu-70b/model.json
 delete mode 100644 extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json
 delete mode 100644 extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json

diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
index e15b09ec0..37a75c01f 100644
--- a/extensions/inference-nitro-extension/package.json
+++ b/extensions/inference-nitro-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-nitro-extension",
   "productName": "Nitro Inference Engine",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",
diff --git a/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
index b82a430a1..8497aa11c 100644
--- a/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
-    "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf"
+    "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
index 031dc88d4..fdf638d83 100644
--- a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 131072,
       "prompt_template": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
-      "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf"
+      "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf",
+      "ngl": 40
     },
     "parameters": {
       "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
index 49814adf2..f8fe7344c 100644
--- a/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
-    "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf"
+    "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf",
+    "ngl": 24
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
index 9d3f6a4b2..b488e6bbb 100644
--- a/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
-    "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf"
+    "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf",
+    "ngl": 62
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json b/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json
deleted file mode 100644
index b2a837bf0..000000000
--- a/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-    "sources": [
-      {
-        "url": "https://huggingface.co/TheBloke/dolphin-2_6-phi-2-GGUF/resolve/main/dolphin-2_6-phi-2.Q8_0.gguf",
-        "filename": "dolphin-2_6-phi-2.Q8_0.gguf"
-      }
-    ],
-    "id": "dolphin-phi-2",
-    "object": "model",
-    "name": "Dolphin Phi-2 2.7B Q8",
-    "version": "1.0",
-    "description": "Dolphin Phi-2 is a good alternative for Phi-2 in chatting",
-    "format": "gguf",
-    "settings": {
-      "ctx_len": 4096,
-      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-      "llama_model_path": "dolphin-2_6-phi-2.Q8_0.gguf"
-    },
-    "parameters": {
-      "max_tokens": 4096,
-      "stop": ["<|im_end|>"]
-    },
-    "metadata": {
-      "author": "Cognitive Computations, Microsoft",
-      "tags": [
-        "3B",
-        "Finetuned"
-      ],
-      "size": 2960000000
-    },
-    "engine": "nitro"
-  }
diff --git a/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
index 9673b62dd..a9acb6ef8 100644
--- a/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
-    "llama_model_path": "gemma-2b-it-q4_k_m.gguf"
+    "llama_model_path": "gemma-2b-it-q4_k_m.gguf",
+    "ngl": 18
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
index d7384a8b9..96afe7a61 100644
--- a/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
-    "llama_model_path": "gemma-7b-it-q4_K_M.gguf"
+    "llama_model_path": "gemma-7b-it-q4_K_M.gguf",
+    "ngl": 28
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json
index 34180604b..4b255c9e2 100644
--- a/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
-    "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf"
+    "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf",
+    "ngl": 80
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json
index 4f6d0b9e3..b7d3eeb80 100644
--- a/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
-    "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf"
+    "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json
index 4dbb941ef..7bed6e43c 100644
--- a/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json
+++ b/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json
@@ -8,19 +8,20 @@
     "id": "llama3-8b-instruct",
     "object": "model",
     "name": "Llama 3 8B Q4",
-    "version": "1.0",
+    "version": "1.1",
     "description": "Meta's Llama 3 excels at general usage situations, including chat, general world knowledge, and coding.",
     "format": "gguf",
     "settings": {
       "ctx_len": 8192,
       "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
-      "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf"
+      "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,
       "top_p": 0.95,
       "stream": true,
-      "max_tokens": 4096,
+      "max_tokens": 8192,
       "stop": ["<|end_of_text|>","<|eot_id|>"],
       "frequency_penalty": 0,
       "presence_penalty": 0
diff --git a/extensions/inference-nitro-extension/resources/models/hermes-pro-7b/model.json b/extensions/inference-nitro-extension/resources/models/llama3-hermes-8b/model.json
similarity index 54%
rename from extensions/inference-nitro-extension/resources/models/hermes-pro-7b/model.json
rename to extensions/inference-nitro-extension/resources/models/llama3-hermes-8b/model.json
index e478ff4cd..16d50b9f9 100644
--- a/extensions/inference-nitro-extension/resources/models/hermes-pro-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/llama3-hermes-8b/model.json
@@ -1,35 +1,38 @@
 {
     "sources": [
       {
-        "filename": "Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf",
-        "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/resolve/main/Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf"
+        "filename": "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
+        "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF/resolve/main/Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf"
       }
     ],
-    "id": "hermes-pro-7b",
+    "id": "llama3-hermes-8b",
     "object": "model",
-    "name": "Hermes Pro 7B Q4",
+    "name": "Hermes Pro Llama 3 8B Q4",
     "version": "1.1",
-    "description": "Hermes Pro is superior in Roleplaying, Reasoning and Explaining problem.",
+    "description": "Hermes Pro is well-designed for General chat and JSON output.",
     "format": "gguf",
     "settings": {
-      "ctx_len": 4096,
+      "ctx_len": 8192,
       "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-      "llama_model_path": "Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf"
+      "llama_model_path": "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,
       "top_p": 0.95,
       "stream": true,
-      "max_tokens": 4096,
+      "max_tokens": 8192,
       "stop": [],
       "frequency_penalty": 0,
       "presence_penalty": 0
     },
     "metadata": {
       "author": "NousResearch",
-      "tags": ["7B", "Finetuned"],
-      "size": 4370000000
+      "tags": [
+        "7B",
+        "Finetuned"
+      ],
+      "size": 4920000000
     },
     "engine": "nitro"
   }
-  
diff --git a/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json
index 056fb9050..b8da24e71 100644
--- a/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 2048,
       "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-      "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf"
+      "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf",
+      "ngl": 22
     },
     "parameters": {
       "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json b/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json
deleted file mode 100644
index 23e110d0e..000000000
--- a/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-    "sources": [
-      {
-        "filename": "miqu-1-70b.q4_k_m.gguf",
-        "url": "https://huggingface.co/miqudev/miqu-1-70b/resolve/main/miqu-1-70b.q4_k_m.gguf"
-      }
-    ],
-    "id": "miqu-70b",
-    "object": "model",
-    "name": "Mistral 70B Q4",
-    "version": "1.0",
-    "description": "A leak weight of Mistral 70B model.",
-    "format": "gguf",
-    "settings": {
-      "ctx_len": 4096,
-      "prompt_template": "[INST] {prompt} [/INST]",
-      "llama_model_path": "miqu-1-70b.q4_k_m.gguf"
-    },
-    "parameters": {
-      "temperature": 0.7,
-      "top_p": 0.95,
-      "stream": true,
-      "max_tokens": 4096,
-      "frequency_penalty": 0,
-      "presence_penalty": 0
-    },
-    "metadata": {
-      "author": "miqudev",
-      "tags": ["70B", "Foundational Model"],
-      "size": 26440000000
-    },
-    "engine": "nitro"
-  }
-  
\ No newline at end of file
diff --git a/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
index 73a908e17..c372aa329 100644
--- a/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
+++ b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "[INST] {prompt} [/INST]",
-    "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+    "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
index 0935a25a3..4413b415c 100644
--- a/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
+++ b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "[INST] {prompt} [/INST]",
-    "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"
+    "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf",
+    "ngl": 100
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
index ab0884936..aa39b62c2 100644
--- a/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf"
+    "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
index b37a470d3..94967962d 100644
--- a/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 8192,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
-    "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf"
+    "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json b/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json
deleted file mode 100644
index dbbc9e0ec..000000000
--- a/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-  "sources": [
-    {
-      "filename": "openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf",
-      "url": "https://huggingface.co/janhq/openhermes-2.5-neural-chat-v3-3-slerp-GGUF/resolve/main/openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf"
-    }
-  ],
-  "id": "openhermes-neural-7b",
-  "object": "model",
-  "name": "OpenHermes Neural 7B Q4",
-  "version": "1.1",
-  "description": "OpenHermes Neural is a merged model using the TIES method. It performs well in various benchmarks.",
-  "format": "gguf",
-  "settings": {
-    "ctx_len": 4096,
-    "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf"
-  },
-  "parameters": {
-    "temperature": 0.7,
-    "top_p": 0.95,
-    "stream": true,
-    "max_tokens": 4096,
-    "frequency_penalty": 0,
-    "presence_penalty": 0
-  },
-  "metadata": {
-    "author": "Intel, Jan",
-    "tags": ["7B", "Merged"],
-    "size": 4370000000,
-    "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/openhermes-neural-7b/cover.png"
-  },
-  "engine": "nitro"
-}
diff --git a/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json b/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
deleted file mode 100644
index 0d789385b..000000000
--- a/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-    "sources": [
-      {
-        "url": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf",
-        "filename": "Phi-3-mini-4k-instruct-q4.gguf"
-      }
-    ],
-    "id": "phi3-3.8b",
-    "object": "model",
-    "name": "Phi-3 Mini",
-    "version": "1.0",
-    "description": "Phi-3 Mini is Microsoft's newest, compact model designed for mobile use.",
-    "format": "gguf",
-    "settings": {
-      "ctx_len": 4096,
-      "prompt_template": "<|system|>\n{system_message}<|end|>\n<|user|>\n{prompt}<|end|>\n<|assistant|>\n",
-      "llama_model_path": "Phi-3-mini-4k-instruct-q4.gguf"
-    },
-    "parameters": {
-      "max_tokens": 4096,
-      "stop": ["<|end|>"]
-    },
-    "metadata": {
-      "author": "Microsoft",
-      "tags": [
-        "3B",
-        "Finetuned"
-      ],
-      "size": 2320000000
-    },
-    "engine": "nitro"
-  }
diff --git a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
index f79f4e2d0..f96fb4a49 100644
--- a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### System Prompt\n{system_message}\n### User Message\n{prompt}\n### Assistant",
-    "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf"
+    "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf",
+    "ngl": 48
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
index 6c0f642e3..202221bd7 100644
--- a/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf"
+    "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json b/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json
index 1e789bf07..81bf4306c 100644
--- a/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 4096,
       "prompt_template": "<|user|>\n{prompt}<|endoftext|>\n<|assistant|>",
-      "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf"
+      "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
index 574b4d893..2848931bb 100644
--- a/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "stealth-v1.3.Q4_K_M.gguf"
+    "llama_model_path": "stealth-v1.3.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json
index 6a9187fa5..443ee7dcd 100644
--- a/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>",
-    "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+    "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+    "ngl": 22
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json
index 9c988e19a..1a98ddb2e 100644
--- a/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 32768,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "trinity-v1.2.Q4_K_M.gguf"
+    "llama_model_path": "trinity-v1.2.Q4_K_M.gguf",
+    "ngl": 32
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json b/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json
index d435fb784..978f8cf54 100644
--- a/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json
@@ -14,7 +14,8 @@
     "settings": {
       "ctx_len": 32768,
       "prompt_template": "[INST] <<SYS>>\n{system_message}\n<</SYS>>\n{prompt} [/INST]",
-      "llama_model_path": "vistral-7b-chat-dpo.Q4_K_M.gguf"
+      "llama_model_path": "vistral-7b-chat-dpo.Q4_K_M.gguf",
+      "ngl": 32
     },
     "parameters": {
       "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json b/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json
index b7e84bf88..5e77faa14 100644
--- a/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 16384,
     "prompt_template": "### Instruction:\n{prompt}\n### Response:",
-    "llama_model_path": "wizardcoder-python-13b-v1.0.Q4_K_M.gguf"
+    "llama_model_path": "wizardcoder-python-13b-v1.0.Q4_K_M.gguf",
+    "ngl": 40
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/resources/models/yi-34b/model.json b/extensions/inference-nitro-extension/resources/models/yi-34b/model.json
index 4bc9b0ba1..637eec453 100644
--- a/extensions/inference-nitro-extension/resources/models/yi-34b/model.json
+++ b/extensions/inference-nitro-extension/resources/models/yi-34b/model.json
@@ -14,7 +14,8 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
-    "llama_model_path": "yi-34b-chat.Q4_K_M.gguf"
+    "llama_model_path": "yi-34b-chat.Q4_K_M.gguf",
+    "ngl": 60
   },
   "parameters": {
     "temperature": 0.7,
diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts
index 497bb6466..f1d3eb32f 100644
--- a/extensions/inference-nitro-extension/rollup.config.ts
+++ b/extensions/inference-nitro-extension/rollup.config.ts
@@ -12,21 +12,17 @@ const codeninja7bJson = require('./resources/models/codeninja-1.0-7b/model.json'
 const commandr34bJson = require('./resources/models/command-r-34b/model.json')
 const deepseekCoder13bJson = require('./resources/models/deepseek-coder-1.3b/model.json')
 const deepseekCoder34bJson = require('./resources/models/deepseek-coder-34b/model.json')
-const dolphinPhi2Json = require('./resources/models/dolphin-phi-2/model.json')
 const gemma2bJson = require('./resources/models/gemma-2b/model.json')
 const gemma7bJson = require('./resources/models/gemma-7b/model.json')
-const hermesPro7bJson = require('./resources/models/hermes-pro-7b/model.json')
 const llama2Chat70bJson = require('./resources/models/llama2-chat-70b/model.json')
 const llama2Chat7bJson = require('./resources/models/llama2-chat-7b/model.json')
 const llamacorn1bJson = require('./resources/models/llamacorn-1.1b/model.json')
 const llava13bJson = require('./resources/models/llava-13b/model.json')
 const llava7bJson = require('./resources/models/llava-7b/model.json')
-const miqu70bJson = require('./resources/models/miqu-70b/model.json')
 const mistralIns7bq4Json = require('./resources/models/mistral-ins-7b-q4/model.json')
 const mixtral8x7bInstructJson = require('./resources/models/mixtral-8x7b-instruct/model.json')
 const noromaid7bJson = require('./resources/models/noromaid-7b/model.json')
 const openchat357bJson = require('./resources/models/openchat-3.5-7b/model.json')
-const openhermesNeural7bJson = require('./resources/models/openhermes-neural-7b/model.json')
 const phind34bJson = require('./resources/models/phind-34b/model.json')
 const qwen7bJson = require('./resources/models/qwen-7b/model.json')
 const stableZephyr3bJson = require('./resources/models/stable-zephyr-3b/model.json')
@@ -37,6 +33,7 @@ const vistral7bJson = require('./resources/models/vistral-7b/model.json')
 const wizardcoder13bJson = require('./resources/models/wizardcoder-13b/model.json')
 const yi34bJson = require('./resources/models/yi-34b/model.json')
 const llama3Json = require('./resources/models/llama3-8b-instruct/model.json')
+const llama3Hermes8bJson = require('./resources/models/llama3-hermes-8b/model.json')
 
 export default [
   {
@@ -56,21 +53,17 @@ export default [
           commandr34bJson,
           deepseekCoder13bJson,
           deepseekCoder34bJson,
-          dolphinPhi2Json,
           gemma2bJson,
           gemma7bJson,
-          hermesPro7bJson,
           llama2Chat70bJson,
           llama2Chat7bJson,
           llamacorn1bJson,
           llava13bJson,
           llava7bJson,
-          miqu70bJson,
           mistralIns7bq4Json,
           mixtral8x7bInstructJson,
           noromaid7bJson,
           openchat357bJson,
-          openhermesNeural7bJson,
           phind34bJson,
           qwen7bJson,
           stableZephyr3bJson,
@@ -80,7 +73,8 @@ export default [
           vistral7bJson,
           wizardcoder13bJson,
           yi34bJson,
-          llama3Json
+          llama3Json,
+          llama3Hermes8bJson
         ]),
         NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
         DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson),

From 08d15e5dc7aa93f0172775e658c02bffaa62a66a Mon Sep 17 00:00:00 2001
From: NamH <NamNh0122@gmail.com>
Date: Mon, 13 May 2024 16:47:27 +0700
Subject: [PATCH 18/26] fix: deeplink when app not open on linux (#2893)

Co-authored-by: James <james@jan.ai>
---
 electron/managers/window.ts                   |  2 +-
 web/containers/LoadingModal/index.tsx         |  6 +-
 web/containers/Providers/DeepLinkListener.tsx | 99 ++++++++++++-------
 3 files changed, 67 insertions(+), 40 deletions(-)

diff --git a/electron/managers/window.ts b/electron/managers/window.ts
index 6052f332a..ab76bb94b 100644
--- a/electron/managers/window.ts
+++ b/electron/managers/window.ts
@@ -29,7 +29,7 @@ class WindowManager {
       },
     })
 
-    if (process.platform === 'win32') {
+    if (process.platform === 'win32' || process.platform === 'linux') {
       /// This is work around for windows deeplink.
       /// second-instance event is not fired when app is not open, so the app
       /// does not received the deeplink.
diff --git a/web/containers/LoadingModal/index.tsx b/web/containers/LoadingModal/index.tsx
index cfaf05c7e..0159134f4 100644
--- a/web/containers/LoadingModal/index.tsx
+++ b/web/containers/LoadingModal/index.tsx
@@ -6,12 +6,10 @@ export type LoadingInfo = {
   message: string
 }
 
-export const loadingModalVisibilityAtom = atom<LoadingInfo | undefined>(
-  undefined
-)
+export const loadingModalInfoAtom = atom<LoadingInfo | undefined>(undefined)
 
 const ResettingModal: React.FC = () => {
-  const loadingInfo = useAtomValue(loadingModalVisibilityAtom)
+  const loadingInfo = useAtomValue(loadingModalInfoAtom)
 
   return (
     <Modal open={loadingInfo != null}>
diff --git a/web/containers/Providers/DeepLinkListener.tsx b/web/containers/Providers/DeepLinkListener.tsx
index ca275e52c..d5941204f 100644
--- a/web/containers/Providers/DeepLinkListener.tsx
+++ b/web/containers/Providers/DeepLinkListener.tsx
@@ -6,7 +6,7 @@ import { useDebouncedCallback } from 'use-debounce'
 
 import { useGetHFRepoData } from '@/hooks/useGetHFRepoData'
 
-import { loadingModalVisibilityAtom as loadingModalInfoAtom } from '../LoadingModal'
+import { loadingModalInfoAtom } from '../LoadingModal'
 import { toaster } from '../Toast'
 
 import {
@@ -27,46 +27,75 @@ const DeepLinkListener: React.FC<Props> = ({ children }) => {
     importHuggingFaceModelStageAtom
   )
 
-  const debounced = useDebouncedCallback(async (searchText) => {
-    if (searchText.indexOf('/') === -1) {
-      toaster({
-        title: 'Failed to get Hugging Face models',
-        description: 'Invalid Hugging Face model URL',
-        type: 'error',
-      })
-      return
-    }
-
-    try {
-      setLoadingInfo({
-        title: 'Getting Hugging Face models',
-        message: 'Please wait..',
-      })
-      const data = await getHfRepoData(searchText)
-      setImportingHuggingFaceRepoData(data)
-      setImportHuggingFaceModelStage('REPO_DETAIL')
-      setLoadingInfo(undefined)
-    } catch (err) {
-      setLoadingInfo(undefined)
-      let errMessage = 'Unexpected Error'
-      if (err instanceof Error) {
-        errMessage = err.message
+  const handleDeepLinkAction = useDebouncedCallback(
+    async (deepLinkAction: DeepLinkAction) => {
+      if (
+        deepLinkAction.action !== 'models' ||
+        deepLinkAction.provider !== 'huggingface'
+      ) {
+        console.error(
+          `Invalid deeplink action (${deepLinkAction.action}) or provider (${deepLinkAction.provider})`
+        )
+        return
       }
-      toaster({
-        title: 'Failed to get Hugging Face models',
-        description: errMessage,
-        type: 'error',
-      })
-      console.error(err)
-    }
-  }, 300)
+
+      try {
+        setLoadingInfo({
+          title: 'Getting Hugging Face models',
+          message: 'Please wait..',
+        })
+        const data = await getHfRepoData(deepLinkAction.resource)
+        setImportingHuggingFaceRepoData(data)
+        setImportHuggingFaceModelStage('REPO_DETAIL')
+        setLoadingInfo(undefined)
+      } catch (err) {
+        setLoadingInfo(undefined)
+        toaster({
+          title: 'Failed to get Hugging Face models',
+          description: err instanceof Error ? err.message : 'Unexpected Error',
+          type: 'error',
+        })
+        console.error(err)
+      }
+    },
+    300
+  )
+
   window.electronAPI?.onDeepLink((_event: string, input: string) => {
     window.core?.api?.ackDeepLink()
-    const url = input.replaceAll('jan://', '')
-    debounced(url)
+
+    const action = deeplinkParser(input)
+    if (!action) return
+    handleDeepLinkAction(action)
   })
 
   return <Fragment>{children}</Fragment>
 }
 
+type DeepLinkAction = {
+  action: string
+  provider: string
+  resource: string
+}
+
+const deeplinkParser = (
+  deepLink: string | undefined
+): DeepLinkAction | undefined => {
+  if (!deepLink) return undefined
+
+  try {
+    const url = new URL(deepLink)
+    const params = url.pathname.split('/').filter((str) => str.length > 0)
+
+    if (params.length < 3) return undefined
+    const action = params[0]
+    const provider = params[1]
+    const resource = params.slice(2).join('/')
+    return { action, provider, resource }
+  } catch (err) {
+    console.error(err)
+    return undefined
+  }
+}
+
 export default DeepLinkListener

From eb7e96393b642e5c1c26d9a49e29294c998e9406 Mon Sep 17 00:00:00 2001
From: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
Date: Tue, 14 May 2024 14:16:12 +0700
Subject: [PATCH 19/26] add: gpt4o (#2899)

---
 .../inference-openai-extension/package.json   |  2 +-
 .../resources/models.json                     | 30 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/extensions/inference-openai-extension/package.json b/extensions/inference-openai-extension/package.json
index 713989e75..cd776257c 100644
--- a/extensions/inference-openai-extension/package.json
+++ b/extensions/inference-openai-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-openai-extension",
   "productName": "OpenAI Inference Engine",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "This extension enables OpenAI chat completion API calls",
   "main": "dist/index.js",
   "module": "dist/module.js",
diff --git a/extensions/inference-openai-extension/resources/models.json b/extensions/inference-openai-extension/resources/models.json
index eb2fc662c..6852a1892 100644
--- a/extensions/inference-openai-extension/resources/models.json
+++ b/extensions/inference-openai-extension/resources/models.json
@@ -89,5 +89,35 @@
       ]
     },
     "engine": "openai"
+  },
+  {
+    "sources": [
+      {
+        "url": "https://openai.com"
+      }
+    ],
+    "id": "gpt-4o",
+    "object": "model",
+    "name": "OpenAI GPT 4o",
+    "version": "1.1",
+    "description": "OpenAI GPT 4o is a new flagship model with fast speed and high quality",
+    "format": "api",
+    "settings": {},
+    "parameters": {
+      "max_tokens": 4096,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
+    },
+    "metadata": {
+      "author": "OpenAI",
+      "tags": [
+        "General"
+      ]
+    },
+    "engine": "openai"
   }
 ]

From aa1f01f4faa3f061a3f2a48a40e0e68a173709d5 Mon Sep 17 00:00:00 2001
From: Van Pham <64197333+Van-QA@users.noreply.github.com>
Date: Wed, 15 May 2024 15:03:29 +0700
Subject: [PATCH 20/26] Revert "chore: remove nutjs" and replace nutjs version
 (#2900)

* Revert "chore: remove nutjs (#2860)"

This reverts commit ce2d8e540582a73423efed02dbe484e213cdf0fe.

* feat: change nut-js to download from GitHub

* feat: change nut-js to use @nut-tree-fork/nut-js

* Revert "Chore: temporary disable cache for server migration (#2742)"

This reverts commit af9a35110ba2462279dfb988cc89748697a97899.

* feat: disable quickask settings on linux

* feat: enable quick ask linux again

* Change to @kirillvakalov/nut-tree__nut-js (#2906)

Co-authored-by: Hien To <tominhhien97@gmail.com>

---------

Co-authored-by: Van-QA <van@jan.ai>
Co-authored-by: hiento09 <136591877+hiento09@users.noreply.github.com>
Co-authored-by: Hien To <tominhhien97@gmail.com>
---
 .../jan-electron-linter-and-test.yml          | 112 +++++++++---------
 electron/package.json                         |   3 +-
 electron/utils/selectedText.ts                |  27 ++---
 3 files changed, 71 insertions(+), 71 deletions(-)

diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml
index f84971be9..828162c57 100644
--- a/.github/workflows/jan-electron-linter-and-test.yml
+++ b/.github/workflows/jan-electron-linter-and-test.yml
@@ -57,19 +57,19 @@ jobs:
           rm -rf ~/jan
           make clean
 
-      # - name: Get Commit Message for PR
-      #   if : github.event_name == 'pull_request'
-      #   run: |
-      #     echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}})" >> $GITHUB_ENV
+      - name: Get Commit Message for PR
+        if : github.event_name == 'pull_request'
+        run: |
+          echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}})" >> $GITHUB_ENV
 
-      # - name: Get Commit Message for push event
-      #   if : github.event_name == 'push'
-      #   run: |
-      #     echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}})" >> $GITHUB_ENV
+      - name: Get Commit Message for push event
+        if : github.event_name == 'push'
+        run: |
+          echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}})" >> $GITHUB_ENV
 
-      # - name: "Config report portal"
-      #   run: |
-      #     make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App macos" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
+      - name: "Config report portal"
+        run: |
+          make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App macos" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
 
       - name: Linter and test
         run: |
@@ -78,9 +78,9 @@ jobs:
           make test
         env:
           CSC_IDENTITY_AUTO_DISCOVERY: "false"
-          # TURBO_API: "${{ secrets.TURBO_API }}"
-          # TURBO_TEAM: "macos"
-          # TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
+          TURBO_API: "${{ secrets.TURBO_API }}"
+          TURBO_TEAM: "macos"
+          TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
 
   test-on-macos-pr-target:
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
@@ -141,16 +141,16 @@ jobs:
           }
           make clean
   
-      # - name: Get Commit Message for push event
-      #   if : github.event_name == 'push'
-      #   shell: bash
-      #   run: |
-      #     echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV
+      - name: Get Commit Message for push event
+        if : github.event_name == 'push'
+        shell: bash
+        run: |
+          echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV
 
-      # - name: "Config report portal"
-      #   shell: bash
-      #   run: |
-      #     make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows ${{ matrix.antivirus-tools }}" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
+      - name: "Config report portal"
+        shell: bash
+        run: |
+          make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows ${{ matrix.antivirus-tools }}" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
 
       - name: Linter and test
         shell: powershell
@@ -158,10 +158,10 @@ jobs:
           npm config set registry ${{ secrets.NPM_PROXY }} --global
           yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: "${{ secrets.TURBO_API }}"
-        #   TURBO_TEAM: "windows"
-        #   TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
+        env:
+          TURBO_API: "${{ secrets.TURBO_API }}"
+          TURBO_TEAM: "windows"
+          TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
   test-on-windows-pr:
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: windows-desktop-default-windows-security
@@ -189,16 +189,16 @@ jobs:
           }
           make clean
 
-      # - name: Get Commit Message for PR
-      #   if : github.event_name == 'pull_request'
-      #   shell: bash
-      #   run: |
-      #     echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV
+      - name: Get Commit Message for PR
+        if : github.event_name == 'pull_request'
+        shell: bash
+        run: |
+          echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV
 
-      # - name: "Config report portal"
-      #   shell: bash
-      #   run: |
-      #     make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
+      - name: "Config report portal"
+        shell: bash
+        run: |
+          make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
 
       - name: Linter and test
         shell: powershell
@@ -206,10 +206,10 @@ jobs:
           npm config set registry ${{ secrets.NPM_PROXY }} --global
           yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: "${{ secrets.TURBO_API }}"
-        #   TURBO_TEAM: "windows"
-        #   TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
+        env:
+          TURBO_API: "${{ secrets.TURBO_API }}"
+          TURBO_TEAM: "windows"
+          TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
 
   test-on-windows-pr-target:
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
@@ -266,20 +266,20 @@ jobs:
           rm -rf ~/jan
           make clean
 
-      # - name: Get Commit Message for PR
-      #   if : github.event_name == 'pull_request'
-      #   run: |
-      #     echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV
+      - name: Get Commit Message for PR
+        if : github.event_name == 'pull_request'
+        run: |
+          echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV
 
-      # - name: Get Commit Message for push event
-      #   if : github.event_name == 'push'
-      #   run: |
-      #     echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV
+      - name: Get Commit Message for push event
+        if : github.event_name == 'push'
+        run: |
+          echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV
 
-      # - name: "Config report portal"
-      #   shell: bash
-      #   run: |
-      #     make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Linux" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
+      - name: "Config report portal"
+        shell: bash
+        run: |
+          make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Linux" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
 
       - name: Linter and test
         run: |
@@ -288,10 +288,10 @@ jobs:
           npm config set registry ${{ secrets.NPM_PROXY }} --global
           yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: "${{ secrets.TURBO_API }}"
-        #   TURBO_TEAM: "linux"
-        #   TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
+        env:
+          TURBO_API: "${{ secrets.TURBO_API }}"
+          TURBO_TEAM: "linux"
+          TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}"
 
   test-on-ubuntu-pr-target:
     runs-on: [self-hosted, Linux, ubuntu-desktop]
diff --git a/electron/package.json b/electron/package.json
index fc81fcb19..48b7eaee2 100644
--- a/electron/package.json
+++ b/electron/package.json
@@ -103,7 +103,8 @@
     "pacote": "^17.0.4",
     "request": "^2.88.2",
     "request-progress": "^3.0.0",
-    "ulidx": "^2.3.0"
+    "ulidx": "^2.3.0",
+    "@kirillvakalov/nut-tree__nut-js": "4.2.1-2"
   },
   "devDependencies": {
     "@electron/notarize": "^2.1.0",
diff --git a/electron/utils/selectedText.ts b/electron/utils/selectedText.ts
index f76146d13..51b2eb762 100644
--- a/electron/utils/selectedText.ts
+++ b/electron/utils/selectedText.ts
@@ -1,24 +1,23 @@
 import { clipboard, globalShortcut } from 'electron'
+import { keyboard, Key } from "@kirillvakalov/nut-tree__nut-js"
 
 /**
  * Gets selected text by synthesizing the keyboard shortcut
  * "CommandOrControl+c" then reading text from the clipboard
  */
 export const getSelectedText = async () => {
-  // TODO: Implement this function
-  // const currentClipboardContent = clipboard.readText() // preserve clipboard content
-  // clipboard.clear()
-  // const hotkeys: Key[] = [
-  //   process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl,
-  //   Key.C,
-  // ]
-  // await keyboard.pressKey(...hotkeys)
-  // await keyboard.releaseKey(...hotkeys)
-  // await new Promise((resolve) => setTimeout(resolve, 200)) // add a delay before checking clipboard
-  // const selectedText = clipboard.readText()
-  // clipboard.writeText(currentClipboardContent)
-  // return selectedText
-  return ''
+  const currentClipboardContent = clipboard.readText() // preserve clipboard content
+  clipboard.clear()
+  const hotkeys: Key[] = [
+    process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl,
+    Key.C,
+  ]
+  await keyboard.pressKey(...hotkeys)
+  await keyboard.releaseKey(...hotkeys)
+  await new Promise((resolve) => setTimeout(resolve, 200)) // add a delay before checking clipboard
+  const selectedText = clipboard.readText()
+  clipboard.writeText(currentClipboardContent)
+  return selectedText
 }
 
 /**

From 1130979008de90e56a60fec24a3661542b7ce473 Mon Sep 17 00:00:00 2001
From: Louis <louis@jan.ai>
Date: Wed, 15 May 2024 17:27:37 +0700
Subject: [PATCH 21/26] fix: cohere stream param does not work (#2907)

---
 .../browser/extensions/engines/helpers/sse.ts | 19 +++++++++++-------
 .../inference-cohere-extension/src/index.ts   | 20 +++++++++++++------
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/core/src/browser/extensions/engines/helpers/sse.ts b/core/src/browser/extensions/engines/helpers/sse.ts
index 7ae68142f..024ced470 100644
--- a/core/src/browser/extensions/engines/helpers/sse.ts
+++ b/core/src/browser/extensions/engines/helpers/sse.ts
@@ -68,14 +68,19 @@ export function requestInference(
             let cachedLines = ''
             for (const line of lines) {
               try {
-                const toParse = cachedLines + line
-                if (!line.includes('data: [DONE]')) {
-                  const data = JSON.parse(toParse.replace('data: ', ''))
-                  content += data.choices[0]?.delta?.content ?? ''
-                  if (content.startsWith('assistant: ')) {
-                    content = content.replace('assistant: ', '')
+                if (transformResponse) {
+                  content += transformResponse(line)
+                  subscriber.next(content ?? '')
+                } else {
+                  const toParse = cachedLines + line
+                  if (!line.includes('data: [DONE]')) {
+                    const data = JSON.parse(toParse.replace('data: ', ''))
+                    content += data.choices[0]?.delta?.content ?? ''
+                    if (content.startsWith('assistant: ')) {
+                      content = content.replace('assistant: ', '')
+                    }
+                    if (content !== '') subscriber.next(content)
                   }
-                  if (content !== '') subscriber.next(content)
                 }
               } catch {
                 cachedLines = line
diff --git a/extensions/inference-cohere-extension/src/index.ts b/extensions/inference-cohere-extension/src/index.ts
index 24cc5935b..dd7f03317 100644
--- a/extensions/inference-cohere-extension/src/index.ts
+++ b/extensions/inference-cohere-extension/src/index.ts
@@ -26,8 +26,8 @@ enum RoleType {
 
 type CoherePayloadType = {
   chat_history?: Array<{ role: RoleType; message: string }>
-  message?: string,
-  preamble?: string,
+  message?: string
+  preamble?: string
 }
 
 /**
@@ -82,18 +82,24 @@ export default class JanInferenceCohereExtension extends RemoteOAIEngine {
     if (payload.messages.length === 0) {
       return {}
     }
+
+    const { messages, ...params } = payload
     const convertedData: CoherePayloadType = {
+      ...params,
       chat_history: [],
       message: '',
     }
-    payload.messages.forEach((item, index) => {
+    messages.forEach((item, index) => {
       // Assign the message of the last item to the `message` property
-      if (index === payload.messages.length - 1) {
+      if (index === messages.length - 1) {
         convertedData.message = item.content as string
         return
       }
       if (item.role === ChatCompletionRole.User) {
-        convertedData.chat_history.push({ role: RoleType.user, message: item.content as string })
+        convertedData.chat_history.push({
+          role: RoleType.user,
+          message: item.content as string,
+        })
       } else if (item.role === ChatCompletionRole.Assistant) {
         convertedData.chat_history.push({
           role: RoleType.chatbot,
@@ -106,5 +112,7 @@ export default class JanInferenceCohereExtension extends RemoteOAIEngine {
     return convertedData
   }
 
-  transformResponse = (data: any) => data.text
+  transformResponse = (data: any) => {
+    return typeof data === 'object' ? data.text : JSON.parse(data).text ?? ''
+  }
 }

From 33697be6642b42bf3b1b8161d629ee182c40cfe3 Mon Sep 17 00:00:00 2001
From: hiento09 <136591877+hiento09@users.noreply.github.com>
Date: Thu, 16 May 2024 10:05:49 +0700
Subject: [PATCH 22/26] Change mac arm64 build use github runner (#2910)

Co-authored-by: Hien To <tominhhien97@gmail.com>
---
 .../workflows/template-build-macos-arm64.yml  | 44 ++++++++-----------
 .../workflows/template-build-macos-x64.yml    |  3 +-
 2 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/template-build-macos-arm64.yml b/.github/workflows/template-build-macos-arm64.yml
index 2ef40b7c0..a5bc1e539 100644
--- a/.github/workflows/template-build-macos-arm64.yml
+++ b/.github/workflows/template-build-macos-arm64.yml
@@ -41,7 +41,7 @@ on:
 
 jobs:
   build-macos:
-    runs-on: macos-silicon
+    runs-on: macos-latest
     environment: production
     permissions:
       contents: write
@@ -55,15 +55,9 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 20
-      - name: Unblock keychain
-        run: |
-          security unlock-keychain -p ${{ secrets.KEYCHAIN_PASSWORD }} ~/Library/Keychains/login.keychain-db
-      # - uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.11'
 
-      # - name: Install jq
-      #   uses: dcarbone/install-jq-action@v2.0.1
+      - name: Install jq
+        uses: dcarbone/install-jq-action@v2.0.1
 
       - name: Update app version based on latest release tag with build number
         if: inputs.public_provider != 'github'
@@ -101,17 +95,17 @@ jobs:
         env:
           VERSION_TAG: ${{ inputs.new_version }}
 
-      # - name: Get Cer for code signing
-      #   run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
-      #   shell: bash
-      #   env:
-      #     CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+      - name: Get Cer for code signing
+        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
+        shell: bash
+        env:
+          CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
 
-      # - uses: apple-actions/import-codesign-certs@v2
-      #   continue-on-error: true
-      #   with:
-      #     p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
-      #     p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+      - uses: apple-actions/import-codesign-certs@v2
+        continue-on-error: true
+        with:
+          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
 
       - name: Build and publish app to cloudflare r2 or github artifactory
         if: inputs.public_provider != 'github'
@@ -125,9 +119,9 @@ jobs:
           fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # CSC_LINK: "/tmp/codesign.p12"
-          # CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
-          # CSC_IDENTITY_AUTO_DISCOVERY: "true"
+          CSC_LINK: "/tmp/codesign.p12"
+          CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+          CSC_IDENTITY_AUTO_DISCOVERY: "true"
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
           APP_PATH: "."
@@ -143,9 +137,9 @@ jobs:
           make build-and-publish
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # CSC_LINK: "/tmp/codesign.p12"
-          # CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
-          # CSC_IDENTITY_AUTO_DISCOVERY: "true"
+          CSC_LINK: "/tmp/codesign.p12"
+          CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+          CSC_IDENTITY_AUTO_DISCOVERY: "true"
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
           APP_PATH: "."
diff --git a/.github/workflows/template-build-macos-x64.yml b/.github/workflows/template-build-macos-x64.yml
index 85d4a9b3e..d9543194d 100644
--- a/.github/workflows/template-build-macos-x64.yml
+++ b/.github/workflows/template-build-macos-x64.yml
@@ -158,5 +158,4 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: latest-mac-x64
-          path: ./electron/dist/latest-mac.yml
-
+          path: ./electron/dist/latest-mac.yml
\ No newline at end of file

From 06be30876fcd5fb8c186b137ef9760e440146273 Mon Sep 17 00:00:00 2001
From: hiento09 <136591877+hiento09@users.noreply.github.com>
Date: Thu, 16 May 2024 13:27:31 +0700
Subject: [PATCH 23/26] Revert "Change mac arm64 build use github runner
 (#2910)" (#2911)

This reverts commit 33697be6642b42bf3b1b8161d629ee182c40cfe3.
---
 .../workflows/template-build-macos-arm64.yml  | 44 +++++++++++--------
 .../workflows/template-build-macos-x64.yml    |  3 +-
 2 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/template-build-macos-arm64.yml b/.github/workflows/template-build-macos-arm64.yml
index a5bc1e539..2ef40b7c0 100644
--- a/.github/workflows/template-build-macos-arm64.yml
+++ b/.github/workflows/template-build-macos-arm64.yml
@@ -41,7 +41,7 @@ on:
 
 jobs:
   build-macos:
-    runs-on: macos-latest
+    runs-on: macos-silicon
     environment: production
     permissions:
       contents: write
@@ -55,9 +55,15 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 20
+      - name: Unblock keychain
+        run: |
+          security unlock-keychain -p ${{ secrets.KEYCHAIN_PASSWORD }} ~/Library/Keychains/login.keychain-db
+      # - uses: actions/setup-python@v5
+      #   with:
+      #     python-version: '3.11'
 
-      - name: Install jq
-        uses: dcarbone/install-jq-action@v2.0.1
+      # - name: Install jq
+      #   uses: dcarbone/install-jq-action@v2.0.1
 
       - name: Update app version based on latest release tag with build number
         if: inputs.public_provider != 'github'
@@ -95,17 +101,17 @@ jobs:
         env:
           VERSION_TAG: ${{ inputs.new_version }}
 
-      - name: Get Cer for code signing
-        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
-        shell: bash
-        env:
-          CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+      # - name: Get Cer for code signing
+      #   run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
+      #   shell: bash
+      #   env:
+      #     CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
 
-      - uses: apple-actions/import-codesign-certs@v2
-        continue-on-error: true
-        with:
-          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
-          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+      # - uses: apple-actions/import-codesign-certs@v2
+      #   continue-on-error: true
+      #   with:
+      #     p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+      #     p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
 
       - name: Build and publish app to cloudflare r2 or github artifactory
         if: inputs.public_provider != 'github'
@@ -119,9 +125,9 @@ jobs:
           fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CSC_LINK: "/tmp/codesign.p12"
-          CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
-          CSC_IDENTITY_AUTO_DISCOVERY: "true"
+          # CSC_LINK: "/tmp/codesign.p12"
+          # CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+          # CSC_IDENTITY_AUTO_DISCOVERY: "true"
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
           APP_PATH: "."
@@ -137,9 +143,9 @@ jobs:
           make build-and-publish
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CSC_LINK: "/tmp/codesign.p12"
-          CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
-          CSC_IDENTITY_AUTO_DISCOVERY: "true"
+          # CSC_LINK: "/tmp/codesign.p12"
+          # CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+          # CSC_IDENTITY_AUTO_DISCOVERY: "true"
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
           APP_PATH: "."
diff --git a/.github/workflows/template-build-macos-x64.yml b/.github/workflows/template-build-macos-x64.yml
index d9543194d..85d4a9b3e 100644
--- a/.github/workflows/template-build-macos-x64.yml
+++ b/.github/workflows/template-build-macos-x64.yml
@@ -158,4 +158,5 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: latest-mac-x64
-          path: ./electron/dist/latest-mac.yml
\ No newline at end of file
+          path: ./electron/dist/latest-mac.yml
+

From 0436224fa1cf4c9db2b7eb917536ff013522ca11 Mon Sep 17 00:00:00 2001
From: hiento09 <136591877+hiento09@users.noreply.github.com>
Date: Thu, 16 May 2024 14:23:54 +0700
Subject: [PATCH 24/26] Revert "Revert "Change mac arm64 build use github
 runner (#2910)" (#2911)" (#2915)

This reverts commit 06be30876fcd5fb8c186b137ef9760e440146273.
---
 .../workflows/template-build-macos-arm64.yml  | 44 ++++++++-----------
 .../workflows/template-build-macos-x64.yml    |  3 +-
 2 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/template-build-macos-arm64.yml b/.github/workflows/template-build-macos-arm64.yml
index 2ef40b7c0..a5bc1e539 100644
--- a/.github/workflows/template-build-macos-arm64.yml
+++ b/.github/workflows/template-build-macos-arm64.yml
@@ -41,7 +41,7 @@ on:
 
 jobs:
   build-macos:
-    runs-on: macos-silicon
+    runs-on: macos-latest
     environment: production
     permissions:
       contents: write
@@ -55,15 +55,9 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 20
-      - name: Unblock keychain
-        run: |
-          security unlock-keychain -p ${{ secrets.KEYCHAIN_PASSWORD }} ~/Library/Keychains/login.keychain-db
-      # - uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.11'
 
-      # - name: Install jq
-      #   uses: dcarbone/install-jq-action@v2.0.1
+      - name: Install jq
+        uses: dcarbone/install-jq-action@v2.0.1
 
       - name: Update app version based on latest release tag with build number
         if: inputs.public_provider != 'github'
@@ -101,17 +95,17 @@ jobs:
         env:
           VERSION_TAG: ${{ inputs.new_version }}
 
-      # - name: Get Cer for code signing
-      #   run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
-      #   shell: bash
-      #   env:
-      #     CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+      - name: Get Cer for code signing
+        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
+        shell: bash
+        env:
+          CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
 
-      # - uses: apple-actions/import-codesign-certs@v2
-      #   continue-on-error: true
-      #   with:
-      #     p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
-      #     p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+      - uses: apple-actions/import-codesign-certs@v2
+        continue-on-error: true
+        with:
+          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
 
       - name: Build and publish app to cloudflare r2 or github artifactory
         if: inputs.public_provider != 'github'
@@ -125,9 +119,9 @@ jobs:
           fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # CSC_LINK: "/tmp/codesign.p12"
-          # CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
-          # CSC_IDENTITY_AUTO_DISCOVERY: "true"
+          CSC_LINK: "/tmp/codesign.p12"
+          CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+          CSC_IDENTITY_AUTO_DISCOVERY: "true"
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
           APP_PATH: "."
@@ -143,9 +137,9 @@ jobs:
           make build-and-publish
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # CSC_LINK: "/tmp/codesign.p12"
-          # CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
-          # CSC_IDENTITY_AUTO_DISCOVERY: "true"
+          CSC_LINK: "/tmp/codesign.p12"
+          CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+          CSC_IDENTITY_AUTO_DISCOVERY: "true"
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
           APP_PATH: "."
diff --git a/.github/workflows/template-build-macos-x64.yml b/.github/workflows/template-build-macos-x64.yml
index 85d4a9b3e..d9543194d 100644
--- a/.github/workflows/template-build-macos-x64.yml
+++ b/.github/workflows/template-build-macos-x64.yml
@@ -158,5 +158,4 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: latest-mac-x64
-          path: ./electron/dist/latest-mac.yml
-
+          path: ./electron/dist/latest-mac.yml
\ No newline at end of file

From 218259945f98308dea85f8f941d4dd92bc7432e4 Mon Sep 17 00:00:00 2001
From: Hoang Ha <64120343+hahuyhoang411@users.noreply.github.com>
Date: Thu, 16 May 2024 14:58:21 +0700
Subject: [PATCH 25/26] Chore: Add phi3 (#2914)

* init

* version bump

* fix: correct template
---
 .../inference-nitro-extension/package.json    |  2 +-
 .../resources/models/phi3-3.8b/model.json     | 32 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json

diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
index 37a75c01f..7d31a1d38 100644
--- a/extensions/inference-nitro-extension/package.json
+++ b/extensions/inference-nitro-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-nitro-extension",
   "productName": "Nitro Inference Engine",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",
diff --git a/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json b/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
new file mode 100644
index 000000000..6777cb6b6
--- /dev/null
+++ b/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
@@ -0,0 +1,32 @@
+{
+    "sources": [
+      {
+        "url": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf",
+        "filename": "Phi-3-mini-4k-instruct-q4.gguf"
+      }
+    ],
+    "id": "phi3-3.8b",
+    "object": "model",
+    "name": "Phi-3 Mini",
+    "version": "1.0",
+    "description": "Phi-3 Mini is Microsoft's newest, compact model designed for mobile use.",
+    "format": "gguf",
+    "settings": {
+      "ctx_len": 4096,
+      "prompt_template": "<|user|>\n{prompt}<|end|>\n<|assistant|>\n",
+      "llama_model_path": "Phi-3-mini-4k-instruct-q4.gguf"
+    },
+    "parameters": {
+      "max_tokens": 4096,
+      "stop": ["<|end|>"]
+    },
+    "metadata": {
+      "author": "Microsoft",
+      "tags": [
+        "3B",
+        "Finetuned"
+      ],
+      "size": 2320000000
+    },
+    "engine": "nitro"
+  }
\ No newline at end of file

From 537ef20a548bdf054585ec0adfafa9135da9654c Mon Sep 17 00:00:00 2001
From: Louis <louis@jan.ai>
Date: Thu, 16 May 2024 17:46:49 +0700
Subject: [PATCH 26/26] chore: replace nitro by cortex-cpp (#2912)

---
 core/src/node/api/restful/helper/consts.ts    |  6 +-
 .../node/api/restful/helper/startStopModel.ts | 26 ++++----
 core/src/node/helper/resource.ts              |  2 +-
 .../assistant-extension/src/node/index.ts     |  3 +-
 .../assistant-extension/src/node/retrieval.ts |  6 +-
 .../src/tools/retrieval.ts                    |  1 +
 .../inference-nitro-extension/.gitignore      |  2 +
 .../inference-nitro-extension/bin/version.txt |  2 +-
 .../inference-nitro-extension/download.bat    |  4 +-
 .../inference-nitro-extension/package.json    | 10 +--
 .../rollup.config.ts                          |  2 +-
 .../inference-nitro-extension/src/index.ts    |  4 +-
 .../src/node/execute.test.ts                  | 27 +++++---
 .../src/node/execute.ts                       |  6 +-
 .../src/node/index.ts                         | 62 ++++++++++---------
 .../tensorrt-llm-extension/src/node/index.ts  |  4 +-
 16 files changed, 96 insertions(+), 71 deletions(-)
 create mode 100644 extensions/inference-nitro-extension/.gitignore

diff --git a/core/src/node/api/restful/helper/consts.ts b/core/src/node/api/restful/helper/consts.ts
index bc3cfe300..8d8f8e341 100644
--- a/core/src/node/api/restful/helper/consts.ts
+++ b/core/src/node/api/restful/helper/consts.ts
@@ -9,11 +9,11 @@ export const SUPPORTED_MODEL_FORMAT = '.gguf'
 // The URL for the Nitro subprocess
 const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${NITRO_DEFAULT_PORT}`
 // The URL for the Nitro subprocess to load a model
-export const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`
+export const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/loadmodel`
 // The URL for the Nitro subprocess to validate a model
-export const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`
+export const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/modelstatus`
 
 // The URL for the Nitro subprocess to kill itself
 export const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`
 
-export const DEFAULT_CHAT_COMPLETION_URL = `http://${LOCAL_HOST}:${NITRO_DEFAULT_PORT}/inferences/llamacpp/chat_completion` // default nitro url
+export const DEFAULT_CHAT_COMPLETION_URL = `http://${LOCAL_HOST}:${NITRO_DEFAULT_PORT}/inferences/server/chat_completion` // default nitro url
diff --git a/core/src/node/api/restful/helper/startStopModel.ts b/core/src/node/api/restful/helper/startStopModel.ts
index bcd182cb5..8665850da 100644
--- a/core/src/node/api/restful/helper/startStopModel.ts
+++ b/core/src/node/api/restful/helper/startStopModel.ts
@@ -144,12 +144,12 @@ const runNitroAndLoadModel = async (modelId: string, modelSettings: NitroModelSe
 }
 
 const spawnNitroProcess = async (): Promise<void> => {
-  log(`[SERVER]::Debug: Spawning Nitro subprocess...`)
+  log(`[SERVER]::Debug: Spawning cortex subprocess...`)
 
   let binaryFolder = join(
     getJanExtensionsPath(),
     '@janhq',
-    'inference-nitro-extension',
+    'inference-cortex-extension',
     'dist',
     'bin'
   )
@@ -160,7 +160,7 @@ const spawnNitroProcess = async (): Promise<void> => {
   const args: string[] = ['1', LOCAL_HOST, NITRO_DEFAULT_PORT.toString()]
   // Execute the binary
   log(
-    `[SERVER]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`
+    `[SERVER]::Debug: Spawn cortex at path: ${executableOptions.executablePath}, and args: ${args}`
   )
   subprocess = spawn(
     executableOptions.executablePath,
@@ -184,12 +184,12 @@ const spawnNitroProcess = async (): Promise<void> => {
   })
 
   subprocess.on('close', (code: any) => {
-    log(`[SERVER]::Debug: Nitro exited with code: ${code}`)
+    log(`[SERVER]::Debug: cortex exited with code: ${code}`)
     subprocess = undefined
   })
 
   tcpPortUsed.waitUntilUsed(NITRO_DEFAULT_PORT, 300, 30000).then(() => {
-    log(`[SERVER]::Debug: Nitro is ready`)
+    log(`[SERVER]::Debug: cortex is ready`)
   })
 }
 
@@ -203,13 +203,13 @@ const executableNitroFile = (): NitroExecutableOptions => {
   let binaryFolder = join(
     getJanExtensionsPath(),
     '@janhq',
-    'inference-nitro-extension',
+    'inference-cortex-extension',
     'dist',
     'bin'
   )
 
   let cudaVisibleDevices = ''
-  let binaryName = 'nitro'
+  let binaryName = 'cortex-cpp'
   /**
    * The binary folder is different for each platform.
    */
@@ -228,12 +228,16 @@ const executableNitroFile = (): NitroExecutableOptions => {
       }
       cudaVisibleDevices = nvidiaInfo['gpu_highest_vram']
     }
-    binaryName = 'nitro.exe'
+    binaryName = 'cortex-cpp.exe'
   } else if (process.platform === 'darwin') {
     /**
      *  For MacOS: mac-universal both Silicon and InteL
      */
-    binaryFolder = join(binaryFolder, 'mac-universal')
+    if(process.arch === 'arm64') {
+    binaryFolder = join(binaryFolder, 'mac-arm64')
+    } else {
+      binaryFolder = join(binaryFolder, 'mac-amd64')
+    }
   } else {
     /**
      *  For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0
@@ -300,7 +304,7 @@ const loadLLMModel = async (settings: NitroModelSettings): Promise<Response> =>
     retryDelay: 500,
   })
     .then((res: any) => {
-      log(`[SERVER]::Debug: Load model success with response ${JSON.stringify(res)}`)
+      log(`[SERVER]::Debug: Load model request with response ${JSON.stringify(res)}`)
       return Promise.resolve(res)
     })
     .catch((err: any) => {
@@ -327,7 +331,7 @@ export const stopModel = async (_modelId: string) => {
       })
     }, 5000)
     const tcpPortUsed = require('tcp-port-used')
-    log(`[SERVER]::Debug: Request to kill Nitro`)
+    log(`[SERVER]::Debug: Request to kill cortex`)
 
     fetch(NITRO_HTTP_KILL_URL, {
       method: 'DELETE',
diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts
index 6c4a71478..c7bfbf20c 100644
--- a/core/src/node/helper/resource.ts
+++ b/core/src/node/helper/resource.ts
@@ -4,7 +4,7 @@ import { log } from './logger'
 
 export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
   const cpu = await physicalCpuCount()
-  log(`[NITRO]::CPU information - ${cpu}`)
+  log(`[CORTEX]::CPU information - ${cpu}`)
 
   return {
     numCpuPhysicalCore: cpu,
diff --git a/extensions/assistant-extension/src/node/index.ts b/extensions/assistant-extension/src/node/index.ts
index f303dd51d..46835614d 100644
--- a/extensions/assistant-extension/src/node/index.ts
+++ b/extensions/assistant-extension/src/node/index.ts
@@ -10,11 +10,12 @@ export function toolRetrievalUpdateTextSplitter(
 }
 export async function toolRetrievalIngestNewDocument(
   file: string,
+  model: string,
   engine: string
 ) {
   const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file))
   const threadPath = path.dirname(filePath.replace('files', ''))
-  retrieval.updateEmbeddingEngine(engine)
+  retrieval.updateEmbeddingEngine(model, engine)
   return retrieval
     .ingestAgentKnowledge(filePath, `${threadPath}/memory`)
     .catch((err) => {
diff --git a/extensions/assistant-extension/src/node/retrieval.ts b/extensions/assistant-extension/src/node/retrieval.ts
index e89357d5c..52193f221 100644
--- a/extensions/assistant-extension/src/node/retrieval.ts
+++ b/extensions/assistant-extension/src/node/retrieval.ts
@@ -28,14 +28,14 @@ export class Retrieval {
     })
   }
 
-  public updateEmbeddingEngine(engine: string): void {
+  public updateEmbeddingEngine(model: string, engine: string): void {
     // Engine settings are not compatible with the current embedding model params
     // Switch case manually for now
     if (engine === 'nitro') {
       this.embeddingModel = new OpenAIEmbeddings(
-        { openAIApiKey: 'nitro-embedding' },
+        { openAIApiKey: 'nitro-embedding', model },
         // TODO: Raw settings
-        { basePath: 'http://127.0.0.1:3928/v1' }
+        { basePath: 'http://127.0.0.1:3928/v1' },
       )
     } else {
       // Fallback to OpenAI Settings
diff --git a/extensions/assistant-extension/src/tools/retrieval.ts b/extensions/assistant-extension/src/tools/retrieval.ts
index e58305c60..a1a641941 100644
--- a/extensions/assistant-extension/src/tools/retrieval.ts
+++ b/extensions/assistant-extension/src/tools/retrieval.ts
@@ -36,6 +36,7 @@ export class RetrievalTool extends InferenceTool {
           NODE,
           'toolRetrievalIngestNewDocument',
           docFile,
+          data.model?.id,
           data.model?.engine
         )
       } else {
diff --git a/extensions/inference-nitro-extension/.gitignore b/extensions/inference-nitro-extension/.gitignore
new file mode 100644
index 000000000..10780f1d4
--- /dev/null
+++ b/extensions/inference-nitro-extension/.gitignore
@@ -0,0 +1,2 @@
+bin
+!version.txt
\ No newline at end of file
diff --git a/extensions/inference-nitro-extension/bin/version.txt b/extensions/inference-nitro-extension/bin/version.txt
index 0c4b45492..6f2743d65 100644
--- a/extensions/inference-nitro-extension/bin/version.txt
+++ b/extensions/inference-nitro-extension/bin/version.txt
@@ -1 +1 @@
-0.3.22
+0.4.4
diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat
index c99162eba..9bd2d4b07 100644
--- a/extensions/inference-nitro-extension/download.bat
+++ b/extensions/inference-nitro-extension/download.bat
@@ -1,3 +1,3 @@
 @echo off
-set /p NITRO_VERSION=<./bin/version.txt
-.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-avx2-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-avx2-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-avx2.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan
+set /p CORTEX_VERSION=<./bin/version.txt
+.\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64-avx2-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64-avx2-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64-avx2.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan
diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
index 7d31a1d38..d396778d9 100644
--- a/extensions/inference-nitro-extension/package.json
+++ b/extensions/inference-nitro-extension/package.json
@@ -1,8 +1,8 @@
 {
-  "name": "@janhq/inference-nitro-extension",
-  "productName": "Nitro Inference Engine",
+  "name": "@janhq/inference-cortex-extension",
+  "productName": "Cortex Inference Engine",
   "version": "1.0.7",
-  "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
+  "description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",
   "author": "Jan <service@jan.ai>",
@@ -10,8 +10,8 @@
   "scripts": {
     "test": "jest",
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
-    "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-avx2.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro",
-    "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-universal.tar.gz -o ./bin/ && mkdir -p ./bin/mac-universal && tar -zxvf ./bin/nitro-${NITRO_VERSION}-mac-universal.tar.gz --strip-components=1 -C ./bin/mac-universal && rm -rf ./bin/nitro-${NITRO_VERSION}-mac-universal.tar.gz && chmod +x ./bin/mac-universal/nitro",
+    "downloadnitro:linux": "CORTEX_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/cortex/releases/download/v${CORTEX_VERSION}/cortex-cpp-${CORTEX_VERSION}-linux-amd64-avx2.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/cortex-cpp && download https://github.com/janhq/cortex/releases/download/v${CORTEX_VERSION}/cortex-cpp-${CORTEX_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/cortex-cpp && download https://github.com/janhq/cortex/releases/download/v${CORTEX_VERSION}/cortex-cpp-${CORTEX_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/cortex-cpp && download https://github.com/janhq/cortex/releases/download/v${CORTEX_VERSION}/cortex-cpp-${CORTEX_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/cortex-cpp",
+    "downloadnitro:darwin": "CORTEX_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/cortex/releases/download/v${CORTEX_VERSION}/cortex-cpp-${CORTEX_VERSION}-mac-arm64.tar.gz -o ./bin/ && mkdir -p ./bin/mac-arm64 && tar -zxvf ./bin/cortex-cpp-${CORTEX_VERSION}-mac-arm64.tar.gz --strip-components=1 -C ./bin/mac-arm64 && rm -rf ./bin/cortex-cpp-${CORTEX_VERSION}-mac-arm64.tar.gz && chmod +x ./bin/mac-arm64/cortex-cpp && download https://github.com/janhq/cortex/releases/download/v${CORTEX_VERSION}/cortex-cpp-${CORTEX_VERSION}-mac-amd64.tar.gz -o ./bin/ && mkdir -p ./bin/mac-amd64 && tar -zxvf ./bin/cortex-cpp-${CORTEX_VERSION}-mac-amd64.tar.gz --strip-components=1 -C ./bin/mac-amd64 && rm -rf ./bin/cortex-cpp-${CORTEX_VERSION}-mac-amd64.tar.gz && chmod +x ./bin/mac-amd64/cortex-cpp",
     "downloadnitro:win32": "download.bat",
     "downloadnitro": "run-script-os",
     "build:publish:darwin": "rimraf *.tgz --glob && yarn build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",
diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts
index f1d3eb32f..b0707f404 100644
--- a/extensions/inference-nitro-extension/rollup.config.ts
+++ b/extensions/inference-nitro-extension/rollup.config.ts
@@ -80,7 +80,7 @@ export default [
         DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson),
         INFERENCE_URL: JSON.stringify(
           process.env.INFERENCE_URL ||
-            'http://127.0.0.1:3928/inferences/llamacpp/chat_completion'
+            'http://127.0.0.1:3928/inferences/server/chat_completion'
         ),
         TROUBLESHOOTING_URL: JSON.stringify(
           'https://jan.ai/guides/troubleshooting'
diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts
index e6bad64f4..a027e8844 100644
--- a/extensions/inference-nitro-extension/src/index.ts
+++ b/extensions/inference-nitro-extension/src/index.ts
@@ -130,7 +130,7 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine {
     const executableFolderPath = await joinPath([
       janDataFolderPath,
       'engines',
-      this.name ?? 'nitro',
+      this.name ?? 'cortex-cpp',
       this.version ?? '1.0.0',
     ])
 
@@ -179,7 +179,7 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine {
       const executableFolderPath = await joinPath([
         janDataFolderPath,
         'engines',
-        this.name ?? 'nitro',
+        this.name ?? 'cortex-cpp',
         this.version ?? '1.0.0',
       ])
 
diff --git a/extensions/inference-nitro-extension/src/node/execute.test.ts b/extensions/inference-nitro-extension/src/node/execute.test.ts
index dfd26deb8..cf9e84acf 100644
--- a/extensions/inference-nitro-extension/src/node/execute.test.ts
+++ b/extensions/inference-nitro-extension/src/node/execute.test.ts
@@ -33,9 +33,22 @@ describe('test executable nitro file', () => {
     Object.defineProperty(process, 'platform', {
       value: 'darwin',
     })
+    Object.defineProperty(process, 'arch', {
+      value: 'arm64',
+    })
     expect(executableNitroFile(testSettings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`mac-universal${sep}nitro`),
+        executablePath: expect.stringContaining(`mac-arm64${sep}cortex-cpp`),
+        cudaVisibleDevices: '',
+        vkVisibleDevices: '',
+      })
+    )
+    Object.defineProperty(process, 'arch', {
+      value: 'amd64',
+    })
+    expect(executableNitroFile(testSettings)).toEqual(
+      expect.objectContaining({
+        executablePath: expect.stringContaining(`mac-amd64${sep}cortex-cpp`),
         cudaVisibleDevices: '',
         vkVisibleDevices: '',
       })
@@ -56,7 +69,7 @@ describe('test executable nitro file', () => {
     }
     expect(executableNitroFile(settings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`win-cpu${sep}nitro.exe`),
+        executablePath: expect.stringContaining(`win-cpu${sep}cortex-cpp.exe`),
         cudaVisibleDevices: '',
         vkVisibleDevices: '',
       })
@@ -89,7 +102,7 @@ describe('test executable nitro file', () => {
     }
     expect(executableNitroFile(settings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`win-cuda-11-7${sep}nitro.exe`),
+        executablePath: expect.stringContaining(`win-cuda-11-7${sep}cortex-cpp.exe`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
@@ -122,7 +135,7 @@ describe('test executable nitro file', () => {
     }
     expect(executableNitroFile(settings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`win-cuda-12-0${sep}nitro.exe`),
+        executablePath: expect.stringContaining(`win-cuda-12-0${sep}cortex-cpp.exe`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
@@ -139,7 +152,7 @@ describe('test executable nitro file', () => {
     }
     expect(executableNitroFile(settings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`linux-cpu${sep}nitro`),
+        executablePath: expect.stringContaining(`linux-cpu${sep}cortex-cpp`),
         cudaVisibleDevices: '',
         vkVisibleDevices: '',
       })
@@ -172,7 +185,7 @@ describe('test executable nitro file', () => {
     }
     expect(executableNitroFile(settings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`linux-cuda-11-7${sep}nitro`),
+        executablePath: expect.stringContaining(`linux-cuda-11-7${sep}cortex-cpp`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
@@ -205,7 +218,7 @@ describe('test executable nitro file', () => {
     }
     expect(executableNitroFile(settings)).toEqual(
       expect.objectContaining({
-        executablePath: expect.stringContaining(`linux-cuda-12-0${sep}nitro`),
+        executablePath: expect.stringContaining(`linux-cuda-12-0${sep}cortex-cpp`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts
index 2cfcfe4f3..417734afa 100644
--- a/extensions/inference-nitro-extension/src/node/execute.ts
+++ b/extensions/inference-nitro-extension/src/node/execute.ts
@@ -1,4 +1,4 @@
-import { GpuSetting, SystemInformation } from '@janhq/core'
+import { GpuSetting } from '@janhq/core'
 import * as path from 'path'
 
 export interface NitroExecutableOptions {
@@ -24,7 +24,7 @@ const os = (): string => {
   return process.platform === 'win32'
     ? 'win'
     : process.platform === 'darwin'
-      ? 'mac-universal'
+      ? process.arch === 'arm64' ? 'mac-arm64' : 'mac-amd64'
       : 'linux'
 }
 
@@ -52,7 +52,7 @@ export const executableNitroFile = (
     .join('-')
   let cudaVisibleDevices = gpuSetting?.gpus_in_use.join(',') ?? ''
   let vkVisibleDevices = gpuSetting?.gpus_in_use.join(',') ?? ''
-  let binaryName = `nitro${extension()}`
+  let binaryName = `cortex-cpp${extension()}`
 
   return {
     executablePath: path.join(__dirname, '..', 'bin', binaryFolder, binaryName),
diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts
index fbfdb8761..03e172c63 100644
--- a/extensions/inference-nitro-extension/src/node/index.ts
+++ b/extensions/inference-nitro-extension/src/node/index.ts
@@ -34,9 +34,9 @@ const LOCAL_HOST = '127.0.0.1'
 // The URL for the Nitro subprocess
 const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`
 // The URL for the Nitro subprocess to load a model
-const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`
+const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/loadmodel`
 // The URL for the Nitro subprocess to validate a model
-const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`
+const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/modelstatus`
 // The URL for the Nitro subprocess to kill itself
 const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`
 
@@ -50,7 +50,7 @@ const SUPPORTED_MODEL_FORMAT = '.gguf'
 let subprocess: ChildProcessWithoutNullStreams | undefined = undefined
 
 // The current model settings
-let currentSettings: ModelSettingParams | undefined = undefined
+let currentSettings: ModelSettingParams & { model?: string } | undefined = undefined
 
 /**
  * Stops a Nitro subprocess.
@@ -77,7 +77,7 @@ async function loadModel(
   }
 
   if (params.model.engine !== InferenceEngine.nitro) {
-    return Promise.reject('Not a nitro model')
+    return Promise.reject('Not a cortex model')
   } else {
     const nitroResourceProbe = await getSystemResourceInfo()
     // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt
@@ -135,6 +135,7 @@ async function loadModel(
       // model.settings can override the default settings
       ...params.model.settings,
       llama_model_path,
+      model: params.model.id,
       // This is critical and requires real CPU physical core count (or performance core)
       ...(params.model.settings.mmproj && {
         mmproj: path.isAbsolute(params.model.settings.mmproj)
@@ -142,7 +143,7 @@ async function loadModel(
           : path.join(modelFolder, params.model.settings.mmproj),
       }),
     }
-    return runNitroAndLoadModel(systemInfo)
+    return runNitroAndLoadModel(params.model.id, systemInfo)
   }
 }
 
@@ -152,7 +153,7 @@ async function loadModel(
  * 3. Validate model status
  * @returns
  */
-async function runNitroAndLoadModel(systemInfo?: SystemInformation) {
+async function runNitroAndLoadModel(modelId: string, systemInfo?: SystemInformation) {
   // Gather system information for CPU physical cores and memory
   return killSubprocess()
     .then(() =>
@@ -160,10 +161,10 @@ async function runNitroAndLoadModel(systemInfo?: SystemInformation) {
     )
     .then(() => spawnNitroProcess(systemInfo))
     .then(() => loadLLMModel(currentSettings))
-    .then(validateModelStatus)
+    .then(() => validateModelStatus(modelId))
     .catch((err) => {
       // TODO: Broadcast error so app could display proper error message
-      log(`[NITRO]::Error: ${err}`)
+      log(`[CORTEX]::Error: ${err}`)
       return { error: err }
     })
 }
@@ -222,7 +223,7 @@ function loadLLMModel(settings: any): Promise<Response> {
   if (!settings?.ngl) {
     settings.ngl = 100
   }
-  log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`)
+  log(`[CORTEX]::Debug: Loading model with params ${JSON.stringify(settings)}`)
   return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, {
     method: 'POST',
     headers: {
@@ -234,14 +235,14 @@ function loadLLMModel(settings: any): Promise<Response> {
   })
     .then((res) => {
       log(
-        `[NITRO]::Debug: Load model success with response ${JSON.stringify(
+        `[CORTEX]::Debug: Load model success with response ${JSON.stringify(
           res
         )}`
       )
       return Promise.resolve(res)
     })
     .catch((err) => {
-      log(`[NITRO]::Error: Load model failed with error ${err}`)
+      log(`[CORTEX]::Error: Load model failed with error ${err}`)
       return Promise.reject(err)
     })
 }
@@ -252,11 +253,12 @@ function loadLLMModel(settings: any): Promise<Response> {
  * If the model is loaded successfully, the object is empty.
  * If the model is not loaded successfully, the object contains an error message.
  */
-async function validateModelStatus(): Promise<void> {
+async function validateModelStatus(modelId: string): Promise<void> {
   // Send a GET request to the validation URL.
   // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries.
   return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, {
-    method: 'GET',
+    method: 'POST',
+    body: JSON.stringify({ model: modelId }),
     headers: {
       'Content-Type': 'application/json',
     },
@@ -264,7 +266,7 @@ async function validateModelStatus(): Promise<void> {
     retryDelay: 300,
   }).then(async (res: Response) => {
     log(
-      `[NITRO]::Debug: Validate model state with response ${JSON.stringify(
+      `[CORTEX]::Debug: Validate model state with response ${JSON.stringify(
         res.status
       )}`
     )
@@ -275,7 +277,7 @@ async function validateModelStatus(): Promise<void> {
       // Otherwise, return an object with an error message.
       if (body.model_loaded) {
         log(
-          `[NITRO]::Debug: Validate model state success with response ${JSON.stringify(
+          `[CORTEX]::Debug: Validate model state success with response ${JSON.stringify(
             body
           )}`
         )
@@ -283,7 +285,7 @@ async function validateModelStatus(): Promise<void> {
       }
     }
     log(
-      `[NITRO]::Debug: Validate model state failed with response ${JSON.stringify(
+      `[CORTEX]::Debug: Validate model state failed with response ${JSON.stringify(
         res.statusText
       )}`
     )
@@ -298,7 +300,7 @@ async function validateModelStatus(): Promise<void> {
 async function killSubprocess(): Promise<void> {
   const controller = new AbortController()
   setTimeout(() => controller.abort(), 5000)
-  log(`[NITRO]::Debug: Request to kill Nitro`)
+  log(`[CORTEX]::Debug: Request to kill cortex`)
 
   const killRequest = () => {
     return fetch(NITRO_HTTP_KILL_URL, {
@@ -309,17 +311,17 @@ async function killSubprocess(): Promise<void> {
       .then(() =>
         tcpPortUsed.waitUntilFree(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 5000)
       )
-      .then(() => log(`[NITRO]::Debug: Nitro process is terminated`))
+      .then(() => log(`[CORTEX]::Debug: cortex process is terminated`))
       .catch((err) => {
         log(
-          `[NITRO]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}`
+          `[CORTEX]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}`
         )
         throw 'PORT_NOT_AVAILABLE'
       })
   }
 
   if (subprocess?.pid) {
-    log(`[NITRO]::Debug: Killing PID ${subprocess.pid}`)
+    log(`[CORTEX]::Debug: Killing PID ${subprocess.pid}`)
     const pid = subprocess.pid
     return new Promise((resolve, reject) => {
       terminate(pid, function (err) {
@@ -329,7 +331,7 @@ async function killSubprocess(): Promise<void> {
           tcpPortUsed
             .waitUntilFree(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 5000)
             .then(() => resolve())
-            .then(() => log(`[NITRO]::Debug: Nitro process is terminated`))
+            .then(() => log(`[CORTEX]::Debug: cortex process is terminated`))
             .catch(() => {
               killRequest().then(resolve).catch(reject)
             })
@@ -346,22 +348,24 @@ async function killSubprocess(): Promise<void> {
  * @returns A promise that resolves when the Nitro subprocess is started.
  */
 function spawnNitroProcess(systemInfo?: SystemInformation): Promise<any> {
-  log(`[NITRO]::Debug: Spawning Nitro subprocess...`)
+  log(`[CORTEX]::Debug: Spawning cortex subprocess...`)
 
   return new Promise<void>(async (resolve, reject) => {
-    let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default
     let executableOptions = executableNitroFile(systemInfo?.gpuSetting)
 
     const args: string[] = ['1', LOCAL_HOST, PORT.toString()]
     // Execute the binary
     log(
-      `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`
+      `[CORTEX]::Debug: Spawn cortex at path: ${executableOptions.executablePath}, and args: ${args}`
+    )
+    log(
+      path.parse(executableOptions.executablePath).dir
     )
     subprocess = spawn(
       executableOptions.executablePath,
       ['1', LOCAL_HOST, PORT.toString()],
       {
-        cwd: binaryFolder,
+        cwd: path.join(path.parse(executableOptions.executablePath).dir),
         env: {
           ...process.env,
           CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices,
@@ -375,15 +379,15 @@ function spawnNitroProcess(systemInfo?: SystemInformation): Promise<any> {
 
     // Handle subprocess output
     subprocess.stdout.on('data', (data: any) => {
-      log(`[NITRO]::Debug: ${data}`)
+      log(`[CORTEX]::Debug: ${data}`)
     })
 
     subprocess.stderr.on('data', (data: any) => {
-      log(`[NITRO]::Error: ${data}`)
+      log(`[CORTEX]::Error: ${data}`)
     })
 
     subprocess.on('close', (code: any) => {
-      log(`[NITRO]::Debug: Nitro exited with code: ${code}`)
+      log(`[CORTEX]::Debug: cortex exited with code: ${code}`)
       subprocess = undefined
       reject(`child process exited with code ${code}`)
     })
@@ -391,7 +395,7 @@ function spawnNitroProcess(systemInfo?: SystemInformation): Promise<any> {
     tcpPortUsed
       .waitUntilUsed(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 30000)
       .then(() => {
-        log(`[NITRO]::Debug: Nitro is ready`)
+        log(`[CORTEX]::Debug: cortex is ready`)
         resolve()
       })
   })
diff --git a/extensions/tensorrt-llm-extension/src/node/index.ts b/extensions/tensorrt-llm-extension/src/node/index.ts
index eb92c98af..c8bc48459 100644
--- a/extensions/tensorrt-llm-extension/src/node/index.ts
+++ b/extensions/tensorrt-llm-extension/src/node/index.ts
@@ -97,7 +97,7 @@ function unloadModel(): Promise<void> {
   }
 
   if (subprocess?.pid) {
-    log(`[NITRO]::Debug: Killing PID ${subprocess.pid}`)
+    log(`[CORTEX]::Debug: Killing PID ${subprocess.pid}`)
     const pid = subprocess.pid
     return new Promise((resolve, reject) => {
       terminate(pid, function (err) {
@@ -107,7 +107,7 @@ function unloadModel(): Promise<void> {
           return tcpPortUsed
             .waitUntilFree(parseInt(ENGINE_PORT), PORT_CHECK_INTERVAL, 5000)
             .then(() => resolve())
-            .then(() => log(`[NITRO]::Debug: Nitro process is terminated`))
+            .then(() => log(`[CORTEX]::Debug: cortex process is terminated`))
             .catch(() => {
               killRequest()
             })