feat: support multiple model binaries (#1659)

* feat: Support multiple model binaries

* fix: Update downloadModel with multiple binaries handler

* feat: Add 3 models with multiple binaries

* chore: fix model download

* fix: model file lookup & model path

* chore: add .prettierrc

* chore: refactor docs

* chore: bump model version

* fix(capybara): add filename

* fix(codeninja): add file name + llama model path

* fix(default): add llama model path

* fix(deepseek coder): add filename

* fix(deepseek 33B): add filename

* fix(dolphin mixtral): add filename

* fix(llama2-chat): add filename

* fix(llama2-70B): add filename

* fix(mistral 7b): add filename + model path

* fix(bakllava): correct size model

* fix(llava-7b): correct size model

* fix(llava-13b): correct size model

* fix(mixtral-8x7b): add file name + modelpath

* fix(noramaid-7b): add file name + modelpath

* fix(openchat-7b): add file name + modelpath

* fix(openhermes-7b): add file name + modelpath

* fix(phi2-3b): add file name + modelpath

* fix(phind): add file name + modelpath

* fix(solarslerp): add file name + modelpath

* fix(starling): add file name + modelpath

* fix(stealth): add file name + modelpath

* fix(tinyllama): add file name + modelpath

* fix(trinity): add file name + modelpath

* fix(tulu): add file name + modelpath

* fix(wizardcoder): add file name + modelpath

* fix(yi): add file name + modelpath

* update from source -> sources

Signed-off-by: James <james@jan.ai>

---------

Signed-off-by: James <james@jan.ai>
Co-authored-by: hiro <vuonghoainam.work@gmail.com>
Co-authored-by: hahuyhoang411 <hahuyhoanghhh41@gmail.com>
Co-authored-by: James <james@jan.ai>

core/.prettierignore

@@ -0,0 +1,5 @@
+.next/
+node_modules/
+dist/
+*.hbs
+*.mdx

core/src/node/api/common/builder.ts

@@ -265,19 +265,22 @@ export const downloadModel = async (
   const modelBinaryPath = join(directoryPath, modelId)
 
   const request = require('request')
-  const rq = request({ url: model.source_url, strictSSL, proxy })
   const progress = require('request-progress')
+
+  for (const source of model.sources) {
+    const rq = request({ url: source, strictSSL, proxy })
     progress(rq, {})
       .on('progress', function (state: any) {
-      console.log('progress', JSON.stringify(state, null, 2))
+        console.debug('progress', JSON.stringify(state, null, 2))
       })
       .on('error', function (err: Error) {
         console.error('error', err)
       })
       .on('end', function () {
-      console.log('end')
+        console.debug('end')
       })
       .pipe(fs.createWriteStream(modelBinaryPath))
+  }
 
   return {
     message: `Starting download ${modelId}`,

core/src/types/model/modelEntity.ts

@@ -21,6 +21,11 @@ export enum InferenceEngine {
   hf_endpoint = 'hf_endpoint',
 }
 
+export type ModelArtifact = {
+  filename: string
+  url: string
+}
+
 /**
  * Model type defines the shape of a model object.
  * @stored
@@ -45,7 +50,7 @@ export type Model = {
   /**
    * The model download source. It can be an external url or a local filepath.
    */
-  source_url: string
+  sources: ModelArtifact[]
 
   /**
    * The model identifier, which can be referenced in the API endpoints.
@@ -107,6 +112,8 @@ export type ModelSettingParams = {
   system_prompt?: string
   ai_prompt?: string
   user_prompt?: string
+  llama_model_path?: string
+  mmproj?: string
 }
 
 /**

docs/docs/docs/03-engineering/models.md

@@ -56,7 +56,6 @@ jan/                               # Jan root folder
 
 - Each `model` folder contains a `model.json` file, which is a representation of a model.
 - `model.json` contains metadata and default parameters used to run a model.
-- The only required field is `source_url`.
 
 ### Example
 
@@ -64,36 +63,43 @@ Here's a standard example `model.json` for a GGUF model.
 
 ```js
 {
-"id": "zephyr-7b",                  // Defaults to foldername
+  "id": "zephyr-7b",        // Defaults to foldername
-"object": "model",                  // Defaults to "model"
+  "object": "model",        // Defaults to "model"
-"source_url": "https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf",
+  "sources": [
-"name": "Zephyr 7B",                // Defaults to foldername
+    {
-"owned_by": "you",                  // Defaults to "you"
+      "filename": "zephyr-7b-beta.Q4_K_M.gguf",
-"version": "1",                     // Defaults to 1
+      "url": "https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf"
-"created": 1231231,                 // Defaults to file creation time
+    }
-"description": null,                // Defaults to null
+  ],
-"state": enum[null, "ready"]
+  "name": "Zephyr 7B",      // Defaults to foldername
-"format": "ggufv3",                 // Defaults to "ggufv3"
+  "owned_by": "you",        // Defaults to "you"
-"engine": "nitro",                // engine_id specified in jan/engine folder
+  "version": "1",           // Defaults to 1
-"engine_parameters": {              // Engine parameters inside model.json can override
+  "created": 1231231,       // Defaults to file creation time
-    "ctx_len": 2048,                // the value inside the base engine.json
+  "description": null,      // Defaults to null
+  "format": "ggufv3",       // Defaults to "ggufv3"
+  "engine": "nitro",        // engine_id specified in jan/engine folder
+  "engine_parameters": {
+    // Engine parameters inside model.json can override
+    "ctx_len": 4096,        // the value inside the base engine.json
     "ngl": 100,
     "embedding": true,
-    "n_parallel": 4,
+    "n_parallel": 4
-},
+  },
-"model_parameters": {                     // Models are called parameters
+  "model_parameters": {
+    // Models are called parameters
     "stream": true,
-    "max_tokens": 2048,
+    "max_tokens": 4096,
     "stop": ["<endofstring>"], // This usually can be left blank, only used with specific need from model author
     "frequency_penalty": 0,
     "presence_penalty": 0,
     "temperature": 0.7,
     "top_p": 0.95
-},
+  },
-"metadata": {},                     // Defaults to {}
+  "metadata": {},           // Defaults to {}
-"assets": [                         // Defaults to current dir
+  "assets": [
-    "file://.../zephyr-7b-q4_k_m.bin",
+    // Defaults to current dir
-]
+    "file://.../zephyr-7b-q4_k_m.bin"
+  ]
 }
 ```
 

docs/docs/guides/04-using-models/02-import-manually.mdx

@@ -31,7 +31,6 @@ In this section, we will show you how to import a GGUF model from [HuggingFace](
 
 ## Manually Importing a Downloaded Model (nightly versions and v0.4.4+)
 
-
 ### 1. Create a Model Folder
 
 Navigate to the `~/jan/models` folder. You can find this folder by going to `App Settings` > `Advanced` > `Open App Directory`.
@@ -190,14 +189,18 @@ Edit `model.json` and include the following configurations:
 - Ensure the filename must be `model.json`.
 - Ensure the `id` property matches the folder name you created.
 - Ensure the GGUF filename should match the `id` property exactly.
-- Ensure the `source_url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab.
+- Ensure the `source.url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab.
 - Ensure you are using the correct `prompt_template`. This is usually provided in the HuggingFace model's description page.
-- Ensure the `state` property is set to `ready`.
 
 ```json title="model.json"
 {
   // highlight-start
-  "source_url": "https://huggingface.co/janhq/trinity-v1-GGUF/resolve/main/trinity-v1.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "trinity-v1.Q4_K_M.gguf",
+      "url": "https://huggingface.co/janhq/trinity-v1-GGUF/resolve/main/trinity-v1.Q4_K_M.gguf"
+    }
+  ],
   "id": "trinity-v1-7b",
   // highlight-end
   "object": "model",
@@ -208,7 +211,8 @@ Edit `model.json` and include the following configurations:
   "settings": {
     "ctx_len": 4096,
     // highlight-next-line
-    "prompt_template": "{system_message}\n### Instruction:\n{prompt}\n### Response:"
+    "prompt_template": "{system_message}\n### Instruction:\n{prompt}\n### Response:",
+    "llama_model_path": "trinity-v1.Q4_K_M.gguf"
   },
   "parameters": {
     "max_tokens": 4096
@@ -218,9 +222,7 @@ Edit `model.json` and include the following configurations:
     "tags": ["7B", "Merged"],
     "size": 4370000000
   },
-  "engine": "nitro",
+  "engine": "nitro"
-  // highlight-next-line
-  "state": "ready"
 }
 ```
 

docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx

@@ -40,7 +40,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `gpt-3.5-turbo-16k`
 
 ```json title="~/jan/models/gpt-3.5-turbo-16k/model.json"
 {
-  "source_url": "https://openai.com",
+  "sources": [
+    {
+      "filename": "openai",
+      "url": "https://openai.com"
+    }
+  ],
   // highlight-next-line
   "id": "gpt-3.5-turbo-16k",
   "object": "model",
@@ -55,8 +60,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `gpt-3.5-turbo-16k`
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]
   },
-  "engine": "openai",
+  "engine": "openai"
-  "state": "ready"
   // highlight-end
 }
 ```
@@ -118,7 +122,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4`
 
 ```json title="~/jan/models/mistral-ins-7b-q4/model.json"
 {
-  "source_url": "https://jan.ai",
+  "sources": [
+    {
+      "filename": "janai",
+      "url": "https://jan.ai"
+    }
+  ],
   // highlight-next-line
   "id": "mistral-ins-7b-q4",
   "object": "model",
@@ -134,8 +143,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4`
     "tags": ["remote", "awesome"]
   },
   // highlight-start
-  "engine": "openai",
+  "engine": "openai"
-  "state": "ready"
   // highlight-end
 }
 ```

docs/docs/guides/07-integrations/02-integrate-openrouter.mdx

@@ -49,7 +49,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `<openrouter-modeln
 
 ```json title="~/jan/models/openrouter-dolphin-mixtral-8x7b/model.json"
 {
-  "source_url": "https://openrouter.ai/",
+  "sources": [
+    {
+      "filename": "openrouter",
+      "url": "https://openrouter.ai/"
+    }
+  ],
   "id": "cognitivecomputations/dolphin-mixtral-8x7b",
   "object": "model",
   "name": "Dolphin 2.6 Mixtral 8x7B",
@@ -63,8 +68,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `<openrouter-modeln
     "tags": ["General", "Big Context Length"]
   },
   // highlight-start
-  "engine": "openai",
+  "engine": "openai"
-  "state": "ready"
   // highlight-end
 }
 ```

docs/docs/guides/07-integrations/03-integrate-azure-openai-service.mdx

@@ -59,7 +59,12 @@ Navigate to the `~/jan/models` folder. Create a folder named `<your-deployment-n
 
 ```json title="~/jan/models/gpt-35-hieu-jan/model.json"
 {
-  "source_url": "https://hieujan.openai.azure.com",
+  "sources": [
+    {
+      "filename": "azure_openai",
+      "url": "https://hieujan.openai.azure.com"
+    }
+  ],
   // highlight-next-line
   "id": "gpt-35-hieu-jan",
   "object": "model",
@@ -75,8 +80,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `<your-deployment-n
     "tags": ["General", "Big Context Length"]
   },
   // highlight-start
-  "engine": "openai",
+  "engine": "openai"
-  "state": "ready"
   // highlight-end
 }
 ```

docs/openapi/specs/models.yaml

@@ -59,18 +59,26 @@ components:
           type: string
           description: State format of the model, distinct from the engine.
           example: ggufv3
-        source_url:
+        source:
-          type: string
+          type: array
+          items:
+            type: object
+            properties:
+              url:
                 format: uri
                 description: URL to the source of the model.
                 example: https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf
+              filename:
+                type: string
+                description: Filename of the model.
+                example: zephyr-7b-beta.Q4_K_M.gguf
         settings:
           type: object
           properties:
             ctx_len:
               type: string
               description: Context length.
-              example: "2048"
+              example: "4096"
             ngl:
               type: string
               description: Number of layers.
@@ -94,7 +102,7 @@ components:
             token_limit:
               type: string
               description: Token limit for the model.
-              example: "2048"
+              example: "4096"
             top_k:
               type: string
               description: Top-k setting for the model.
@@ -117,7 +125,7 @@ components:
             type: string
           description: List of assets related to the model.
       required:
-        - source_url
+        - source
     ModelObject:
       type: object
       properties:
@@ -169,11 +177,19 @@ components:
             - running
             - stopped
           description: The current state of the model.
-        source_url:
+        source:
-          type: string
+          type: array
+          items:
+            type: object
+            properties:
+              url:
                 format: uri
                 description: URL to the source of the model.
                 example: https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q4_K_M.gguf
+              filename:
+                type: string
+                description: Filename of the model.
+                example: zephyr-7b-beta.Q4_K_M.gguf
         engine_parameters:
           type: object
           properties:
@@ -208,8 +224,8 @@ components:
                 model.
               minimum: 128
               maximum: 4096
-              default: 2048
+              default: 4096
-              example: 2048
+              example: 4096
             n_parallel:
               type: integer
               description: Number of parallel operations, relevant when continuous batching is
@@ -241,8 +257,8 @@ components:
               description: Maximum context length the model can handle.
               minimum: 0
               maximum: 4096
-              default: 2048
+              default: 4096
-              example: 2048
+              example: 4096
             ngl:
               type: integer
               description: Number of layers in the neural network.
@@ -276,8 +292,8 @@ components:
                 response.
               minimum: 1
               maximum: 4096
-              default: 2048
+              default: 4096
-              example: 2048
+              example: 4096
             top_k:
               type: integer
               description: Limits the model to consider only the top k most likely next tokens
@@ -318,7 +334,7 @@ components:
         - created
         - owned_by
         - state
-        - source_url
+        - source
         - parameters
         - metadata
     DeleteModelResponse:

extensions/inference-nitro-extension/src/node/index.ts

@@ -119,11 +119,19 @@ async function runModel(
       wrapper.model.settings.ai_prompt = prompt.ai_prompt;
     }
 
+    const modelFolderPath = path.join(janRoot, "models", wrapper.model.id);
+    const modelPath = wrapper.model.settings.llama_model_path
+      ? path.join(modelFolderPath, wrapper.model.settings.llama_model_path)
+      : currentModelFile;
+
     currentSettings = {
-      llama_model_path: currentModelFile,
       ...wrapper.model.settings,
+      llama_model_path: modelPath,
       // This is critical and requires real CPU physical core count (or performance core)
       cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore),
+      ...(wrapper.model.settings.mmproj && {
+        mmproj: path.join(modelFolderPath, wrapper.model.settings.mmproj),
+      }),
     };
     console.log(currentSettings);
     return runNitroAndLoadModel();

extensions/model-extension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@janhq/model-extension",
-  "version": "1.0.22",
+  "version": "1.0.23",
   "description": "Model Management Extension provides model exploration and seamless downloads",
   "main": "dist/index.js",
   "module": "dist/module.js",

extensions/model-extension/src/index.ts

@@ -80,16 +80,34 @@ export default class JanModelExtension extends ModelExtension {
     const modelDirPath = await joinPath([JanModelExtension._homeDir, model.id])
     if (!(await fs.existsSync(modelDirPath))) await fs.mkdirSync(modelDirPath)
 
-    // try to retrieve the download file name from the source url
+    if (model.sources.length > 1) {
-    // if it fails, use the model ID as the file name
+      // path to model binaries
-    const extractedFileName = await model.source_url.split('/').pop()
+      for (const source of model.sources) {
+        let path = this.extractFileName(source.url)
+        if (source.filename) {
+          path = await joinPath([modelDirPath, source.filename])
+        }
+
+        downloadFile(source.url, path, network)
+      }
+    } else {
+      const fileName = this.extractFileName(model.sources[0]?.url)
+      const path = await joinPath([modelDirPath, fileName])
+      downloadFile(model.sources[0]?.url, path, network)
+    }
+  }
+
+  /**
+   *  try to retrieve the download file name from the source url
+   */
+  private extractFileName(url: string): string {
+    const extractedFileName = url.split('/').pop()
     const fileName = extractedFileName
       .toLowerCase()
       .endsWith(JanModelExtension._supportedModelFormat)
       ? extractedFileName
-      : model.id
+      : extractedFileName + JanModelExtension._supportedModelFormat
-    const path = await joinPath([modelDirPath, fileName])
+    return fileName
-    downloadFile(model.source_url, path, network)
   }
 
   /**
@@ -98,6 +116,7 @@ export default class JanModelExtension extends ModelExtension {
    * @returns {Promise<void>} A promise that resolves when the download has been cancelled.
    */
   async cancelModelDownload(modelId: string): Promise<void> {
+    const model = await this.getConfiguredModels()
     return abortDownload(
       await joinPath([JanModelExtension._homeDir, modelId, modelId])
     ).then(async () => {
@@ -163,15 +182,16 @@ export default class JanModelExtension extends ModelExtension {
           .then((files: string[]) => {
             // or model binary exists in the directory
             // model binary name can match model ID or be a .gguf file and not be an incompleted model file
+            // TODO: Check diff between urls, filenames
             return (
               files.includes(modelDir) ||
-              files.some(
+              files.filter(
                 (file) =>
                   file
                     .toLowerCase()
                     .includes(JanModelExtension._supportedModelFormat) &&
                   !file.endsWith(JanModelExtension._incompletedModelFileName)
-              )
+              )?.length >= model.sources.length
             )
           })
       }
@@ -198,7 +218,6 @@ export default class JanModelExtension extends ModelExtension {
 
       const readJsonPromises = allDirectories.map(async (dirName) => {
         // filter out directories that don't match the selector
-
         // read model.json
         const jsonPath = await joinPath([
           JanModelExtension._homeDir,
@@ -226,7 +245,21 @@ export default class JanModelExtension extends ModelExtension {
       const modelData = results.map((result) => {
         if (result.status === 'fulfilled') {
           try {
-            return result.value as Model
+            // This to ensure backward compatibility with `model.json` with `source_url`
+            const tmpModel =
+              typeof result.value === 'object'
+                ? result.value
+                : JSON.parse(result.value)
+            if (tmpModel['source_url'] != null) {
+              tmpModel['source'] = [
+                {
+                  filename: tmpModel.id,
+                  url: tmpModel['source_url'],
+                },
+              ]
+            }
+
+            return tmpModel as Model
           } catch {
             console.debug(`Unable to parse model metadata: ${result.value}`)
             return undefined

models/bakllava-1/model.json

@@ -0,0 +1,33 @@
+{
+  "sources": [
+    {
+      "filename": "ggml-model-q5_k.gguf",
+      "url": "https://huggingface.co/mys/ggml_bakllava-1/resolve/main/ggml-model-q5_k.gguf"
+    },
+    {
+      "filename": "mmproj-model-f16.gguf",
+      "url": "https://huggingface.co/mys/ggml_bakllava-1/resolve/main/mmproj-model-f16.gguf"
+    }
+  ],
+  "id": "bakllava-1",
+  "object": "model",
+  "name": "BakLlava 1",
+  "version": "1.0",
+  "description": "BakLlava 1 can bring vision understanding to Jan",
+  "format": "gguf",
+  "settings": {
+    "ctx_len": 4096,
+    "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
+    "llama_model_path": "ggml-model-q5_k.gguf",
+    "mmproj": "mmproj-model-f16.gguf"
+  },
+  "parameters": {
+    "max_tokens": 4096
+  },
+  "metadata": {
+    "author": "Mys",
+    "tags": ["Vision"],
+    "size": 5750000000
+  },
+  "engine": "nitro"
+}

models/capybara-34b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Nous-Capybara-34B-GGUF/resolve/main/nous-capybara-34b.Q5_K_M.gguf",
+  "sources": [
+    {
+      "filename": "nous-capybara-34b.Q5_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Nous-Capybara-34B-GGUF/resolve/main/nous-capybara-34b.Q5_K_M.gguf"
+    }
+  ],
   "id": "capybara-34b",
   "object": "model",
   "name": "Capybara 200k 34B Q5",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "USER:\n{prompt}\nASSISTANT:"
+    "prompt_template": "USER:\n{prompt}\nASSISTANT:",
+    "llama_model_path": "nous-capybara-34b.Q5_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 24320000000
   },
   "engine": "nitro"
-  }
+}
-  

models/codeninja-1.0-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/beowolx/CodeNinja-1.0-OpenChat-7B-GGUF/resolve/main/codeninja-1.0-openchat-7b.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "codeninja-1.0-openchat-7b.Q4_K_M.gguf",
+      "url": "https://huggingface.co/beowolx/CodeNinja-1.0-OpenChat-7B-GGUF/resolve/main/codeninja-1.0-openchat-7b.Q4_K_M.gguf"
+    }
+  ],
   "id": "codeninja-1.0-7b",
   "object": "model",
   "name": "CodeNinja 7B Q4",
@@ -8,14 +13,14 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:"
+    "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
+    "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": ["<|end_of_turn|>"],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
@@ -25,5 +30,4 @@
     "size": 4370000000
   },
   "engine": "nitro"
-  }
+}
-  

models/config/default-model.json

@@ -2,7 +2,12 @@
   "object": "model",
   "version": 1,
   "format": "gguf",
-  "source_url": "N/A",
+  "sources": [
+    {
+      "url": "N/A",
+      "filename": "N/A"
+    }
+  ],
   "id": "N/A",
   "name": "N/A",
   "created": 0,
@@ -10,7 +15,8 @@
   "settings": {
     "ctx_len": 4096,
     "embedding": false,
-    "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:"
+    "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:",
+    "llama_model_path": "N/A"
   },
   "parameters": {
     "temperature": 0.7,

models/deepseek-coder-1.3b/model.json

@@ -1,6 +1,10 @@
-
 {
-    "source_url": "https://huggingface.co/TheBloke/deepseek-coder-1.3b-instruct-GGUF/resolve/main/deepseek-coder-1.3b-instruct.Q8_0.gguf",
+  "sources": [
+    {
+      "filename": "deepseek-coder-1.3b-instruct.Q8_0.gguf",
+      "url": "https://huggingface.co/TheBloke/deepseek-coder-1.3b-instruct-GGUF/resolve/main/deepseek-coder-1.3b-instruct.Q8_0.gguf"
+    }
+  ],
   "id": "deepseek-coder-1.3b",
   "object": "model",
   "name": "Deepseek Coder 1.3B Q8",
@@ -9,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "### Instruction:\n{prompt}\n### Response:"
+    "prompt_template": "### Instruction:\n{prompt}\n### Response:",
+    "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -26,4 +31,4 @@
     "size": 1430000000
   },
   "engine": "nitro"
-  }
+}

models/deepseek-coder-34b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/deepseek-coder-33B-instruct-GGUF/resolve/main/deepseek-coder-33b-instruct.Q5_K_M.gguf",
+  "sources": [
+    {
+      "filename": "deepseek-coder-33b-instruct.Q5_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/deepseek-coder-33B-instruct-GGUF/resolve/main/deepseek-coder-33b-instruct.Q5_K_M.gguf"
+    }
+  ],
   "id": "deepseek-coder-34b",
   "object": "model",
   "name": "Deepseek Coder 33B Q5",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "### Instruction:\n{prompt}\n### Response:"
+    "prompt_template": "### Instruction:\n{prompt}\n### Response:",
+    "llama_model_path": "deepseek-coder-33b-instruct.Q5_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 19940000000
   },
   "engine": "nitro"
-  }
+}
-  

models/dolphin-2.7-mixtral-8x7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/dolphin-2.7-mixtral-8x7b-GGUF/resolve/main/dolphin-2.7-mixtral-8x7b.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "dolphin-2.7-mixtral-8x7b.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/dolphin-2.7-mixtral-8x7b-GGUF/resolve/main/dolphin-2.7-mixtral-8x7b.Q4_K_M.gguf"
+    }
+  ],
   "id": "dolphin-2.7-mixtral-8x7b",
   "object": "model",
   "name": "Dolphin 8x7B Q4",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
+    "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+    "llama_model_path": "dolphin-2.7-mixtral-8x7b.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,4 +31,4 @@
     "size": 26440000000
   },
   "engine": "nitro"
-  }
+}

models/gpt-3.5-turbo-16k-0613/model.json

@@ -1,5 +1,9 @@
 {
-    "source_url": "https://openai.com",
+  "sources": [
+    {
+      "url": "https://openai.com"
+    }
+  ],
   "id": "gpt-3.5-turbo-16k-0613",
   "object": "model",
   "name": "OpenAI GPT 3.5 Turbo 16k 0613",
@@ -12,7 +16,5 @@
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]
   },
-    "engine": "openai",
+  "engine": "openai"
-    "state": "ready"
 }
-  

models/gpt-3.5-turbo/model.json

@@ -1,5 +1,9 @@
 {
-    "source_url": "https://openai.com",
+  "sources": [
+    {
+      "url": "https://openai.com"
+    }
+  ],
   "id": "gpt-3.5-turbo",
   "object": "model",
   "name": "OpenAI GPT 3.5 Turbo",
@@ -12,7 +16,5 @@
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]
   },
-    "engine": "openai",
+  "engine": "openai"
-    "state": "ready"
 }
-  

models/gpt-4/model.json

@@ -1,5 +1,9 @@
 {
-    "source_url": "https://openai.com",
+  "sources": [
+    {
+      "url": "https://openai.com"
+    }
+  ],
   "id": "gpt-4",
   "object": "model",
   "name": "OpenAI GPT 4",
@@ -12,7 +16,5 @@
     "author": "OpenAI",
     "tags": ["General", "Big Context Length"]
   },
-    "engine": "openai",
+  "engine": "openai"
-    "state": "ready"
 }
-  

models/llama2-chat-70b-q4/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Llama-2-70B-Chat-GGUF/resolve/main/llama-2-70b-chat.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "llama-2-70b-chat.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Llama-2-70B-Chat-GGUF/resolve/main/llama-2-70b-chat.Q4_K_M.gguf"
+    }
+  ],
   "id": "llama2-chat-70b-q4",
   "object": "model",
   "name": "Llama 2 Chat 70B Q4",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]"
+    "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
+    "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 43920000000
   },
   "engine": "nitro"
-  }
+}
-  

models/llama2-chat-7b-q4/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "llama-2-7b-chat.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf"
+    }
+  ],
   "id": "llama2-chat-7b-q4",
   "object": "model",
   "name": "Llama 2 Chat 7B Q4",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]"
+    "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
+    "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 4080000000
   },
   "engine": "nitro"
-  }
+}
-  

models/llava-1.5-13b-q5/model.json

@@ -0,0 +1,33 @@
+{
+  "sources": [
+    {
+      "filename": "ggml-model-q5_k.gguf",
+      "url": "https://huggingface.co/mys/ggml_llava-v1.5-13b/resolve/main/ggml-model-q5_k.gguf"
+    },
+    {
+      "filename": "mmproj-model-f16.gguf",
+      "url": "https://huggingface.co/mys/ggml_llava-v1.5-13b/resolve/main/mmproj-model-f16.gguf"
+    }
+  ],
+  "id": "llava-1.5-13b-q5",
+  "object": "model",
+  "name": "LlaVa 1.5 13B Q5 K",
+  "version": "1.0",
+  "description": "LlaVa 1.5 can bring vision understanding to Jan",
+  "format": "gguf",
+  "settings": {
+    "ctx_len": 4096,
+    "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
+    "llama_model_path": "ggml-model-q5_k.gguf",
+    "mmproj": "mmproj-model-f16.gguf"
+  },
+  "parameters": {
+    "max_tokens": 4096
+  },
+  "metadata": {
+    "author": "Mys",
+    "tags": ["Vision"],
+    "size": 9850000000
+  },
+  "engine": "nitro"
+}

models/llava-1.5-7b-q5/model.json

@@ -0,0 +1,33 @@
+{
+  "sources": [
+    {
+      "filename": "ggml-model-q5_k.gguf",
+      "url": "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q5_k.gguf"
+    },
+    {
+      "filename": "mmproj-model-f16.gguf",
+      "url": "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf"
+    }
+  ],
+  "id": "llava-1.5-7b-q5",
+  "object": "model",
+  "name": "LlaVa 1.5 7B Q5 K",
+  "version": "1.0",
+  "description": "LlaVa 1.5 can bring vision understanding to Jan",
+  "format": "gguf",
+  "settings": {
+    "ctx_len": 4096,
+    "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
+    "llama_model_path": "ggml-model-q5_k.gguf",
+    "mmproj": "mmproj-model-f16.gguf"
+  },
+  "parameters": {
+    "max_tokens": 4096
+  },
+  "metadata": {
+    "author": "Mys",
+    "tags": ["Vision"],
+    "size": 5400000000
+  },
+  "engine": "nitro"
+}

models/mistral-ins-7b-q4/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "mistral-7b-instruct-v0.2.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+    }
+  ],
   "id": "mistral-ins-7b-q4",
   "object": "model",
   "name": "Mistral Instruct 7B Q4",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "[INST] {prompt} [/INST]"
+    "prompt_template": "[INST] {prompt} [/INST]",
+    "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -26,5 +32,4 @@
     "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/mistral-ins-7b-q4/cover.png"
   },
   "engine": "nitro"
-  }
+}
-  

models/mixtral-8x7b-instruct/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"
+    }
+  ],
   "id": "mixtral-8x7b-instruct",
   "object": "model",
   "name": "Mixtral 8x7B Instruct Q4",
@@ -8,14 +13,14 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "[INST] {prompt} [/INST]"
+    "prompt_template": "[INST] {prompt} [/INST]",
+    "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
@@ -25,4 +30,4 @@
     "size": 26440000000
   },
   "engine": "nitro"
-  }
+}

models/noromaid-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/NeverSleep/Noromaid-7b-v0.1.1-GGUF/resolve/main/Noromaid-7b-v0.1.1.q5_k_m.gguf",
+  "sources": [
+    {
+      "filename": "Noromaid-7b-v0.1.1.q5_k_m.gguf",
+      "url": "https://huggingface.co/NeverSleep/Noromaid-7b-v0.1.1-GGUF/resolve/main/Noromaid-7b-v0.1.1.q5_k_m.gguf"
+    }
+  ],
   "id": "noromaid-7b",
   "object": "model",
   "name": "Noromaid 7B Q5",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "### Instruction:{prompt}\n### Response:"
+    "prompt_template": "### Instruction:{prompt}\n### Response:",
+    "llama_model_path": "Noromaid-7b-v0.1.1.q5_k_m.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 4370000000
   },
   "engine": "nitro"
-  }
+}
-  

models/openchat-3.5-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/openchat-3.5-1210-GGUF/resolve/main/openchat-3.5-1210.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "openchat-3.5-1210.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/openchat-3.5-1210-GGUF/resolve/main/openchat-3.5-1210.Q4_K_M.gguf"
+    }
+  ],
   "id": "openchat-3.5-7b",
   "object": "model",
   "name": "Openchat-3.5 7B Q4",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:"
+    "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
+    "llama_model_path": "openchat-3.5-1210.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,4 +31,4 @@
     "size": 4370000000
   },
   "engine": "nitro"
-  }
+}

models/openhermes-neural-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/janhq/openhermes-2.5-neural-chat-v3-3-slerp-GGUF/resolve/main/openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf",
+      "url": "https://huggingface.co/janhq/openhermes-2.5-neural-chat-v3-3-slerp-GGUF/resolve/main/openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf"
+    }
+  ],
   "id": "openhermes-neural-7b",
   "object": "model",
   "name": "OpenHermes Neural 7B Q4",
@@ -8,14 +13,14 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
+    "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+    "llama_model_path": "openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
@@ -26,4 +31,4 @@
     "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/openhermes-neural-7b/cover.png"
   },
   "engine": "nitro"
-  }
+}

models/phi-2-3b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/phi-2-GGUF/resolve/main/phi-2.Q8_0.gguf",
+  "sources": [
+    {
+      "filename": "phi-2.Q8_0.gguf",
+      "url": "https://huggingface.co/TheBloke/phi-2-GGUF/resolve/main/phi-2.Q8_0.gguf"
+    }
+  ],
   "id": "phi-2-3b",
   "object": "model",
   "name": "Phi-2 3B Q8",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "Intruct:\n{prompt}\nOutput:"
+    "prompt_template": "Intruct:\n{prompt}\nOutput:",
+    "llama_model_path": "phi-2.Q8_0.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -21,9 +27,8 @@
   },
   "metadata": {
     "author": "Microsoft",
-      "tags": ["3B","Foundational Model"],
+    "tags": ["3B", "Foundational Model"],
     "size": 2960000000
   },
   "engine": "nitro"
-  }
+}
-  

models/phind-34b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Phind-CodeLlama-34B-v2-GGUF/resolve/main/phind-codellama-34b-v2.Q5_K_M.gguf",
+  "sources": [
+    {
+      "filename": "phind-codellama-34b-v2.Q5_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Phind-CodeLlama-34B-v2-GGUF/resolve/main/phind-codellama-34b-v2.Q5_K_M.gguf"
+    }
+  ],
   "id": "phind-34b",
   "object": "model",
   "name": "Phind 34B Q5",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "### System Prompt\n{system_message}\n### User Message\n{prompt}\n### Assistant"
+    "prompt_template": "### System Prompt\n{system_message}\n### User Message\n{prompt}\n### Assistant",
+    "llama_model_path": "phind-codellama-34b-v2.Q5_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 20220000000
   },
   "engine": "nitro"
-  }
+}
-  

models/solar-10.7b-slerp/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/janhq/Solar-10.7B-SLERP-GGUF/resolve/main/solar-10.7b-slerp.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "solar-10.7b-slerp.Q4_K_M.gguf",
+      "url": "https://huggingface.co/janhq/Solar-10.7B-SLERP-GGUF/resolve/main/solar-10.7b-slerp.Q4_K_M.gguf"
+    }
+  ],
   "id": "solar-10.7b-slerp",
   "object": "model",
   "name": "Solar Slerp 10.7B Q4",
@@ -8,22 +13,21 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "### User: {prompt}\n### Assistant:"
+    "prompt_template": "### User: {prompt}\n### Assistant:",
+    "llama_model_path": "solar-10.7b-slerp.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
   "metadata": {
     "author": "Jan",
-      "tags": ["13B","Finetuned"],
+    "tags": ["13B", "Finetuned"],
     "size": 6360000000
   },
   "engine": "nitro"
-  }
+}
-  

models/starling-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Starling-LM-7B-alpha-GGUF/resolve/main/starling-lm-7b-alpha.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "starling-lm-7b-alpha.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Starling-LM-7B-alpha-GGUF/resolve/main/starling-lm-7b-alpha.Q4_K_M.gguf"
+    }
+  ],
   "id": "starling-7b",
   "object": "model",
   "name": "Starling alpha 7B Q4",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "GPT4 User: {prompt}<|end_of_turn|>GPT4 Assistant:"
+    "prompt_template": "GPT4 User: {prompt}<|end_of_turn|>GPT4 Assistant:",
+    "llama_model_path": "starling-lm-7b-alpha.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -21,9 +27,8 @@
   },
   "metadata": {
     "author": "Berkeley-nest, The Bloke",
-      "tags": ["7B","Finetuned"],
+    "tags": ["7B", "Finetuned"],
     "size": 4370000000
   },
   "engine": "nitro"
-  }
+}
-  

models/stealth-v1.2-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/janhq/stealth-v1.3-GGUF/resolve/main/stealth-v1.3.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "stealth-v1.3.Q4_K_M.gguf",
+      "url": "https://huggingface.co/janhq/stealth-v1.3-GGUF/resolve/main/stealth-v1.3.Q4_K_M.gguf"
+    }
+  ],
   "id": "stealth-v1.2-7b",
   "object": "model",
   "name": "Stealth 7B Q4",
@@ -8,25 +13,21 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
+    "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+    "llama_model_path": "stealth-v1.3.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
   "metadata": {
     "author": "Jan",
-      "tags": [
+    "tags": ["7B", "Finetuned", "Featured"],
-        "7B",
-        "Finetuned",
-        "Featured"
-      ],
     "size": 4370000000
   },
   "engine": "nitro"
-  }
+}

models/tinyllama-1.1b/model.json

@@ -1,5 +1,10 @@
 {
-  "source_url": "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+    }
+  ],
   "id": "tinyllama-1.1b",
   "object": "model",
   "name": "TinyLlama Chat 1.1B Q4",
@@ -7,8 +12,9 @@
   "description": "TinyLlama is a tiny model with only 1.1B. It's a good model for less powerful computers.",
   "format": "gguf",
   "settings": {
-      "ctx_len": 2048,
+    "ctx_len": 4096,
-      "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>"
+    "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>",
+    "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,

models/trinity-v1.2-7b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/janhq/trinity-v1.2-GGUF/resolve/main/trinity-v1.2.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "trinity-v1.2.Q4_K_M.gguf",
+      "url": "https://huggingface.co/janhq/trinity-v1.2-GGUF/resolve/main/trinity-v1.2.Q4_K_M.gguf"
+    }
+  ],
   "id": "trinity-v1.2-7b",
   "object": "model",
   "name": "Trinity-v1.2 7B Q4",
@@ -8,14 +13,14 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
+    "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+    "llama_model_path": "trinity-v1.2.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
@@ -26,4 +31,4 @@
     "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/trinity-v1.2-7b/cover.png"
   },
   "engine": "nitro"
-  }  
+}

models/tulu-2-70b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/tulu-2-dpo-70B-GGUF/resolve/main/tulu-2-dpo-70b.Q4_K_M.gguf",
+  "sources": [
+    {
+      "filename": "tulu-2-dpo-70b.Q4_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/tulu-2-dpo-70B-GGUF/resolve/main/tulu-2-dpo-70b.Q4_K_M.gguf"
+    }
+  ],
   "id": "tulu-2-70b",
   "object": "model",
   "name": "Tulu 2 70B Q4",
@@ -8,14 +13,14 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "<|user|>\n{prompt}\n<|assistant|>"
+    "prompt_template": "<|user|>\n{prompt}\n<|assistant|>",
+    "llama_model_path": "tulu-2-dpo-70b.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
@@ -25,4 +30,4 @@
     "size": 41400000000
   },
   "engine": "nitro"
-  }
+}

models/wizardcoder-13b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/WizardCoder-Python-13B-V1.0-GGUF/resolve/main/wizardcoder-python-13b-v1.0.Q5_K_M.gguf",
+  "sources": [
+    {
+      "filename": "wizardcoder-python-13b-v1.0.Q5_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/WizardCoder-Python-13B-V1.0-GGUF/resolve/main/wizardcoder-python-13b-v1.0.Q5_K_M.gguf"
+    }
+  ],
   "id": "wizardcoder-13b",
   "object": "model",
   "name": "Wizard Coder Python 13B Q5",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "### Instruction:\n{prompt}\n### Response:"
+    "prompt_template": "### Instruction:\n{prompt}\n### Response:",
+    "llama_model_path": "wizardcoder-python-13b-v1.0.Q5_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 7870000000
   },
   "engine": "nitro"
-  }
+}
-  

models/yarn-mistral-7b/model.json

@@ -1,5 +1,9 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Yarn-Mistral-7B-128k-GGUF/resolve/main/yarn-mistral-7b-128k.Q4_K_M.gguf",
+  "sources": [
+    {
+      "url": "https://huggingface.co/TheBloke/Yarn-Mistral-7B-128k-GGUF/resolve/main/yarn-mistral-7b-128k.Q4_K_M.gguf"
+    }
+  ],
   "id": "yarn-mistral-7b",
   "object": "model",
   "name": "Yarn Mistral 7B Q4",
@@ -15,15 +19,13 @@
     "top_p": 0.95,
     "stream": true,
     "max_tokens": 4096,
-      "stop": [],
     "frequency_penalty": 0,
     "presence_penalty": 0
   },
   "metadata": {
     "author": "NousResearch, The Bloke",
-      "tags": ["7B","Finetuned"],
+    "tags": ["7B", "Finetuned"],
     "size": 4370000000
   },
   "engine": "nitro"
-  }
+}
-  

models/yi-34b/model.json

@@ -1,5 +1,10 @@
 {
-    "source_url": "https://huggingface.co/TheBloke/Yi-34B-Chat-GGUF/resolve/main/yi-34b-chat.Q5_K_M.gguf",
+  "sources": [
+    {
+      "filename": "yi-34b-chat.Q5_K_M.gguf",
+      "url": "https://huggingface.co/TheBloke/Yi-34B-Chat-GGUF/resolve/main/yi-34b-chat.Q5_K_M.gguf"
+    }
+  ],
   "id": "yi-34b",
   "object": "model",
   "name": "Yi 34B Q5",
@@ -8,7 +13,8 @@
   "format": "gguf",
   "settings": {
     "ctx_len": 4096,
-      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
+    "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+    "llama_model_path": "yi-34b-chat.Q5_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,
@@ -25,5 +31,4 @@
     "size": 20660000000
   },
   "engine": "nitro"
-  }
+}
-  

web/containers/Providers/EventListener.tsx

@@ -105,6 +105,7 @@ export default function EventListenerWrapper({ children }: PropsWithChildren) {
       })
     }
     return () => {}
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
   return (

web/hooks/useDownloadModel.ts

@@ -6,6 +6,7 @@ import {
   ModelExtension,
   abortDownload,
   joinPath,
+  ModelArtifact,
 } from '@janhq/core'
 
 import { useSetAtom } from 'jotai'
@@ -25,6 +26,23 @@ export default function useDownloadModel() {
   const addNewDownloadingModel = useSetAtom(addNewDownloadingModelAtom)
 
   const downloadModel = async (model: Model) => {
+    const childrenDownloadProgress: DownloadState[] = []
+    model.sources.forEach((source: ModelArtifact) => {
+      childrenDownloadProgress.push({
+        modelId: source.filename,
+        time: {
+          elapsed: 0,
+          remaining: 0,
+        },
+        speed: 0,
+        percent: 0,
+        size: {
+          total: 0,
+          transferred: 0,
+        },
+      })
+    })
+
     // set an initial download state
     setDownloadState({
       modelId: model.id,
@@ -38,6 +56,7 @@ export default function useDownloadModel() {
         total: 0,
         transferred: 0,
       },
+      children: childrenDownloadProgress,
     })
 
     addNewDownloadingModel(model)
@@ -46,6 +65,7 @@ export default function useDownloadModel() {
       .get<ModelExtension>(ExtensionTypeEnum.Model)
       ?.downloadModel(model, { ignoreSSL, proxy })
   }
+
   const abortModelDownload = async (model: Model) => {
     await abortDownload(
       await joinPath(['models', model.id, modelBinFileName(model)])

web/hooks/useGetConfiguredModels.ts

@@ -1,7 +1,6 @@
 import { useEffect, useState } from 'react'
 
-import { ExtensionTypeEnum, ModelExtension } from '@janhq/core'
+import { ExtensionTypeEnum, ModelExtension, Model } from '@janhq/core'
-import { Model } from '@janhq/core'
 
 import { extensionManager } from '@/extension/ExtensionManager'
 
@@ -25,6 +24,7 @@ export function useGetConfiguredModels() {
 
   useEffect(() => {
     fetchModels()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
   return { loading, models }

web/hooks/useGetDownloadedModels.ts

@@ -6,7 +6,7 @@ import { atom, useAtom } from 'jotai'
 
 import { extensionManager } from '@/extension/ExtensionManager'
 
-const downloadedModelsAtom = atom<Model[]>([])
+export const downloadedModelsAtom = atom<Model[]>([])
 
 export function useGetDownloadedModels() {
   const [downloadedModels, setDownloadedModels] = useAtom(downloadedModelsAtom)
@@ -15,7 +15,8 @@ export function useGetDownloadedModels() {
     getDownloadedModels().then((downloadedModels) => {
       setDownloadedModels(downloadedModels)
     })
-  }, [setDownloadedModels])
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
 
   return { downloadedModels, setDownloadedModels }
 }

web/hooks/useRecommendedModel.ts

@@ -110,6 +110,7 @@ export default function useRecommendedModel() {
 
     console.debug(`Using last used model ${lastUsedModel.id}`)
     setRecommendedModel(lastUsedModel)
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [getAndSortDownloadedModels, activeThread])
 
   useEffect(() => {

web/hooks/useSettings.ts

@@ -13,6 +13,7 @@ export const useSettings = () => {
 
   useEffect(() => {
     setTimeout(() => validateSettings, 3000)
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
   const validateSettings = async () => {

web/package.json

@@ -21,7 +21,7 @@
     "class-variance-authority": "^0.7.0",
     "framer-motion": "^10.16.4",
     "highlight.js": "^11.9.0",
-    "jotai": "^2.4.0",
+    "jotai": "^2.6.0",
     "lodash": "^4.17.21",
     "lucide-react": "^0.291.0",
     "marked": "^9.1.2",

web/screens/ExploreModels/ExploreModelItemHeader/index.tsx

@@ -26,7 +26,7 @@ import { useCreateNewThread } from '@/hooks/useCreateNewThread'
 import useDownloadModel from '@/hooks/useDownloadModel'
 import { useDownloadState } from '@/hooks/useDownloadState'
 import { getAssistants } from '@/hooks/useGetAssistants'
-import { useGetDownloadedModels } from '@/hooks/useGetDownloadedModels'
+import { downloadedModelsAtom } from '@/hooks/useGetDownloadedModels'
 import { useMainViewState } from '@/hooks/useMainViewState'
 
 import { toGibibytes } from '@/utils/converter'
@@ -43,8 +43,8 @@ type Props = {
 
 const ExploreModelItemHeader: React.FC<Props> = ({ model, onClick, open }) => {
   const { downloadModel } = useDownloadModel()
-  const { downloadedModels } = useGetDownloadedModels()
+  const downloadedModels = useAtomValue(downloadedModelsAtom)
-  const { modelDownloadStateAtom, downloadStates } = useDownloadState()
+  const { modelDownloadStateAtom } = useDownloadState()
   const { requestCreateNewThread } = useCreateNewThread()
   const totalRam = useAtomValue(totalRamAtom)
   const serverEnabled = useAtomValue(serverEnabledAtom)
@@ -100,9 +100,7 @@ const ExploreModelItemHeader: React.FC<Props> = ({ model, onClick, open }) => {
         )}
       </Tooltip>
     )
-  }
+  } else if (downloadState != null) {
-
-  if (downloadState != null && downloadStates.length > 0) {
     downloadButton = <ModalCancelDownload model={model} />
   }
 

web/types/downloadState.d.ts

@@ -4,6 +4,8 @@ type DownloadState = {
   speed: number
   percent: number
   size: DownloadSize
+  isFinished?: boolean
+  children?: DownloadState[]
   error?: string
 }
 

web/utils/model.ts

@@ -2,7 +2,7 @@ import { Model } from '@janhq/core'
 
 export const modelBinFileName = (model: Model) => {
   const modelFormatExt = '.gguf'
-  const extractedFileName = model.source_url?.split('/').pop() ?? model.id
+  const extractedFileName = model.sources[0]?.url.split('/').pop() ?? model.id
   const fileName = extractedFileName.toLowerCase().endsWith(modelFormatExt)
     ? extractedFileName
     : model.id

web/utils/modelParam.ts

@@ -40,6 +40,8 @@ export const toSettingParams = (
     n_parallel: undefined,
     cpu_threads: undefined,
     prompt_template: undefined,
+    llama_model_path: undefined,
+    mmproj: undefined,
   }
   const settingParams: ModelSettingParams = {}