chore: default context length to 2048 (#2746)

core/src/browser/extensions/model.ts

@@ -32,4 +32,5 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter
   abstract importModels(models: ImportingModel[], optionType: OptionType): Promise<void>
   abstract updateModelInfo(modelInfo: Partial<Model>): Promise<Model>
   abstract fetchHuggingFaceRepoData(repoId: string): Promise<HuggingFaceRepoData>
+  abstract getDefaultModel(): Promise<Model>
 }

extensions/inference-nitro-extension/resources/default_settings.json

@@ -27,7 +27,7 @@
       "min": 0,
       "max": 4096,
       "step": 128,
-      "value": 4096
+      "value": 2048
     }
   }
 ]

extensions/model-extension/resources/default-model.json

@@ -13,7 +13,7 @@
   "created": 0,
   "description": "User self import model",
   "settings": {
-    "ctx_len": 4096,
+    "ctx_len": 2048,
     "embedding": false,
     "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:",
     "llama_model_path": "N/A"

extensions/model-extension/src/index.ts

@@ -551,7 +551,7 @@ export default class JanModelExtension extends ModelExtension {
     return model
   }
 
-  private async getDefaultModel(): Promise<Model> {
+  override async getDefaultModel(): Promise<Model> {
     const defaultModel = DEFAULT_MODEL as Model
     return defaultModel
   }

web/helpers/atoms/Model.atom.ts

@@ -46,6 +46,8 @@ export const removeDownloadedModelAtom = atom(
 
 export const configuredModelsAtom = atom<Model[]>([])
 
+export const defaultModelAtom = atom<Model | undefined>(undefined)
+
 /// TODO: move this part to another atom
 // store the paths of the models that are being imported
 export const importingModelsAtom = atom<ImportingModel[]>([])

web/hooks/useModels.ts

@@ -13,25 +13,37 @@ import { useSetAtom } from 'jotai'
 import { extensionManager } from '@/extension'
 import {
   configuredModelsAtom,
+  defaultModelAtom,
   downloadedModelsAtom,
 } from '@/helpers/atoms/Model.atom'
 
 const useModels = () => {
   const setDownloadedModels = useSetAtom(downloadedModelsAtom)
   const setConfiguredModels = useSetAtom(configuredModelsAtom)
+  const setDefaultModel = useSetAtom(defaultModelAtom)
 
   const getData = useCallback(() => {
     const getDownloadedModels = async () => {
       const models = await getLocalDownloadedModels()
       setDownloadedModels(models)
     }
+
     const getConfiguredModels = async () => {
       const models = await getLocalConfiguredModels()
       setConfiguredModels(models)
     }
-    getDownloadedModels()
-    getConfiguredModels()
-  }, [setDownloadedModels, setConfiguredModels])
+
+    const getDefaultModel = async () => {
+      const defaultModel = await getLocalDefaultModel()
+      setDefaultModel(defaultModel)
+    }
+
+    Promise.all([
+      getDownloadedModels(),
+      getConfiguredModels(),
+      getDefaultModel(),
+    ])
+  }, [setDownloadedModels, setConfiguredModels, setDefaultModel])
 
   useEffect(() => {
     // Try get data on mount
@@ -46,6 +58,11 @@ const useModels = () => {
   }, [getData])
 }
 
+const getLocalDefaultModel = async (): Promise<Model | undefined> =>
+  extensionManager
+    .get<ModelExtension>(ExtensionTypeEnum.Model)
+    ?.getDefaultModel()
+
 const getLocalConfiguredModels = async (): Promise<Model[]> =>
   extensionManager
     .get<ModelExtension>(ExtensionTypeEnum.Model)

web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx

@@ -3,7 +3,6 @@ import { useCallback, useMemo } from 'react'
 import {
   DownloadState,
   HuggingFaceRepoData,
-  InferenceEngine,
   Model,
   Quantization,
 } from '@janhq/core'
@@ -23,7 +22,10 @@ import { mainViewStateAtom } from '@/helpers/atoms/App.atom'
 import { assistantsAtom } from '@/helpers/atoms/Assistant.atom'
 
 import { importHuggingFaceModelStageAtom } from '@/helpers/atoms/HuggingFace.atom'
-import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom'
+import {
+  defaultModelAtom,
+  downloadedModelsAtom,
+} from '@/helpers/atoms/Model.atom'
 
 type Props = {
   index: number
@@ -52,15 +54,15 @@ const ModelDownloadRow: React.FC<Props> = ({
   const isDownloaded = downloadedModels.find((md) => md.id === fileName) != null
 
   const setHfImportingStage = useSetAtom(importHuggingFaceModelStageAtom)
+  const defaultModel = useAtomValue(defaultModelAtom)
 
   const model = useMemo(() => {
-    const promptData: string =
-      (repoData.cardData['prompt_template'] as string) ??
-      '{system_message}\n### Instruction: {prompt}\n### Response:'
+    if (!defaultModel) {
+      return undefined
+    }
+
     const model: Model = {
-      object: 'model',
-      version: '1.0',
-      format: 'gguf',
+      ...defaultModel,
       sources: [
         {
           url: downloadUrl,
@@ -70,38 +72,26 @@ const ModelDownloadRow: React.FC<Props> = ({
       id: fileName,
       name: fileName,
       created: Date.now(),
-      description: 'User self import model',
-      settings: {
-        ctx_len: 4096,
-        embedding: false,
-        prompt_template: promptData,
-        llama_model_path: 'N/A',
-      },
-      parameters: {
-        temperature: 0.7,
-        top_p: 0.95,
-        stream: true,
-        max_tokens: 2048,
-        stop: ['<endofstring>'],
-        frequency_penalty: 0.7,
-        presence_penalty: 0,
-      },
       metadata: {
         author: 'User',
         tags: repoData.tags,
         size: fileSize,
       },
-      engine: InferenceEngine.nitro,
     }
+    console.log('NamH model: ', JSON.stringify(model))
     return model
-  }, [fileName, fileSize, repoData, downloadUrl])
+  }, [fileName, fileSize, repoData, downloadUrl, defaultModel])
 
   const onAbortDownloadClick = useCallback(() => {
-    abortModelDownload(model)
+    if (model) {
+      abortModelDownload(model)
+    }
   }, [model, abortModelDownload])
 
   const onDownloadClick = useCallback(async () => {
-    downloadModel(model)
+    if (model) {
+      downloadModel(model)
+    }
   }, [model, downloadModel])
 
   const onUseModelClick = useCallback(async () => {
@@ -120,6 +110,10 @@ const ModelDownloadRow: React.FC<Props> = ({
     setHfImportingStage,
   ])
 
+  if (!model) {
+    return null
+  }
+
   return (
     <div className="flex w-[662px] flex-row items-center justify-between space-x-1 rounded border border-border p-3">
       <div className="flex">