diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index 0295f5512..b0d64ff09 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -1,6 +1,11 @@ import { useCallback, useEffect, useState } from 'react' -import { InferenceEngine, Model } from '@janhq/core' +import { + InferenceEngine, + Model, + ModelRuntimeParams, + ModelSettingParams, +} from '@janhq/core' import { Button, Select, @@ -29,6 +34,7 @@ import useRecommendedModel from '@/hooks/useRecommendedModel' import { toGibibytes } from '@/utils/converter' import { + ModelParams, activeThreadAtom, getActiveThreadIdAtom, setThreadModelParamsAtom, @@ -59,6 +65,15 @@ export default function DropdownListSidebar() { const { recommendedModel, downloadedModels } = useRecommendedModel() + /** + * Default value for max_tokens and ctx_len + * Its to avoid OOM issue since a model can set a big number for these settings + */ + const defaultValue = (value?: number) => { + if (value && value < 4096) return value + return 4096 + } + useEffect(() => { setSelected(recommendedModel) setSelectedModel(recommendedModel) @@ -66,9 +81,12 @@ export default function DropdownListSidebar() { if (activeThread) { const finishInit = threadStates[activeThread.id].isFinishInit ?? true if (finishInit) return - const modelParams = { + const modelParams: ModelParams = { ...recommendedModel?.parameters, ...recommendedModel?.settings, + // This is to set default value for these settings instead of maximum value + max_tokens: defaultValue(recommendedModel?.parameters.max_tokens), + ctx_len: defaultValue(recommendedModel?.settings.ctx_len), } setThreadModelParams(activeThread.id, modelParams) } diff --git a/web/utils/componentSettings.ts b/web/utils/componentSettings.ts index 76b0a12c0..3536bfd99 100644 --- a/web/utils/componentSettings.ts +++ b/web/utils/componentSettings.ts @@ -12,11 +12,6 @@ export const getConfigurationsData = ( ) => { const componentData: SettingComponentData[] = [] - const defaultValue = (value?: number) => { - if (value && value < 4096) return value - return 4096 - } - Object.keys(settings).forEach((key: string) => { const componentSetting = presetConfiguration[key] @@ -32,16 +27,10 @@ export const getConfigurationsData = ( case 'max_tokens': componentSetting.controllerData.max = selectedModel?.parameters.max_tokens || 4096 - componentSetting.controllerData.value = defaultValue( - selectedModel?.parameters.max_tokens - ) break case 'ctx_len': componentSetting.controllerData.max = selectedModel?.settings.ctx_len || 4096 - componentSetting.controllerData.value = defaultValue( - selectedModel?.settings.ctx_len - ) break } }