Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: disable flash_attn if vulkan is enabled (#3080)

Browse Source 
Signed-off-by: James <namnh0122@gmail.com>
This commit is contained in:
NamH 2024-06-21 14:27:16 +07:00 committed by GitHub
parent a91fe5c584
commit d2284070fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
web/hooks/useActiveModel.ts
View File

@ -7,6 +7,7 @@ import { toaster } from '@/containers/Toast'
import { LAST_USED_MODEL_ID } from './useRecommendedModel' import { LAST_USED_MODEL_ID } from './useRecommendedModel'
import { vulkanEnabledAtom } from '@/helpers/atoms/AppConfig.atom'
import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom'
import { activeThreadAtom } from '@/helpers/atoms/Thread.atom' import { activeThreadAtom } from '@/helpers/atoms/Thread.atom'
@ -34,6 +35,7 @@ export function useActiveModel() {
const downloadedModels = useAtomValue(downloadedModelsAtom) const downloadedModels = useAtomValue(downloadedModelsAtom)
const setLoadModelError = useSetAtom(loadModelErrorAtom) const setLoadModelError = useSetAtom(loadModelErrorAtom)
const [pendingModelLoad, setPendingModelLoad] = useAtom(pendingModelLoadAtom) const [pendingModelLoad, setPendingModelLoad] = useAtom(pendingModelLoadAtom)
const isVulkanEnabled = useAtomValue(vulkanEnabledAtom)
const downloadedModelsRef = useRef<Model[]>([]) const downloadedModelsRef = useRef<Model[]>([])
@ -90,6 +92,11 @@ export function useActiveModel() {
} }
} }
if (isVulkanEnabled) {
// @ts-expect-error flash_attn is newly added and will be migrate to cortex in the future
model.settings['flash_attn'] = false
}
localStorage.setItem(LAST_USED_MODEL_ID, model.id) localStorage.setItem(LAST_USED_MODEL_ID, model.id)
const engine = EngineManager.instance().get(model.engine) const engine = EngineManager.instance().get(model.engine)
return engine return engine