diff --git a/extensions/engine-management-extension/src/index.ts b/extensions/engine-management-extension/src/index.ts index 03970ac94..aef6b75ef 100644 --- a/extensions/engine-management-extension/src/index.ts +++ b/extensions/engine-management-extension/src/index.ts @@ -271,8 +271,16 @@ export default class JanEngineManagementExtension extends EngineManagementExtens * This is to use built-in engine variant in case there is no default engine set */ async updateDefaultEngine() { + const systemInfo = await systemInformation() try { const variant = await this.getDefaultEngineVariant('llama-cpp') + if ( + (systemInfo.gpuSetting.vulkan && !variant.variant.includes('vulkan')) || + (systemInfo.gpuSetting.vulkan === false && + variant.variant.includes('vulkan')) + ) { + throw new EngineError('Switch engine.') + } const installedEngines = await this.getInstalledEngines('llama-cpp') if ( !installedEngines.some( @@ -289,7 +297,6 @@ export default class JanEngineManagementExtension extends EngineManagementExtens (error instanceof HTTPError && error.response.status === 400) || error instanceof EngineError ) { - const systemInfo = await systemInformation() const variant = await engineVariant(systemInfo.gpuSetting) // TODO: Use correct provider name when moving to llama.cpp extension await this.setDefaultEngineVariant('llama-cpp', { diff --git a/extensions/engine-management-extension/src/utils.ts b/extensions/engine-management-extension/src/utils.ts index fc80d255f..bc5b09fd3 100644 --- a/extensions/engine-management-extension/src/utils.ts +++ b/extensions/engine-management-extension/src/utils.ts @@ -45,7 +45,9 @@ const os = (settings?: GpuSetting): string => { * @param settings * @returns */ -const cudaVersion = (settings?: GpuSetting): 'cu12.0' | 'cu11.7' | undefined => { +const cudaVersion = ( + settings?: GpuSetting +): 'cu12.0' | 'cu11.7' | undefined => { return settings.gpus?.some((gpu) => gpu.version.includes('12')) ? 'cu12.0' : 'cu11.7' @@ -70,9 +72,9 @@ export const engineVariant = async ( const runMode = gpuRunMode(gpuSetting) // Only Nvidia GPUs have addition_information set and activated by default let engineVariant = - !gpuSetting?.vulkan || - !gpuSetting.gpus?.length || - gpuSetting.gpus.some((e) => e.additional_information && e.activated) + !gpuSetting?.vulkan && + (!gpuSetting.gpus?.length || + gpuSetting.gpus.some((e) => e.additional_information && e.activated)) ? [ platform, ...(runMode === RunMode.Cuda diff --git a/extensions/inference-cortex-extension/resources/default_settings.json b/extensions/inference-cortex-extension/resources/default_settings.json index c7c88b842..881f74404 100644 --- a/extensions/inference-cortex-extension/resources/default_settings.json +++ b/extensions/inference-cortex-extension/resources/default_settings.json @@ -58,7 +58,7 @@ "description": "Optimizes memory usage and speeds up model inference using an efficient attention implementation.", "controllerType": "checkbox", "controllerProps": { - "value": true + "value": false } }, { diff --git a/web-app/src/constants/localStorage.ts b/web-app/src/constants/localStorage.ts index 5eef965cd..2c11ee7c9 100644 --- a/web-app/src/constants/localStorage.ts +++ b/web-app/src/constants/localStorage.ts @@ -10,6 +10,7 @@ export const localStorageKey = { settingLocalApiServer: 'setting-local-api-server', settingProxyConfig: 'setting-proxy-config', settingHardware: 'setting-hardware', + settingVulkan: 'setting-vulkan', productAnalyticPrompt: 'productAnalyticPrompt', productAnalytic: 'productAnalytic', toolApproval: 'tool-approval', diff --git a/web-app/src/hooks/useVulkan.ts b/web-app/src/hooks/useVulkan.ts new file mode 100644 index 000000000..d71e04603 --- /dev/null +++ b/web-app/src/hooks/useVulkan.ts @@ -0,0 +1,34 @@ +import { create } from 'zustand' +import { persist, createJSONStorage } from 'zustand/middleware' +import { localStorageKey } from '@/constants/localStorage' + +interface VulkanStore { + // Vulkan state + vulkanEnabled: boolean + + // Update functions + setVulkanEnabled: (enabled: boolean) => void + toggleVulkan: () => void +} + +export const useVulkan = create()( + persist( + (set) => ({ + vulkanEnabled: false, + + setVulkanEnabled: (enabled) => + set({ + vulkanEnabled: enabled, + }), + + toggleVulkan: () => + set((state) => ({ + vulkanEnabled: !state.vulkanEnabled, + })), + }), + { + name: localStorageKey.settingVulkan, + storage: createJSONStorage(() => localStorage), + } + ) +) diff --git a/web-app/src/lib/service.ts b/web-app/src/lib/service.ts index ce1a45cdf..99e958ff2 100644 --- a/web-app/src/lib/service.ts +++ b/web-app/src/lib/service.ts @@ -6,6 +6,7 @@ import { } from '@janhq/core' import { invoke, InvokeArgs } from '@tauri-apps/api/core' import { ExtensionManager } from './extension' +import { useVulkan } from '@/hooks/useVulkan' export const AppRoutes = [ 'installExtensions', @@ -54,7 +55,7 @@ export const systemInformation = async () => { const gpuSettingInfo = { gpus: hardwareInfo.gpus.filter((gpu) => gpu.total_vram > 0), - vulkan: false, + vulkan: useVulkan.getState().vulkanEnabled, cpu: hardwareInfo.cpu, } diff --git a/web-app/src/routes/settings/hardware.tsx b/web-app/src/routes/settings/hardware.tsx index e3014b280..287be617c 100644 --- a/web-app/src/routes/settings/hardware.tsx +++ b/web-app/src/routes/settings/hardware.tsx @@ -7,6 +7,7 @@ import { Switch } from '@/components/ui/switch' import { Progress } from '@/components/ui/progress' import { useTranslation } from 'react-i18next' import { useHardware } from '@/hooks/useHardware' +import { useVulkan } from '@/hooks/useVulkan' import type { GPU, HardwareData } from '@/hooks/useHardware' import { useEffect } from 'react' import { @@ -122,6 +123,7 @@ function Hardware() { updateRAMAvailable, reorderGPUs, } = useHardware() + const { vulkanEnabled, setVulkanEnabled } = useVulkan() useEffect(() => { getHardwareInfo().then((data) => @@ -337,6 +339,27 @@ function Hardware() { /> + {/* Vulkan Settings */} + + + { + setVulkanEnabled(checked) + setTimeout(() => { + window.location.reload() + }, 500) // Reload after 500ms to apply changes + }} + /> + + } + /> + + {/* GPU Information */} {hardwareData.gpus.length > 0 ? (