feat: setting toggle vulkan (#5126)

* feat: setting toggle vulkan

* feat: add vulkan toggle setting

* chore: default flash attention disable

* chore: fix vulkan retrieval

* fix: vulkan setting does not affect engine run

* Update web-app/src/routes/settings/hardware.tsx

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

---------

Co-authored-by: Louis <louis@jan.ai>
Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

extensions/engine-management-extension/src/index.ts

@@ -271,8 +271,16 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
    * This is to use built-in engine variant in case there is no default engine set
    */
   async updateDefaultEngine() {
+    const systemInfo = await systemInformation()
     try {
       const variant = await this.getDefaultEngineVariant('llama-cpp')
+      if (
+        (systemInfo.gpuSetting.vulkan && !variant.variant.includes('vulkan')) ||
+        (systemInfo.gpuSetting.vulkan === false &&
+          variant.variant.includes('vulkan'))
+      ) {
+        throw new EngineError('Switch engine.')
+      }
       const installedEngines = await this.getInstalledEngines('llama-cpp')
       if (
         !installedEngines.some(
@@ -289,7 +297,6 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
         (error instanceof HTTPError && error.response.status === 400) ||
         error instanceof EngineError
       ) {
-        const systemInfo = await systemInformation()
         const variant = await engineVariant(systemInfo.gpuSetting)
         // TODO: Use correct provider name when moving to llama.cpp extension
         await this.setDefaultEngineVariant('llama-cpp', {

extensions/engine-management-extension/src/utils.ts

@@ -45,7 +45,9 @@ const os = (settings?: GpuSetting): string => {
  * @param settings
  * @returns
  */
-const cudaVersion = (settings?: GpuSetting): 'cu12.0' | 'cu11.7' | undefined => {
+const cudaVersion = (
+  settings?: GpuSetting
+): 'cu12.0' | 'cu11.7' | undefined => {
   return settings.gpus?.some((gpu) => gpu.version.includes('12'))
     ? 'cu12.0'
     : 'cu11.7'
@@ -70,9 +72,9 @@ export const engineVariant = async (
   const runMode = gpuRunMode(gpuSetting)
   // Only Nvidia GPUs have addition_information set and activated by default
   let engineVariant =
-    !gpuSetting?.vulkan ||
-    !gpuSetting.gpus?.length ||
-    gpuSetting.gpus.some((e) => e.additional_information && e.activated)
+    !gpuSetting?.vulkan &&
+    (!gpuSetting.gpus?.length ||
+      gpuSetting.gpus.some((e) => e.additional_information && e.activated))
       ? [
           platform,
           ...(runMode === RunMode.Cuda

extensions/inference-cortex-extension/resources/default_settings.json

@@ -58,7 +58,7 @@
     "description": "Optimizes memory usage and speeds up model inference using an efficient attention implementation.",
     "controllerType": "checkbox",
     "controllerProps": {
-      "value": true
+      "value": false
     }
   },
   {

web-app/src/constants/localStorage.ts

@@ -10,6 +10,7 @@ export const localStorageKey = {
   settingLocalApiServer: 'setting-local-api-server',
   settingProxyConfig: 'setting-proxy-config',
   settingHardware: 'setting-hardware',
+  settingVulkan: 'setting-vulkan',
   productAnalyticPrompt: 'productAnalyticPrompt',
   productAnalytic: 'productAnalytic',
   toolApproval: 'tool-approval',

web-app/src/hooks/useVulkan.ts

@@ -0,0 +1,34 @@
+import { create } from 'zustand'
+import { persist, createJSONStorage } from 'zustand/middleware'
+import { localStorageKey } from '@/constants/localStorage'
+
+interface VulkanStore {
+  // Vulkan state
+  vulkanEnabled: boolean
+
+  // Update functions
+  setVulkanEnabled: (enabled: boolean) => void
+  toggleVulkan: () => void
+}
+
+export const useVulkan = create<VulkanStore>()(
+  persist(
+    (set) => ({
+      vulkanEnabled: false,
+
+      setVulkanEnabled: (enabled) =>
+        set({
+          vulkanEnabled: enabled,
+        }),
+
+      toggleVulkan: () =>
+        set((state) => ({
+          vulkanEnabled: !state.vulkanEnabled,
+        })),
+    }),
+    {
+      name: localStorageKey.settingVulkan,
+      storage: createJSONStorage(() => localStorage),
+    }
+  )
+)

web-app/src/lib/service.ts

@@ -6,6 +6,7 @@ import {
 } from '@janhq/core'
 import { invoke, InvokeArgs } from '@tauri-apps/api/core'
 import { ExtensionManager } from './extension'
+import { useVulkan } from '@/hooks/useVulkan'
 
 export const AppRoutes = [
   'installExtensions',
@@ -54,7 +55,7 @@ export const systemInformation = async () => {
 
   const gpuSettingInfo = {
     gpus: hardwareInfo.gpus.filter((gpu) => gpu.total_vram > 0),
-    vulkan: false,
+    vulkan: useVulkan.getState().vulkanEnabled,
     cpu: hardwareInfo.cpu,
   }
 

web-app/src/routes/settings/hardware.tsx

@@ -7,6 +7,7 @@ import { Switch } from '@/components/ui/switch'
 import { Progress } from '@/components/ui/progress'
 import { useTranslation } from 'react-i18next'
 import { useHardware } from '@/hooks/useHardware'
+import { useVulkan } from '@/hooks/useVulkan'
 import type { GPU, HardwareData } from '@/hooks/useHardware'
 import { useEffect } from 'react'
 import {
@@ -122,6 +123,7 @@ function Hardware() {
     updateRAMAvailable,
     reorderGPUs,
   } = useHardware()
+  const { vulkanEnabled, setVulkanEnabled } = useVulkan()
 
   useEffect(() => {
     getHardwareInfo().then((data) =>
@@ -337,6 +339,27 @@ function Hardware() {
               />
             </Card>
 
+            {/* Vulkan Settings */}
+            <Card title="Vulkan">
+              <CardItem
+                title="Enable Vulkan"
+                description="Enable Vulkan API for GPU acceleration"
+                actions={
+                  <div className="flex items-center gap-4">
+                    <Switch
+                      checked={vulkanEnabled}
+                      onCheckedChange={(checked) => {
+                        setVulkanEnabled(checked)
+                        setTimeout(() => {
+                          window.location.reload()
+                        }, 500) // Reload after 500ms to apply changes
+                      }}
+                    />
+                  </div>
+                }
+              />
+            </Card>
+
             {/* GPU Information */}
             <Card title="GPUs">
               {hardwareData.gpus.length > 0 ? (