From 19fc399ae13d48831acf89daf0ad215af5764c8e Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Thu, 3 Jul 2025 23:18:38 +0700 Subject: [PATCH] enhancement: gpu list based on backend --- web-app/src/hooks/useHardware.ts | 250 ++++++++++++++++-- web-app/src/routes/settings/hardware.tsx | 151 +++++++++-- .../settings/providers/$providerName.tsx | 13 + 3 files changed, 363 insertions(+), 51 deletions(-) diff --git a/web-app/src/hooks/useHardware.ts b/web-app/src/hooks/useHardware.ts index ac7a7eac2..f59fcd7f5 100644 --- a/web-app/src/hooks/useHardware.ts +++ b/web-app/src/hooks/useHardware.ts @@ -22,6 +22,7 @@ export interface GPU { vendor: string uuid: string driver_version: string + activated?: boolean nvidia_info: { index: number compute_capability: string @@ -99,6 +100,9 @@ interface HardwareStore { // Update entire hardware data at once setHardwareData: (data: HardwareData) => void + // Update hardware data while preserving GPU order + updateHardwareDataPreservingGpuOrder: (data: HardwareData) => void + // Update individual GPU updateGPU: (index: number, gpu: GPU) => void @@ -119,6 +123,12 @@ interface HardwareStore { // Reorder GPUs reorderGPUs: (oldIndex: number, newIndex: number) => void + + // Get activated GPU device string + getActivatedDeviceString: (backendType?: string) => string + + // Update GPU activation states from device string + updateGPUActivationFromDeviceString: (deviceString: string) => void } export const useHardware = create()( @@ -172,7 +182,64 @@ export const useHardware = create()( setHardwareData: (data) => set({ - hardwareData: data, + hardwareData: { + ...data, + gpus: data.gpus.map(gpu => ({ + ...gpu, + activated: gpu.activated ?? false + })) + }, + }), + + updateHardwareDataPreservingGpuOrder: (data) => + set((state) => { + // If we have existing GPU data, preserve the order and activation state + if (state.hardwareData.gpus.length > 0) { + + // Reorder fresh GPU data to match existing order, adding new GPUs at the end + const reorderedGpus: GPU[] = [] + const processedUuids = new Set() + + // First, add existing GPUs in their current order, preserving activation state + state.hardwareData.gpus.forEach(existingGpu => { + const freshGpu = data.gpus.find(gpu => gpu.uuid === existingGpu.uuid) + if (freshGpu) { + reorderedGpus.push({ + ...freshGpu, + activated: existingGpu.activated ?? false + }) + processedUuids.add(freshGpu.uuid) + } + }) + + // Then, add any new GPUs that weren't in the existing order (default to inactive) + data.gpus.forEach(freshGpu => { + if (!processedUuids.has(freshGpu.uuid)) { + reorderedGpus.push({ + ...freshGpu, + activated: false + }) + } + }) + + return { + hardwareData: { + ...data, + gpus: reorderedGpus + } + } + } else { + // No existing GPU data, initialize all GPUs as inactive + return { + hardwareData: { + ...data, + gpus: data.gpus.map(gpu => ({ + ...gpu, + activated: false + })) + } + } + } }), updateGPU: (index, gpu) => @@ -195,35 +262,75 @@ export const useHardware = create()( })), toggleGPUActivation: async (index) => { - const { pausePolling, setGpuLoading } = get() + const { pausePolling, resumePolling, setGpuLoading } = get() pausePolling() setGpuLoading(index, true) - // try { - // await new Promise((resolve) => setTimeout(resolve, 200)) // Simulate async, replace with real API if needed - // set((state) => { - // const newGPUs = [...state.hardwareData.gpus] - // if (index >= 0 && index < newGPUs.length) { - // newGPUs[index] = { - // ...newGPUs[index], - // activated: !newGPUs[index].activated, - // } - // } - // setActiveGpus({ - // gpus: newGPUs - // .filter((e) => e.activated) - // .map((e) => parseInt(e.id)), - // }) - // return { - // hardwareData: { - // ...state.hardwareData, - // gpus: newGPUs, - // }, - // } - // }) - // } finally { - // setGpuLoading(index, false) - // setTimeout(resumePolling, 1000) // Resume polling after 1s - // } + + try { + await new Promise((resolve) => setTimeout(resolve, 200)) // Simulate async operation + + set((state) => { + const newGPUs = [...state.hardwareData.gpus] + if (index >= 0 && index < newGPUs.length) { + newGPUs[index] = { + ...newGPUs[index], + activated: !newGPUs[index].activated, + } + } + + return { + hardwareData: { + ...state.hardwareData, + gpus: newGPUs, + }, + } + }) + + // Update the device setting after state change + const updatedState = get() + + // Import and get backend type + const { useModelProvider } = await import('./useModelProvider') + const { updateProvider, getProviderByName } = useModelProvider.getState() + + const llamacppProvider = getProviderByName('llamacpp') + const backendType = llamacppProvider?.settings.find(s => s.key === 'version_backend')?.controller_props.value as string + + const deviceString = updatedState.getActivatedDeviceString(backendType) + + console.log(`GPU ${index} activation toggled. Backend: "${backendType}", New device string: "${deviceString}"`) + console.log('Activated GPUs:', updatedState.hardwareData.gpus.filter(gpu => gpu.activated).map((gpu, i) => ({ + name: gpu.name, + nvidia: gpu.nvidia_info?.index, + vulkan: gpu.vulkan_info?.index, + activated: gpu.activated + }))) + + if (llamacppProvider) { + const updatedSettings = llamacppProvider.settings.map(setting => { + if (setting.key === 'device') { + return { + ...setting, + controller_props: { + ...setting.controller_props, + value: deviceString + } + } + } + return setting + }) + + updateProvider('llamacpp', { + settings: updatedSettings + }) + + console.log(`Updated llamacpp device setting to: "${deviceString}"`) + } + + } finally { + setGpuLoading(index, false) + setTimeout(resumePolling, 1000) // Resume polling after 1s + } }, reorderGPUs: (oldIndex, newIndex) => @@ -246,6 +353,93 @@ export const useHardware = create()( }, } }), + + getActivatedDeviceString: (backendType?: string) => { + const { hardwareData } = get() + + // Get activated GPUs and generate appropriate device format based on backend + const activatedDevices = hardwareData.gpus + .filter(gpu => gpu.activated) + .map(gpu => { + const isCudaBackend = backendType?.includes('cuda') + const isVulkanBackend = backendType?.includes('vulkan') + + // Handle different backend scenarios + if (isCudaBackend && isVulkanBackend) { + // Mixed backend - prefer CUDA for NVIDIA GPUs, Vulkan for others + if (gpu.nvidia_info) { + return `cuda:${gpu.nvidia_info.index}` + } else if (gpu.vulkan_info) { + return `vulkan:${gpu.vulkan_info.index}` + } + } else if (isCudaBackend && gpu.nvidia_info) { + // CUDA backend - only use CUDA-compatible GPUs + return `cuda:${gpu.nvidia_info.index}` + } else if (isVulkanBackend && gpu.vulkan_info) { + // Vulkan backend - only use Vulkan-compatible GPUs + return `vulkan:${gpu.vulkan_info.index}` + } else if (!backendType) { + // No backend specified, use GPU's preferred type + if (gpu.nvidia_info) { + return `cuda:${gpu.nvidia_info.index}` + } else if (gpu.vulkan_info) { + return `vulkan:${gpu.vulkan_info.index}` + } + } + return null + }) + .filter(device => device !== null) as string[] + + const deviceString = activatedDevices.join(',') + return deviceString + }, + + updateGPUActivationFromDeviceString: (deviceString: string) => { + set((state) => { + const newGPUs = [...state.hardwareData.gpus] + + // Parse device string to get active device indices + const activeDevices = deviceString + .split(',') + .map(device => device.trim()) + .filter(device => device.length > 0) + .map(device => { + const match = device.match(/^(cuda|vulkan):(\d+)$/) + if (match) { + return { + type: match[1] as 'cuda' | 'vulkan', + index: parseInt(match[2]) + } + } + return null + }) + .filter(device => device !== null) as Array<{type: 'cuda' | 'vulkan', index: number}> + + // Update GPU activation states + newGPUs.forEach((gpu, gpuIndex) => { + const shouldBeActive = activeDevices.some(device => { + if (device.type === 'cuda' && gpu.nvidia_info) { + return gpu.nvidia_info.index === device.index + } else if (device.type === 'vulkan' && gpu.vulkan_info) { + return gpu.vulkan_info.index === device.index + } + return false + }) + + newGPUs[gpuIndex] = { + ...gpu, + activated: shouldBeActive + } + }) + + return { + hardwareData: { + ...state.hardwareData, + gpus: newGPUs + } + } + }) + }, }), { name: localStorageKey.settingHardware, diff --git a/web-app/src/routes/settings/hardware.tsx b/web-app/src/routes/settings/hardware.tsx index d9501d8c9..9c174a259 100644 --- a/web-app/src/routes/settings/hardware.tsx +++ b/web-app/src/routes/settings/hardware.tsx @@ -7,9 +7,9 @@ import { Switch } from '@/components/ui/switch' import { Progress } from '@/components/ui/progress' import { useTranslation } from '@/i18n/react-i18next-compat' import { useHardware } from '@/hooks/useHardware' -import { useVulkan } from '@/hooks/useVulkan' +// import { useVulkan } from '@/hooks/useVulkan' import type { GPU, HardwareData } from '@/hooks/useHardware' -import { useEffect } from 'react' +import { useEffect, useState } from 'react' import { DndContext, closestCenter, @@ -34,13 +34,14 @@ import { WebviewWindow } from '@tauri-apps/api/webviewWindow' import { formatMegaBytes } from '@/lib/utils' import { windowKey } from '@/constants/windows' import { toNumber } from '@/utils/number' +import { useModelProvider } from '@/hooks/useModelProvider' // eslint-disable-next-line @typescript-eslint/no-explicit-any export const Route = createFileRoute(route.settings.hardware as any)({ component: Hardware, }) -function SortableGPUItem({ gpu, index }: { gpu: GPU; index: number }) { +function SortableGPUItem({ gpu, index, isCompatible, isActivated }: { gpu: GPU; index: number; isCompatible: boolean; isActivated: boolean }) { const { attributes, listeners, @@ -63,7 +64,7 @@ function SortableGPUItem({ gpu, index }: { gpu: GPU; index: number }) { } return ( -
+
@@ -75,13 +76,18 @@ function SortableGPUItem({ gpu, index }: { gpu: GPU; index: number }) {
{gpu.name} + {!isCompatible && ( + + Incompatible with current backend + + )}
} actions={
toggleGPUActivation(index)} />
@@ -126,17 +132,109 @@ function Hardware() { hardwareData, systemUsage, setHardwareData, + updateHardwareDataPreservingGpuOrder, updateSystemUsage, reorderGPUs, pollingPaused, } = useHardware() - const { vulkanEnabled, setVulkanEnabled } = useVulkan() + // const { vulkanEnabled, setVulkanEnabled } = useVulkan() + + const { providers } = useModelProvider() + const llamacpp = providers.find((p) => p.provider === 'llamacpp') + const versionBackend = llamacpp?.settings.find((s) => s.key === "version_backend")?.controller_props.value + + // Determine backend type and filter GPUs accordingly + const isCudaBackend = typeof versionBackend === 'string' && versionBackend.includes('cuda') + const isVulkanBackend = typeof versionBackend === 'string' && versionBackend.includes('vulkan') + + // Filter and prepare GPUs based on backend + const getFilteredGPUs = () => { + // Always show all GPUs, but compatibility will be determined by isGPUActive + return hardwareData.gpus + } + + const filteredGPUs = getFilteredGPUs() + + // Check if GPU should be active based on backend compatibility + const isGPUCompatible = (gpu: GPU) => { + if (isCudaBackend) { + return gpu.nvidia_info !== null + } else if (isVulkanBackend) { + return gpu.vulkan_info !== null + } else { + // No valid backend - all GPUs are inactive + return false + } + } + + // Check if GPU is actually activated + const isGPUActive = (gpu: GPU) => { + return isGPUCompatible(gpu) && (gpu.activated ?? false) + } useEffect(() => { - getHardwareInfo().then((data) => - setHardwareData(data as unknown as HardwareData) - ) - }, [setHardwareData]) + getHardwareInfo().then((freshData) => { + const data = freshData as unknown as HardwareData + updateHardwareDataPreservingGpuOrder(data) + }) + }, [updateHardwareDataPreservingGpuOrder]) + + // Hardware and provider sync logic + const { getActivatedDeviceString, updateGPUActivationFromDeviceString } = useHardware() + const { updateProvider, getProviderByName } = useModelProvider() + const [isInitialized, setIsInitialized] = useState(false) + + // Initialize GPU activations from device setting on first load + useEffect(() => { + if (hardwareData.gpus.length > 0 && !isInitialized) { + const llamacppProvider = getProviderByName('llamacpp') + const currentDeviceSetting = llamacppProvider?.settings.find(s => s.key === 'device')?.controller_props.value as string + + if (currentDeviceSetting) { + console.log(`Initializing GPU activations from device setting: "${currentDeviceSetting}"`) + updateGPUActivationFromDeviceString(currentDeviceSetting) + } + + setIsInitialized(true) + } + }, [hardwareData.gpus.length, isInitialized, getProviderByName, updateGPUActivationFromDeviceString]) + + // Sync device setting when GPU activations change (only after initialization) + const gpuActivationStates = hardwareData.gpus.map(gpu => gpu.activated) + + useEffect(() => { + if (isInitialized && hardwareData.gpus.length > 0) { + const llamacppProvider = getProviderByName('llamacpp') + const backendType = llamacppProvider?.settings.find(s => s.key === 'version_backend')?.controller_props.value as string + const deviceString = getActivatedDeviceString(backendType) + + if (llamacppProvider) { + const currentDeviceSetting = llamacppProvider.settings.find(s => s.key === 'device') + + // Sync device string when GPU activations change (only after initialization) + if (currentDeviceSetting && currentDeviceSetting.controller_props.value !== deviceString) { + console.log(`Syncing device string from "${currentDeviceSetting.controller_props.value}" to "${deviceString}"`) + + const updatedSettings = llamacppProvider.settings.map(setting => { + if (setting.key === 'device') { + return { + ...setting, + controller_props: { + ...setting.controller_props, + value: deviceString + } + } + } + return setting + }) + + updateProvider('llamacpp', { + settings: updatedSettings + }) + } + } + } + }, [isInitialized, gpuActivationStates, versionBackend, getActivatedDeviceString, updateProvider, getProviderByName, hardwareData.gpus.length]) // Set up DnD sensors const sensors = useSensors( @@ -149,13 +247,12 @@ function Hardware() { const { active, over } = event if (over && active.id !== over.id) { - // Find the indices of the dragged item and the drop target - const oldIndex = hardwareData.gpus.findIndex( - (_, index) => index === active.id - ) - const newIndex = hardwareData.gpus.findIndex( - (_, index) => index === over.id - ) + // Find the actual indices in the original hardwareData.gpus array + const activeGpu = filteredGPUs[active.id as number] + const overGpu = filteredGPUs[over.id as number] + + const oldIndex = hardwareData.gpus.findIndex(gpu => gpu.uuid === activeGpu.uuid) + const newIndex = hardwareData.gpus.findIndex(gpu => gpu.uuid === overGpu.uuid) if (oldIndex !== -1 && newIndex !== -1) { reorderGPUs(oldIndex, newIndex) @@ -356,7 +453,7 @@ function Hardware() { {/* Vulkan Settings */} - {hardwareData.gpus.length > 0 && ( + {/* {hardwareData.gpus.length > 0 && ( - )} + )} */} {/* GPU Information */} {!IS_MACOS ? ( + + {hardwareData.gpus.length > 0 ? ( index)} + items={filteredGPUs.map((_, index) => index)} strategy={verticalListSortingStrategy} > - {hardwareData.gpus.map((gpu, index) => ( - + {filteredGPUs.map((gpu, index) => ( + ))} diff --git a/web-app/src/routes/settings/providers/$providerName.tsx b/web-app/src/routes/settings/providers/$providerName.tsx index 7d27e16be..40761c12a 100644 --- a/web-app/src/routes/settings/providers/$providerName.tsx +++ b/web-app/src/routes/settings/providers/$providerName.tsx @@ -2,6 +2,7 @@ import { Card, CardItem } from '@/containers/Card' import HeaderPage from '@/containers/HeaderPage' import SettingsMenu from '@/containers/SettingsMenu' import { useModelProvider } from '@/hooks/useModelProvider' +import { useHardware } from '@/hooks/useHardware' import { cn, getProviderTitle } from '@/lib/utils' import { open } from '@tauri-apps/plugin-dialog' import { @@ -77,6 +78,7 @@ function ProviderDetail() { const [refreshingModels, setRefreshingModels] = useState(false) const { providerName } = useParams({ from: Route.id }) const { getProviderByName, setProviders, updateProvider } = useModelProvider() + const { updateGPUActivationFromDeviceString } = useHardware() const provider = getProviderByName(providerName) const isSetup = step === 'setup_remote_provider' const navigate = useNavigate() @@ -282,6 +284,17 @@ function ProviderDetail() { ) { updateObj.base_url = newValue } + + // Special handling for device setting changes + if ( + settingKey === 'device' && + typeof newValue === 'string' && + provider.provider === 'llamacpp' + ) { + console.log(`Device setting manually changed to: "${newValue}"`) + updateGPUActivationFromDeviceString(newValue) + } + updateSettings( providerName, updateObj.settings ?? []