enhancement: gpu list based on backend
This commit is contained in:
Faisal Amir
parent
40f1fd4ffd
commit
19fc399ae1
@ -22,6 +22,7 @@ export interface GPU {
|
||||
vendor: string
|
||||
uuid: string
|
||||
driver_version: string
|
||||
activated?: boolean
|
||||
nvidia_info: {
|
||||
index: number
|
||||
compute_capability: string
|
||||
@ -99,6 +100,9 @@ interface HardwareStore {
|
||||
// Update entire hardware data at once
|
||||
setHardwareData: (data: HardwareData) => void
|
||||
|
||||
// Update hardware data while preserving GPU order
|
||||
updateHardwareDataPreservingGpuOrder: (data: HardwareData) => void
|
||||
|
||||
// Update individual GPU
|
||||
updateGPU: (index: number, gpu: GPU) => void
|
||||
|
||||
@ -119,6 +123,12 @@ interface HardwareStore {
|
||||
|
||||
// Reorder GPUs
|
||||
reorderGPUs: (oldIndex: number, newIndex: number) => void
|
||||
|
||||
// Get activated GPU device string
|
||||
getActivatedDeviceString: (backendType?: string) => string
|
||||
|
||||
// Update GPU activation states from device string
|
||||
updateGPUActivationFromDeviceString: (deviceString: string) => void
|
||||
}
|
||||
|
||||
export const useHardware = create<HardwareStore>()(
|
||||
@ -172,7 +182,64 @@ export const useHardware = create<HardwareStore>()(
|
||||
|
||||
setHardwareData: (data) =>
|
||||
set({
|
||||
hardwareData: data,
|
||||
hardwareData: {
|
||||
...data,
|
||||
gpus: data.gpus.map(gpu => ({
|
||||
...gpu,
|
||||
activated: gpu.activated ?? false
|
||||
}))
|
||||
},
|
||||
}),
|
||||
|
||||
updateHardwareDataPreservingGpuOrder: (data) =>
|
||||
set((state) => {
|
||||
// If we have existing GPU data, preserve the order and activation state
|
||||
if (state.hardwareData.gpus.length > 0) {
|
||||
|
||||
// Reorder fresh GPU data to match existing order, adding new GPUs at the end
|
||||
const reorderedGpus: GPU[] = []
|
||||
const processedUuids = new Set()
|
||||
|
||||
// First, add existing GPUs in their current order, preserving activation state
|
||||
state.hardwareData.gpus.forEach(existingGpu => {
|
||||
const freshGpu = data.gpus.find(gpu => gpu.uuid === existingGpu.uuid)
|
||||
if (freshGpu) {
|
||||
reorderedGpus.push({
|
||||
...freshGpu,
|
||||
activated: existingGpu.activated ?? false
|
||||
})
|
||||
processedUuids.add(freshGpu.uuid)
|
||||
}
|
||||
})
|
||||
|
||||
// Then, add any new GPUs that weren't in the existing order (default to inactive)
|
||||
data.gpus.forEach(freshGpu => {
|
||||
if (!processedUuids.has(freshGpu.uuid)) {
|
||||
reorderedGpus.push({
|
||||
...freshGpu,
|
||||
activated: false
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
hardwareData: {
|
||||
...data,
|
||||
gpus: reorderedGpus
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No existing GPU data, initialize all GPUs as inactive
|
||||
return {
|
||||
hardwareData: {
|
||||
...data,
|
||||
gpus: data.gpus.map(gpu => ({
|
||||
...gpu,
|
||||
activated: false
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
|
||||
updateGPU: (index, gpu) =>
|
||||
@ -195,35 +262,75 @@ export const useHardware = create<HardwareStore>()(
|
||||
})),
|
||||
|
||||
toggleGPUActivation: async (index) => {
|
||||
const { pausePolling, setGpuLoading } = get()
|
||||
const { pausePolling, resumePolling, setGpuLoading } = get()
|
||||
pausePolling()
|
||||
setGpuLoading(index, true)
|
||||
// try {
|
||||
// await new Promise((resolve) => setTimeout(resolve, 200)) // Simulate async, replace with real API if needed
|
||||
// set((state) => {
|
||||
// const newGPUs = [...state.hardwareData.gpus]
|
||||
// if (index >= 0 && index < newGPUs.length) {
|
||||
// newGPUs[index] = {
|
||||
// ...newGPUs[index],
|
||||
// activated: !newGPUs[index].activated,
|
||||
// }
|
||||
// }
|
||||
// setActiveGpus({
|
||||
// gpus: newGPUs
|
||||
// .filter((e) => e.activated)
|
||||
// .map((e) => parseInt(e.id)),
|
||||
// })
|
||||
// return {
|
||||
// hardwareData: {
|
||||
// ...state.hardwareData,
|
||||
// gpus: newGPUs,
|
||||
// },
|
||||
// }
|
||||
// })
|
||||
// } finally {
|
||||
// setGpuLoading(index, false)
|
||||
// setTimeout(resumePolling, 1000) // Resume polling after 1s
|
||||
// }
|
||||
|
||||
try {
|
||||
await new Promise((resolve) => setTimeout(resolve, 200)) // Simulate async operation
|
||||
|
||||
set((state) => {
|
||||
const newGPUs = [...state.hardwareData.gpus]
|
||||
if (index >= 0 && index < newGPUs.length) {
|
||||
newGPUs[index] = {
|
||||
...newGPUs[index],
|
||||
activated: !newGPUs[index].activated,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
hardwareData: {
|
||||
...state.hardwareData,
|
||||
gpus: newGPUs,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
// Update the device setting after state change
|
||||
const updatedState = get()
|
||||
|
||||
// Import and get backend type
|
||||
const { useModelProvider } = await import('./useModelProvider')
|
||||
const { updateProvider, getProviderByName } = useModelProvider.getState()
|
||||
|
||||
const llamacppProvider = getProviderByName('llamacpp')
|
||||
const backendType = llamacppProvider?.settings.find(s => s.key === 'version_backend')?.controller_props.value as string
|
||||
|
||||
const deviceString = updatedState.getActivatedDeviceString(backendType)
|
||||
|
||||
console.log(`GPU ${index} activation toggled. Backend: "${backendType}", New device string: "${deviceString}"`)
|
||||
console.log('Activated GPUs:', updatedState.hardwareData.gpus.filter(gpu => gpu.activated).map((gpu, i) => ({
|
||||
name: gpu.name,
|
||||
nvidia: gpu.nvidia_info?.index,
|
||||
vulkan: gpu.vulkan_info?.index,
|
||||
activated: gpu.activated
|
||||
})))
|
||||
|
||||
if (llamacppProvider) {
|
||||
const updatedSettings = llamacppProvider.settings.map(setting => {
|
||||
if (setting.key === 'device') {
|
||||
return {
|
||||
...setting,
|
||||
controller_props: {
|
||||
...setting.controller_props,
|
||||
value: deviceString
|
||||
}
|
||||
}
|
||||
}
|
||||
return setting
|
||||
})
|
||||
|
||||
updateProvider('llamacpp', {
|
||||
settings: updatedSettings
|
||||
})
|
||||
|
||||
console.log(`Updated llamacpp device setting to: "${deviceString}"`)
|
||||
}
|
||||
|
||||
} finally {
|
||||
setGpuLoading(index, false)
|
||||
setTimeout(resumePolling, 1000) // Resume polling after 1s
|
||||
}
|
||||
},
|
||||
|
||||
reorderGPUs: (oldIndex, newIndex) =>
|
||||
@ -246,6 +353,93 @@ export const useHardware = create<HardwareStore>()(
|
||||
},
|
||||
}
|
||||
}),
|
||||
|
||||
getActivatedDeviceString: (backendType?: string) => {
|
||||
const { hardwareData } = get()
|
||||
|
||||
// Get activated GPUs and generate appropriate device format based on backend
|
||||
const activatedDevices = hardwareData.gpus
|
||||
.filter(gpu => gpu.activated)
|
||||
.map(gpu => {
|
||||
const isCudaBackend = backendType?.includes('cuda')
|
||||
const isVulkanBackend = backendType?.includes('vulkan')
|
||||
|
||||
// Handle different backend scenarios
|
||||
if (isCudaBackend && isVulkanBackend) {
|
||||
// Mixed backend - prefer CUDA for NVIDIA GPUs, Vulkan for others
|
||||
if (gpu.nvidia_info) {
|
||||
return `cuda:${gpu.nvidia_info.index}`
|
||||
} else if (gpu.vulkan_info) {
|
||||
return `vulkan:${gpu.vulkan_info.index}`
|
||||
}
|
||||
} else if (isCudaBackend && gpu.nvidia_info) {
|
||||
// CUDA backend - only use CUDA-compatible GPUs
|
||||
return `cuda:${gpu.nvidia_info.index}`
|
||||
} else if (isVulkanBackend && gpu.vulkan_info) {
|
||||
// Vulkan backend - only use Vulkan-compatible GPUs
|
||||
return `vulkan:${gpu.vulkan_info.index}`
|
||||
} else if (!backendType) {
|
||||
// No backend specified, use GPU's preferred type
|
||||
if (gpu.nvidia_info) {
|
||||
return `cuda:${gpu.nvidia_info.index}`
|
||||
} else if (gpu.vulkan_info) {
|
||||
return `vulkan:${gpu.vulkan_info.index}`
|
||||
}
|
||||
}
|
||||
return null
|
||||
})
|
||||
.filter(device => device !== null) as string[]
|
||||
|
||||
const deviceString = activatedDevices.join(',')
|
||||
return deviceString
|
||||
},
|
||||
|
||||
updateGPUActivationFromDeviceString: (deviceString: string) => {
|
||||
set((state) => {
|
||||
const newGPUs = [...state.hardwareData.gpus]
|
||||
|
||||
// Parse device string to get active device indices
|
||||
const activeDevices = deviceString
|
||||
.split(',')
|
||||
.map(device => device.trim())
|
||||
.filter(device => device.length > 0)
|
||||
.map(device => {
|
||||
const match = device.match(/^(cuda|vulkan):(\d+)$/)
|
||||
if (match) {
|
||||
return {
|
||||
type: match[1] as 'cuda' | 'vulkan',
|
||||
index: parseInt(match[2])
|
||||
}
|
||||
}
|
||||
return null
|
||||
})
|
||||
.filter(device => device !== null) as Array<{type: 'cuda' | 'vulkan', index: number}>
|
||||
|
||||
// Update GPU activation states
|
||||
newGPUs.forEach((gpu, gpuIndex) => {
|
||||
const shouldBeActive = activeDevices.some(device => {
|
||||
if (device.type === 'cuda' && gpu.nvidia_info) {
|
||||
return gpu.nvidia_info.index === device.index
|
||||
} else if (device.type === 'vulkan' && gpu.vulkan_info) {
|
||||
return gpu.vulkan_info.index === device.index
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
newGPUs[gpuIndex] = {
|
||||
...gpu,
|
||||
activated: shouldBeActive
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
hardwareData: {
|
||||
...state.hardwareData,
|
||||
gpus: newGPUs
|
||||
}
|
||||
}
|
||||
})
|
||||
},
|
||||
}),
|
||||
{
|
||||
name: localStorageKey.settingHardware,
|
||||
|
||||
@ -7,9 +7,9 @@ import { Switch } from '@/components/ui/switch'
|
||||
import { Progress } from '@/components/ui/progress'
|
||||
import { useTranslation } from '@/i18n/react-i18next-compat'
|
||||
import { useHardware } from '@/hooks/useHardware'
|
||||
import { useVulkan } from '@/hooks/useVulkan'
|
||||
// import { useVulkan } from '@/hooks/useVulkan'
|
||||
import type { GPU, HardwareData } from '@/hooks/useHardware'
|
||||
import { useEffect } from 'react'
|
||||
import { useEffect, useState } from 'react'
|
||||
import {
|
||||
DndContext,
|
||||
closestCenter,
|
||||
@ -34,13 +34,14 @@ import { WebviewWindow } from '@tauri-apps/api/webviewWindow'
|
||||
import { formatMegaBytes } from '@/lib/utils'
|
||||
import { windowKey } from '@/constants/windows'
|
||||
import { toNumber } from '@/utils/number'
|
||||
import { useModelProvider } from '@/hooks/useModelProvider'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export const Route = createFileRoute(route.settings.hardware as any)({
|
||||
component: Hardware,
|
||||
})
|
||||
|
||||
function SortableGPUItem({ gpu, index }: { gpu: GPU; index: number }) {
|
||||
function SortableGPUItem({ gpu, index, isCompatible, isActivated }: { gpu: GPU; index: number; isCompatible: boolean; isActivated: boolean }) {
|
||||
const {
|
||||
attributes,
|
||||
listeners,
|
||||
@ -63,7 +64,7 @@ function SortableGPUItem({ gpu, index }: { gpu: GPU; index: number }) {
|
||||
}
|
||||
|
||||
return (
|
||||
<div ref={setNodeRef} style={style} className="mb-4 last:mb-0">
|
||||
<div ref={setNodeRef} style={style} className={`mb-4 last:mb-0 ${!isCompatible ? 'opacity-60' : ''}`}>
|
||||
<CardItem
|
||||
title={
|
||||
<div className="flex items-center gap-2">
|
||||
@ -75,13 +76,18 @@ function SortableGPUItem({ gpu, index }: { gpu: GPU; index: number }) {
|
||||
<IconGripVertical size={18} className="text-main-view-fg/60" />
|
||||
</div>
|
||||
<span className="text-main-view-fg/80">{gpu.name}</span>
|
||||
{!isCompatible && (
|
||||
<span className="text-xs bg-destructive/10 text-destructive px-2 py-1 rounded-sm">
|
||||
Incompatible with current backend
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
actions={
|
||||
<div className="flex items-center gap-4">
|
||||
<Switch
|
||||
checked={true}
|
||||
disabled={!!gpuLoading[index]}
|
||||
checked={isActivated}
|
||||
disabled={!!gpuLoading[index] || !isCompatible}
|
||||
onCheckedChange={() => toggleGPUActivation(index)}
|
||||
/>
|
||||
</div>
|
||||
@ -126,17 +132,109 @@ function Hardware() {
|
||||
hardwareData,
|
||||
systemUsage,
|
||||
setHardwareData,
|
||||
updateHardwareDataPreservingGpuOrder,
|
||||
updateSystemUsage,
|
||||
reorderGPUs,
|
||||
pollingPaused,
|
||||
} = useHardware()
|
||||
const { vulkanEnabled, setVulkanEnabled } = useVulkan()
|
||||
// const { vulkanEnabled, setVulkanEnabled } = useVulkan()
|
||||
|
||||
const { providers } = useModelProvider()
|
||||
const llamacpp = providers.find((p) => p.provider === 'llamacpp')
|
||||
const versionBackend = llamacpp?.settings.find((s) => s.key === "version_backend")?.controller_props.value
|
||||
|
||||
// Determine backend type and filter GPUs accordingly
|
||||
const isCudaBackend = typeof versionBackend === 'string' && versionBackend.includes('cuda')
|
||||
const isVulkanBackend = typeof versionBackend === 'string' && versionBackend.includes('vulkan')
|
||||
|
||||
// Filter and prepare GPUs based on backend
|
||||
const getFilteredGPUs = () => {
|
||||
// Always show all GPUs, but compatibility will be determined by isGPUActive
|
||||
return hardwareData.gpus
|
||||
}
|
||||
|
||||
const filteredGPUs = getFilteredGPUs()
|
||||
|
||||
// Check if GPU should be active based on backend compatibility
|
||||
const isGPUCompatible = (gpu: GPU) => {
|
||||
if (isCudaBackend) {
|
||||
return gpu.nvidia_info !== null
|
||||
} else if (isVulkanBackend) {
|
||||
return gpu.vulkan_info !== null
|
||||
} else {
|
||||
// No valid backend - all GPUs are inactive
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Check if GPU is actually activated
|
||||
const isGPUActive = (gpu: GPU) => {
|
||||
return isGPUCompatible(gpu) && (gpu.activated ?? false)
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
getHardwareInfo().then((data) =>
|
||||
setHardwareData(data as unknown as HardwareData)
|
||||
)
|
||||
}, [setHardwareData])
|
||||
getHardwareInfo().then((freshData) => {
|
||||
const data = freshData as unknown as HardwareData
|
||||
updateHardwareDataPreservingGpuOrder(data)
|
||||
})
|
||||
}, [updateHardwareDataPreservingGpuOrder])
|
||||
|
||||
// Hardware and provider sync logic
|
||||
const { getActivatedDeviceString, updateGPUActivationFromDeviceString } = useHardware()
|
||||
const { updateProvider, getProviderByName } = useModelProvider()
|
||||
const [isInitialized, setIsInitialized] = useState(false)
|
||||
|
||||
// Initialize GPU activations from device setting on first load
|
||||
useEffect(() => {
|
||||
if (hardwareData.gpus.length > 0 && !isInitialized) {
|
||||
const llamacppProvider = getProviderByName('llamacpp')
|
||||
const currentDeviceSetting = llamacppProvider?.settings.find(s => s.key === 'device')?.controller_props.value as string
|
||||
|
||||
if (currentDeviceSetting) {
|
||||
console.log(`Initializing GPU activations from device setting: "${currentDeviceSetting}"`)
|
||||
updateGPUActivationFromDeviceString(currentDeviceSetting)
|
||||
}
|
||||
|
||||
setIsInitialized(true)
|
||||
}
|
||||
}, [hardwareData.gpus.length, isInitialized, getProviderByName, updateGPUActivationFromDeviceString])
|
||||
|
||||
// Sync device setting when GPU activations change (only after initialization)
|
||||
const gpuActivationStates = hardwareData.gpus.map(gpu => gpu.activated)
|
||||
|
||||
useEffect(() => {
|
||||
if (isInitialized && hardwareData.gpus.length > 0) {
|
||||
const llamacppProvider = getProviderByName('llamacpp')
|
||||
const backendType = llamacppProvider?.settings.find(s => s.key === 'version_backend')?.controller_props.value as string
|
||||
const deviceString = getActivatedDeviceString(backendType)
|
||||
|
||||
if (llamacppProvider) {
|
||||
const currentDeviceSetting = llamacppProvider.settings.find(s => s.key === 'device')
|
||||
|
||||
// Sync device string when GPU activations change (only after initialization)
|
||||
if (currentDeviceSetting && currentDeviceSetting.controller_props.value !== deviceString) {
|
||||
console.log(`Syncing device string from "${currentDeviceSetting.controller_props.value}" to "${deviceString}"`)
|
||||
|
||||
const updatedSettings = llamacppProvider.settings.map(setting => {
|
||||
if (setting.key === 'device') {
|
||||
return {
|
||||
...setting,
|
||||
controller_props: {
|
||||
...setting.controller_props,
|
||||
value: deviceString
|
||||
}
|
||||
}
|
||||
}
|
||||
return setting
|
||||
})
|
||||
|
||||
updateProvider('llamacpp', {
|
||||
settings: updatedSettings
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [isInitialized, gpuActivationStates, versionBackend, getActivatedDeviceString, updateProvider, getProviderByName, hardwareData.gpus.length])
|
||||
|
||||
// Set up DnD sensors
|
||||
const sensors = useSensors(
|
||||
@ -149,13 +247,12 @@ function Hardware() {
|
||||
const { active, over } = event
|
||||
|
||||
if (over && active.id !== over.id) {
|
||||
// Find the indices of the dragged item and the drop target
|
||||
const oldIndex = hardwareData.gpus.findIndex(
|
||||
(_, index) => index === active.id
|
||||
)
|
||||
const newIndex = hardwareData.gpus.findIndex(
|
||||
(_, index) => index === over.id
|
||||
)
|
||||
// Find the actual indices in the original hardwareData.gpus array
|
||||
const activeGpu = filteredGPUs[active.id as number]
|
||||
const overGpu = filteredGPUs[over.id as number]
|
||||
|
||||
const oldIndex = hardwareData.gpus.findIndex(gpu => gpu.uuid === activeGpu.uuid)
|
||||
const newIndex = hardwareData.gpus.findIndex(gpu => gpu.uuid === overGpu.uuid)
|
||||
|
||||
if (oldIndex !== -1 && newIndex !== -1) {
|
||||
reorderGPUs(oldIndex, newIndex)
|
||||
@ -356,7 +453,7 @@ function Hardware() {
|
||||
</Card>
|
||||
|
||||
{/* Vulkan Settings */}
|
||||
{hardwareData.gpus.length > 0 && (
|
||||
{/* {hardwareData.gpus.length > 0 && (
|
||||
<Card title={t('settings:hardware.vulkan')}>
|
||||
<CardItem
|
||||
title={t('settings:hardware.enableVulkan')}
|
||||
@ -376,11 +473,13 @@ function Hardware() {
|
||||
}
|
||||
/>
|
||||
</Card>
|
||||
)}
|
||||
)} */}
|
||||
|
||||
{/* GPU Information */}
|
||||
{!IS_MACOS ? (
|
||||
<Card title={t('settings:hardware.gpus')}>
|
||||
|
||||
|
||||
{hardwareData.gpus.length > 0 ? (
|
||||
<DndContext
|
||||
sensors={sensors}
|
||||
@ -388,11 +487,17 @@ function Hardware() {
|
||||
onDragEnd={handleDragEnd}
|
||||
>
|
||||
<SortableContext
|
||||
items={hardwareData.gpus.map((_, index) => index)}
|
||||
items={filteredGPUs.map((_, index) => index)}
|
||||
strategy={verticalListSortingStrategy}
|
||||
>
|
||||
{hardwareData.gpus.map((gpu, index) => (
|
||||
<SortableGPUItem key={index} gpu={gpu} index={index} />
|
||||
{filteredGPUs.map((gpu, index) => (
|
||||
<SortableGPUItem
|
||||
key={index}
|
||||
gpu={gpu}
|
||||
index={index}
|
||||
isCompatible={isGPUCompatible(gpu)}
|
||||
isActivated={isGPUActive(gpu)}
|
||||
/>
|
||||
))}
|
||||
</SortableContext>
|
||||
</DndContext>
|
||||
|
||||
@ -2,6 +2,7 @@ import { Card, CardItem } from '@/containers/Card'
|
||||
import HeaderPage from '@/containers/HeaderPage'
|
||||
import SettingsMenu from '@/containers/SettingsMenu'
|
||||
import { useModelProvider } from '@/hooks/useModelProvider'
|
||||
import { useHardware } from '@/hooks/useHardware'
|
||||
import { cn, getProviderTitle } from '@/lib/utils'
|
||||
import { open } from '@tauri-apps/plugin-dialog'
|
||||
import {
|
||||
@ -77,6 +78,7 @@ function ProviderDetail() {
|
||||
const [refreshingModels, setRefreshingModels] = useState(false)
|
||||
const { providerName } = useParams({ from: Route.id })
|
||||
const { getProviderByName, setProviders, updateProvider } = useModelProvider()
|
||||
const { updateGPUActivationFromDeviceString } = useHardware()
|
||||
const provider = getProviderByName(providerName)
|
||||
const isSetup = step === 'setup_remote_provider'
|
||||
const navigate = useNavigate()
|
||||
@ -282,6 +284,17 @@ function ProviderDetail() {
|
||||
) {
|
||||
updateObj.base_url = newValue
|
||||
}
|
||||
|
||||
// Special handling for device setting changes
|
||||
if (
|
||||
settingKey === 'device' &&
|
||||
typeof newValue === 'string' &&
|
||||
provider.provider === 'llamacpp'
|
||||
) {
|
||||
console.log(`Device setting manually changed to: "${newValue}"`)
|
||||
updateGPUActivationFromDeviceString(newValue)
|
||||
}
|
||||
|
||||
updateSettings(
|
||||
providerName,
|
||||
updateObj.settings ?? []
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user