* feat: Add model compatibility check and memory estimation This commit introduces a new feature to check if a given model is supported based on available device memory. The change includes: - A new `estimateKVCache` method that calculates the required memory for the model's KV cache. It uses GGUF metadata such as `block_count`, `head_count`, `key_length`, and `value_length` to perform the calculation. - An `isModelSupported` method that combines the model file size and the estimated KV cache size to determine the total memory required. It then checks if any available device has sufficient free memory to load the model. - An updated error message for the `version_backend` check to be more user-friendly, suggesting a stable internet connection as a potential solution for backend setup failures. This functionality helps prevent the application from attempting to load models that would exceed the device's memory capacity, leading to more stable and predictable behavior. fixes: #5505 * Update extensions/llamacpp-extension/src/index.ts Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * Update extensions/llamacpp-extension/src/index.ts Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * Extend this to available system RAM if GGML device is not available * fix: Improve model metadata and memory checks This commit refactors the logic for checking if a model is supported by a system's available memory. **Key changes:** - **Remote model support**: The `read_gguf_metadata` function can now fetch metadata from a remote URL by reading the file in chunks. - **Improved KV cache size calculation**: The KV cache size is now estimated more accurately by using `attention.key_length` and `attention.value_length` from the GGUF metadata, with a fallback to `embedding_length`. - **Granular memory check statuses**: The `isModelSupported` function now returns a more specific status (`'RED'`, `'YELLOW'`, `'GREEN'`) to indicate whether the model weights or the KV cache are too large for the available memory. - **Consolidated logic**: The logic for checking local and remote models has been consolidated into a single `isModelSupported` function, improving code clarity and maintainability. These changes provide more robust and informative model compatibility checks, especially for models hosted on remote servers. * Update extensions/llamacpp-extension/src/index.ts Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * Make ctx_size optional and use sum free memory across ggml devices * feat: hub and dropdown model selection handle model compatibility * feat: update bage model info color * chore: enable detail page to get compatibility model * chore: update copy * chore: update shrink indicator UI --------- Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> Co-authored-by: Faisal Amir <urmauur@gmail.com>
227 lines
7.6 KiB
TypeScript
227 lines
7.6 KiB
TypeScript
import {
|
|
HoverCard,
|
|
HoverCardContent,
|
|
HoverCardTrigger,
|
|
} from '@/components/ui/hover-card'
|
|
import { IconInfoCircle } from '@tabler/icons-react'
|
|
import { CatalogModel, ModelQuant } from '@/services/models'
|
|
import { extractDescription } from '@/lib/models'
|
|
|
|
interface ModelInfoHoverCardProps {
|
|
model: CatalogModel
|
|
variant?: ModelQuant
|
|
defaultModelQuantizations: string[]
|
|
modelSupportStatus: Record<string, string>
|
|
onCheckModelSupport: (variant: ModelQuant) => void
|
|
children?: React.ReactNode
|
|
}
|
|
|
|
export const ModelInfoHoverCard = ({
|
|
model,
|
|
variant,
|
|
defaultModelQuantizations,
|
|
modelSupportStatus,
|
|
onCheckModelSupport,
|
|
children,
|
|
}: ModelInfoHoverCardProps) => {
|
|
const isVariantMode = !!variant
|
|
const displayVariant =
|
|
variant ||
|
|
model.quants.find((m) =>
|
|
defaultModelQuantizations.some((e) =>
|
|
m.model_id.toLowerCase().includes(e)
|
|
)
|
|
) ||
|
|
model.quants?.[0]
|
|
|
|
const handleMouseEnter = () => {
|
|
if (displayVariant) {
|
|
onCheckModelSupport(displayVariant)
|
|
}
|
|
}
|
|
|
|
const getCompatibilityStatus = () => {
|
|
const status = displayVariant
|
|
? modelSupportStatus[displayVariant.model_id]
|
|
: null
|
|
|
|
if (status === 'LOADING') {
|
|
return (
|
|
<div className="flex items-start gap-2">
|
|
<div className="size-2 shrink-0 border border-main-view-fg/50 border-t-transparent rounded-full animate-spin mt-1"></div>
|
|
<span className="text-main-view-fg/50">Checking...</span>
|
|
</div>
|
|
)
|
|
} else if (status === 'GREEN') {
|
|
return (
|
|
<div className="flex items-start gap-2">
|
|
<div className="size-2 shrink-0 bg-green-500 rounded-full mt-1"></div>
|
|
<span className="text-green-500 font-medium">
|
|
Recommended for your device
|
|
</span>
|
|
</div>
|
|
)
|
|
} else if (status === 'YELLOW') {
|
|
return (
|
|
<div className="flex items-start gap-2">
|
|
<div className="size-2 shrink-0 bg-yellow-500 rounded-full mt-1"></div>
|
|
<span className="text-yellow-500 font-medium">
|
|
May be slow on your device
|
|
</span>
|
|
</div>
|
|
)
|
|
} else if (status === 'RED') {
|
|
return (
|
|
<div className="flex items-start gap-2">
|
|
<div className="size-2 shrink-0 bg-red-500 rounded-full mt-1"></div>
|
|
<span className="text-red-500 font-medium">
|
|
May be incompatible with your device
|
|
</span>
|
|
</div>
|
|
)
|
|
} else {
|
|
return (
|
|
<div className="flex items-start gap-2">
|
|
<div className="size-2 shrink-0 bg-gray-400 rounded-full mt-1"></div>
|
|
<span className="text-gray-500">Unknown</span>
|
|
</div>
|
|
)
|
|
}
|
|
}
|
|
|
|
return (
|
|
<HoverCard>
|
|
<HoverCardTrigger asChild onMouseEnter={handleMouseEnter}>
|
|
{children || (
|
|
<div className="cursor-pointer">
|
|
<IconInfoCircle
|
|
size={14}
|
|
className="mt-0.5 text-main-view-fg/50 hover:text-main-view-fg/80 transition-colors"
|
|
/>
|
|
</div>
|
|
)}
|
|
</HoverCardTrigger>
|
|
<HoverCardContent className="w-96 p-4" side="left">
|
|
<div className="space-y-4">
|
|
{/* Header */}
|
|
<div className="border-b border-main-view-fg/10 pb-3">
|
|
<h4 className="text-sm font-semibold text-main-view-fg">
|
|
{isVariantMode ? variant.model_id : model.model_name}
|
|
</h4>
|
|
<p className="text-xs text-main-view-fg/60 mt-1">
|
|
{isVariantMode
|
|
? 'Model Variant Information'
|
|
: 'Model Information'}
|
|
</p>
|
|
</div>
|
|
|
|
{/* Main Info Grid */}
|
|
<div className="grid grid-cols-2 gap-3 text-xs">
|
|
<div className="space-y-2">
|
|
{isVariantMode ? (
|
|
<>
|
|
<div>
|
|
<span className="text-main-view-fg/50 block">
|
|
File Size
|
|
</span>
|
|
<span className="text-main-view-fg font-medium mt-1 inline-block">
|
|
{variant.file_size}
|
|
</span>
|
|
</div>
|
|
<div>
|
|
<span className="text-main-view-fg/50 block">
|
|
Quantization
|
|
</span>
|
|
<span className="text-main-view-fg font-medium mt-1 inline-block">
|
|
{variant.model_id.split('-').pop()?.toUpperCase() ||
|
|
'N/A'}
|
|
</span>
|
|
</div>
|
|
</>
|
|
) : (
|
|
<>
|
|
<div>
|
|
<span className="text-main-view-fg/50 block">
|
|
Downloads
|
|
</span>
|
|
<span className="text-main-view-fg font-medium mt-1 inline-block">
|
|
{model.downloads?.toLocaleString() || '0'}
|
|
</span>
|
|
</div>
|
|
<div>
|
|
<span className="text-main-view-fg/50 block">Variants</span>
|
|
<span className="text-main-view-fg font-medium mt-1 inline-block">
|
|
{model.quants?.length || 0}
|
|
</span>
|
|
</div>
|
|
</>
|
|
)}
|
|
</div>
|
|
|
|
<div className="space-y-2">
|
|
{!isVariantMode && (
|
|
<div>
|
|
<span className="text-main-view-fg/50 block">
|
|
Default Size
|
|
</span>
|
|
<span className="text-main-view-fg font-medium mt-1 inline-block">
|
|
{displayVariant?.file_size || 'N/A'}
|
|
</span>
|
|
</div>
|
|
)}
|
|
<div>
|
|
<span className="text-main-view-fg/50 block">
|
|
Compatibility
|
|
</span>
|
|
<div className="flex items-center gap-1.5 mt-1">
|
|
{getCompatibilityStatus()}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Features Section */}
|
|
{(model.num_mmproj > 0 || model.tools) && (
|
|
<div className="border-t border-main-view-fg/10 pt-3">
|
|
<h5 className="text-xs font-medium text-main-view-fg/70 mb-2">
|
|
Features
|
|
</h5>
|
|
<div className="flex flex-wrap gap-2">
|
|
{model.num_mmproj > 0 && (
|
|
<div className="flex items-center gap-1.5 px-2 py-1 bg-main-view-fg/10 rounded-md">
|
|
<span className="text-xs text-main-view-fg font-medium">
|
|
Vision
|
|
</span>
|
|
</div>
|
|
)}
|
|
{model.tools && (
|
|
<div className="flex items-center gap-1.5 px-2 py-1 bg-main-view-fg/10 rounded-md">
|
|
<span className="text-xs text-main-view-fg font-medium">
|
|
Tools
|
|
</span>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* Content Section */}
|
|
<div className="border-t border-main-view-fg/10 pt-3">
|
|
<h5 className="text-xs font-medium text-main-view-fg/70 mb-1">
|
|
{isVariantMode ? 'Download URL' : 'Description'}
|
|
</h5>
|
|
<div className="text-xs text-main-view-fg/60 bg-main-view-fg/5 rounded p-2">
|
|
{isVariantMode ? (
|
|
<div className="font-mono break-all">{variant.path}</div>
|
|
) : (
|
|
extractDescription(model?.description) ||
|
|
'No description available'
|
|
)}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</HoverCardContent>
|
|
</HoverCard>
|
|
)
|
|
}
|