feat: Add GGUF metadata reading functionality (#6120)
* feat: Add GGUF metadata reading functionality This commit introduces a new Tauri command and a corresponding function to read metadata from GGUF model files. The new read_gguf_metadata command in the Rust backend uses the byteorder crate to parse the GGUF file format and extract key metadata. This information, including the file's version, tensor count, and a key-value map of other metadata, is then made available to the TypeScript frontend. This functionality is a foundational step toward providing users with more detailed information about their loaded models directly within the application. This will be refactored later. fixes: #6001 * loadMetadata() should return * Properly throw eror to FE * Use BufReader to improve performance
This commit is contained in:
Akarshan Biswas
GitHub
parent
da31675f64
commit
f4661912b0
@ -100,6 +100,13 @@ interface DeviceList {
|
||||
mem: number
|
||||
free: number
|
||||
}
|
||||
|
||||
interface GgufMetadata {
|
||||
version: number
|
||||
tensor_count: number
|
||||
metadata: Record<string, string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Override the default app.log function to use Jan's logging system.
|
||||
* @param args
|
||||
@ -1591,4 +1598,15 @@ export default class llamacpp_extension extends AIEngine {
|
||||
override getChatClient(sessionId: string): any {
|
||||
throw new Error('method not implemented yet')
|
||||
}
|
||||
|
||||
private async loadMetadata(path: string): Promise<GgufMetadata> {
|
||||
try {
|
||||
const data = await invoke<GgufMetadata>('read_gguf_metadata', {
|
||||
path: path,
|
||||
})
|
||||
return data
|
||||
} catch (err) {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
39
src-tauri/Cargo.lock
generated
39
src-tauri/Cargo.lock
generated
@ -8,6 +8,7 @@ version = "0.6.599"
|
||||
dependencies = [
|
||||
"ash",
|
||||
"base64 0.22.1",
|
||||
"byteorder",
|
||||
"dirs",
|
||||
"env",
|
||||
"fix-path-env",
|
||||
@ -86,21 +87,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloc-no-stdlib"
|
||||
version = "2.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
|
||||
|
||||
[[package]]
|
||||
name = "alloc-stdlib"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android-tzdata"
|
||||
version = "0.1.1"
|
||||
@ -467,27 +453,6 @@ dependencies = [
|
||||
"syn 2.0.104",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
"brotli-decompressor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.19.0"
|
||||
@ -5006,7 +4971,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b54a99a6cd8e01abcfa61508177e6096a4fe2681efecee9214e962f2f073ae4a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"brotli",
|
||||
"ico",
|
||||
"json-patch",
|
||||
"plist",
|
||||
@ -5344,7 +5308,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330c15cabfe1d9f213478c9e8ec2b0c76dab26bb6f314b8ad1c8a568c1d186e"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"brotli",
|
||||
"cargo_metadata",
|
||||
"ctor",
|
||||
"dunce",
|
||||
|
||||
@ -74,6 +74,7 @@ sha2 = "0.10.9"
|
||||
base64 = "0.22.1"
|
||||
libloading = "0.8.7"
|
||||
thiserror = "2.0.12"
|
||||
byteorder = "1.5.0"
|
||||
|
||||
[dependencies.tauri]
|
||||
version = "2.5.0"
|
||||
|
||||
@ -0,0 +1,213 @@
|
||||
use byteorder::{LittleEndian, ReadBytesExt};
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
convert::TryFrom,
|
||||
fs::File,
|
||||
io::{self, Read, Seek, BufReader},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[repr(u32)]
|
||||
enum GgufValueType {
|
||||
Uint8 = 0,
|
||||
Int8 = 1,
|
||||
Uint16 = 2,
|
||||
Int16 = 3,
|
||||
Uint32 = 4,
|
||||
Int32 = 5,
|
||||
Float32 = 6,
|
||||
Bool = 7,
|
||||
String = 8,
|
||||
Array = 9,
|
||||
Uint64 = 10,
|
||||
Int64 = 11,
|
||||
Float64 = 12,
|
||||
}
|
||||
|
||||
impl TryFrom<u32> for GgufValueType {
|
||||
type Error = io::Error;
|
||||
fn try_from(value: u32) -> Result<Self, Self::Error> {
|
||||
match value {
|
||||
0 => Ok(Self::Uint8),
|
||||
1 => Ok(Self::Int8),
|
||||
2 => Ok(Self::Uint16),
|
||||
3 => Ok(Self::Int16),
|
||||
4 => Ok(Self::Uint32),
|
||||
5 => Ok(Self::Int32),
|
||||
6 => Ok(Self::Float32),
|
||||
7 => Ok(Self::Bool),
|
||||
8 => Ok(Self::String),
|
||||
9 => Ok(Self::Array),
|
||||
10 => Ok(Self::Uint64),
|
||||
11 => Ok(Self::Int64),
|
||||
12 => Ok(Self::Float64),
|
||||
_ => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("Unknown GGUF value type: {}", value),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct GgufMetadata {
|
||||
version: u32,
|
||||
tensor_count: u64,
|
||||
metadata: HashMap<String, String>,
|
||||
}
|
||||
|
||||
fn read_gguf_metadata_internal<P: AsRef<Path>>(path: P) -> io::Result<GgufMetadata> {
|
||||
let mut file = BufReader::new(File::open(path)?);
|
||||
|
||||
let mut magic = [0u8; 4];
|
||||
file.read_exact(&mut magic)?;
|
||||
if &magic != b"GGUF" {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "Not a GGUF file"));
|
||||
}
|
||||
|
||||
let version = file.read_u32::<LittleEndian>()?;
|
||||
let tensor_count = file.read_u64::<LittleEndian>()?;
|
||||
let metadata_count = file.read_u64::<LittleEndian>()?;
|
||||
|
||||
let mut metadata_map = HashMap::new();
|
||||
for i in 0..metadata_count {
|
||||
match read_metadata_entry(&mut file, i) {
|
||||
Ok((key, value)) => {
|
||||
metadata_map.insert(key, value);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("Error reading metadata entry {}: {}", i, e),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(GgufMetadata {
|
||||
version,
|
||||
tensor_count,
|
||||
metadata: metadata_map,
|
||||
})
|
||||
}
|
||||
|
||||
fn read_metadata_entry<R: Read + Seek>(reader: &mut R, index: u64) -> io::Result<(String, String)> {
|
||||
let key = read_gguf_string(reader).map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("Failed to read key for metadata entry {}: {}", index, e),
|
||||
)
|
||||
})?;
|
||||
|
||||
let value_type_u32 = reader.read_u32::<LittleEndian>()?;
|
||||
let value_type = GgufValueType::try_from(value_type_u32)?;
|
||||
let value = read_gguf_value(reader, value_type)?;
|
||||
|
||||
Ok((key, value))
|
||||
}
|
||||
|
||||
fn read_gguf_string<R: Read>(reader: &mut R) -> io::Result<String> {
|
||||
let len = reader.read_u64::<LittleEndian>()?;
|
||||
if len > (1024 * 1024) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("String length {} is unreasonably large", len),
|
||||
));
|
||||
}
|
||||
let mut buf = vec![0u8; len as usize];
|
||||
reader.read_exact(&mut buf)?;
|
||||
Ok(String::from_utf8(buf).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?)
|
||||
}
|
||||
|
||||
fn read_gguf_value<R: Read + Seek>(
|
||||
reader: &mut R,
|
||||
value_type: GgufValueType,
|
||||
) -> io::Result<String> {
|
||||
match value_type {
|
||||
GgufValueType::Uint8 => Ok(reader.read_u8()?.to_string()),
|
||||
GgufValueType::Int8 => Ok(reader.read_i8()?.to_string()),
|
||||
GgufValueType::Uint16 => Ok(reader.read_u16::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Int16 => Ok(reader.read_i16::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Uint32 => Ok(reader.read_u32::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Int32 => Ok(reader.read_i32::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Float32 => Ok(reader.read_f32::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Bool => Ok((reader.read_u8()? != 0).to_string()),
|
||||
GgufValueType::String => read_gguf_string(reader),
|
||||
GgufValueType::Uint64 => Ok(reader.read_u64::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Int64 => Ok(reader.read_i64::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Float64 => Ok(reader.read_f64::<LittleEndian>()?.to_string()),
|
||||
GgufValueType::Array => {
|
||||
let elem_type_u32 = reader.read_u32::<LittleEndian>()?;
|
||||
let elem_type = GgufValueType::try_from(elem_type_u32)?;
|
||||
let len = reader.read_u64::<LittleEndian>()?;
|
||||
|
||||
if len > 1_000_000 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("Array length {} is unreasonably large", len),
|
||||
));
|
||||
}
|
||||
|
||||
if len > 24 {
|
||||
skip_array_data(reader, elem_type, len)?;
|
||||
return Ok(format!(
|
||||
"<Array of type {:?} with {} elements, data skipped>",
|
||||
elem_type, len
|
||||
));
|
||||
}
|
||||
|
||||
let mut elems = Vec::with_capacity(len as usize);
|
||||
for _ in 0..len {
|
||||
elems.push(read_gguf_value(reader, elem_type)?);
|
||||
}
|
||||
Ok(format!("[{}]", elems.join(", ")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn skip_array_data<R: Read + Seek>(
|
||||
reader: &mut R,
|
||||
elem_type: GgufValueType,
|
||||
len: u64,
|
||||
) -> io::Result<()> {
|
||||
match elem_type {
|
||||
GgufValueType::Uint8 | GgufValueType::Int8 | GgufValueType::Bool => {
|
||||
reader.seek(io::SeekFrom::Current(len as i64))?;
|
||||
}
|
||||
GgufValueType::Uint16 | GgufValueType::Int16 => {
|
||||
reader.seek(io::SeekFrom::Current((len * 2) as i64))?;
|
||||
}
|
||||
GgufValueType::Uint32 | GgufValueType::Int32 | GgufValueType::Float32 => {
|
||||
reader.seek(io::SeekFrom::Current((len * 4) as i64))?;
|
||||
}
|
||||
GgufValueType::Uint64 | GgufValueType::Int64 | GgufValueType::Float64 => {
|
||||
reader.seek(io::SeekFrom::Current((len * 8) as i64))?;
|
||||
}
|
||||
GgufValueType::String => {
|
||||
for _ in 0..len {
|
||||
let str_len = reader.read_u64::<LittleEndian>()?;
|
||||
reader.seek(io::SeekFrom::Current(str_len as i64))?;
|
||||
}
|
||||
}
|
||||
GgufValueType::Array => {
|
||||
for _ in 0..len {
|
||||
read_gguf_value(reader, elem_type)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn read_gguf_metadata(path: String) -> Result<GgufMetadata, String> {
|
||||
// run the blocking code in a separate thread pool
|
||||
tauri::async_runtime::spawn_blocking(move || {
|
||||
read_gguf_metadata_internal(path)
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
@ -1,2 +1,3 @@
|
||||
pub mod server;
|
||||
pub mod cleanup;
|
||||
pub mod gguf;
|
||||
|
||||
@ -102,6 +102,7 @@ pub fn run() {
|
||||
core::utils::extensions::inference_llamacpp_extension::server::get_loaded_models,
|
||||
core::utils::extensions::inference_llamacpp_extension::server::generate_api_key,
|
||||
core::utils::extensions::inference_llamacpp_extension::server::is_process_running,
|
||||
core::utils::extensions::inference_llamacpp_extension::gguf::read_gguf_metadata,
|
||||
])
|
||||
.manage(AppState {
|
||||
app_token: Some(generate_app_token()),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user