From f4661912b07f24c194265ea62904d214462f244d Mon Sep 17 00:00:00 2001 From: Akarshan Biswas Date: Wed, 13 Aug 2025 22:54:20 +0530 Subject: [PATCH] feat: Add GGUF metadata reading functionality (#6120) * feat: Add GGUF metadata reading functionality This commit introduces a new Tauri command and a corresponding function to read metadata from GGUF model files. The new read_gguf_metadata command in the Rust backend uses the byteorder crate to parse the GGUF file format and extract key metadata. This information, including the file's version, tensor count, and a key-value map of other metadata, is then made available to the TypeScript frontend. This functionality is a foundational step toward providing users with more detailed information about their loaded models directly within the application. This will be refactored later. fixes: #6001 * loadMetadata() should return * Properly throw eror to FE * Use BufReader to improve performance --- extensions/llamacpp-extension/src/index.ts | 18 ++ src-tauri/Cargo.lock | 39 +--- src-tauri/Cargo.toml | 1 + .../inference_llamacpp_extension/gguf.rs | 213 ++++++++++++++++++ .../inference_llamacpp_extension/mod.rs | 1 + src-tauri/src/lib.rs | 1 + 6 files changed, 235 insertions(+), 38 deletions(-) create mode 100644 src-tauri/src/core/utils/extensions/inference_llamacpp_extension/gguf.rs diff --git a/extensions/llamacpp-extension/src/index.ts b/extensions/llamacpp-extension/src/index.ts index 75afb81ae..903654bea 100644 --- a/extensions/llamacpp-extension/src/index.ts +++ b/extensions/llamacpp-extension/src/index.ts @@ -100,6 +100,13 @@ interface DeviceList { mem: number free: number } + +interface GgufMetadata { + version: number + tensor_count: number + metadata: Record +} + /** * Override the default app.log function to use Jan's logging system. * @param args @@ -1591,4 +1598,15 @@ export default class llamacpp_extension extends AIEngine { override getChatClient(sessionId: string): any { throw new Error('method not implemented yet') } + + private async loadMetadata(path: string): Promise { + try { + const data = await invoke('read_gguf_metadata', { + path: path, + }) + return data + } catch (err) { + throw err + } + } } diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 6f2b6dbda..9d1e813c1 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -8,6 +8,7 @@ version = "0.6.599" dependencies = [ "ash", "base64 0.22.1", + "byteorder", "dirs", "env", "fix-path-env", @@ -86,21 +87,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "android-tzdata" version = "0.1.1" @@ -467,27 +453,6 @@ dependencies = [ "syn 2.0.104", ] -[[package]] -name = "brotli" -version = "8.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bumpalo" version = "3.19.0" @@ -5006,7 +4971,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b54a99a6cd8e01abcfa61508177e6096a4fe2681efecee9214e962f2f073ae4a" dependencies = [ "base64 0.22.1", - "brotli", "ico", "json-patch", "plist", @@ -5344,7 +5308,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330c15cabfe1d9f213478c9e8ec2b0c76dab26bb6f314b8ad1c8a568c1d186e" dependencies = [ "anyhow", - "brotli", "cargo_metadata", "ctor", "dunce", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index bcdedcc8a..4080ca861 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -74,6 +74,7 @@ sha2 = "0.10.9" base64 = "0.22.1" libloading = "0.8.7" thiserror = "2.0.12" +byteorder = "1.5.0" [dependencies.tauri] version = "2.5.0" diff --git a/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/gguf.rs b/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/gguf.rs new file mode 100644 index 000000000..c040c5e67 --- /dev/null +++ b/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/gguf.rs @@ -0,0 +1,213 @@ +use byteorder::{LittleEndian, ReadBytesExt}; +use serde::Serialize; +use std::{ + collections::HashMap, + convert::TryFrom, + fs::File, + io::{self, Read, Seek, BufReader}, + path::Path, +}; + +#[derive(Debug, Clone, Copy)] +#[repr(u32)] +enum GgufValueType { + Uint8 = 0, + Int8 = 1, + Uint16 = 2, + Int16 = 3, + Uint32 = 4, + Int32 = 5, + Float32 = 6, + Bool = 7, + String = 8, + Array = 9, + Uint64 = 10, + Int64 = 11, + Float64 = 12, +} + +impl TryFrom for GgufValueType { + type Error = io::Error; + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(Self::Uint8), + 1 => Ok(Self::Int8), + 2 => Ok(Self::Uint16), + 3 => Ok(Self::Int16), + 4 => Ok(Self::Uint32), + 5 => Ok(Self::Int32), + 6 => Ok(Self::Float32), + 7 => Ok(Self::Bool), + 8 => Ok(Self::String), + 9 => Ok(Self::Array), + 10 => Ok(Self::Uint64), + 11 => Ok(Self::Int64), + 12 => Ok(Self::Float64), + _ => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Unknown GGUF value type: {}", value), + )), + } + } +} + +#[derive(Serialize)] +pub struct GgufMetadata { + version: u32, + tensor_count: u64, + metadata: HashMap, +} + +fn read_gguf_metadata_internal>(path: P) -> io::Result { + let mut file = BufReader::new(File::open(path)?); + + let mut magic = [0u8; 4]; + file.read_exact(&mut magic)?; + if &magic != b"GGUF" { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Not a GGUF file")); + } + + let version = file.read_u32::()?; + let tensor_count = file.read_u64::()?; + let metadata_count = file.read_u64::()?; + + let mut metadata_map = HashMap::new(); + for i in 0..metadata_count { + match read_metadata_entry(&mut file, i) { + Ok((key, value)) => { + metadata_map.insert(key, value); + } + Err(e) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Error reading metadata entry {}: {}", i, e), + )); + } + } + } + + Ok(GgufMetadata { + version, + tensor_count, + metadata: metadata_map, + }) +} + +fn read_metadata_entry(reader: &mut R, index: u64) -> io::Result<(String, String)> { + let key = read_gguf_string(reader).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Failed to read key for metadata entry {}: {}", index, e), + ) + })?; + + let value_type_u32 = reader.read_u32::()?; + let value_type = GgufValueType::try_from(value_type_u32)?; + let value = read_gguf_value(reader, value_type)?; + + Ok((key, value)) +} + +fn read_gguf_string(reader: &mut R) -> io::Result { + let len = reader.read_u64::()?; + if len > (1024 * 1024) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("String length {} is unreasonably large", len), + )); + } + let mut buf = vec![0u8; len as usize]; + reader.read_exact(&mut buf)?; + Ok(String::from_utf8(buf).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?) +} + +fn read_gguf_value( + reader: &mut R, + value_type: GgufValueType, +) -> io::Result { + match value_type { + GgufValueType::Uint8 => Ok(reader.read_u8()?.to_string()), + GgufValueType::Int8 => Ok(reader.read_i8()?.to_string()), + GgufValueType::Uint16 => Ok(reader.read_u16::()?.to_string()), + GgufValueType::Int16 => Ok(reader.read_i16::()?.to_string()), + GgufValueType::Uint32 => Ok(reader.read_u32::()?.to_string()), + GgufValueType::Int32 => Ok(reader.read_i32::()?.to_string()), + GgufValueType::Float32 => Ok(reader.read_f32::()?.to_string()), + GgufValueType::Bool => Ok((reader.read_u8()? != 0).to_string()), + GgufValueType::String => read_gguf_string(reader), + GgufValueType::Uint64 => Ok(reader.read_u64::()?.to_string()), + GgufValueType::Int64 => Ok(reader.read_i64::()?.to_string()), + GgufValueType::Float64 => Ok(reader.read_f64::()?.to_string()), + GgufValueType::Array => { + let elem_type_u32 = reader.read_u32::()?; + let elem_type = GgufValueType::try_from(elem_type_u32)?; + let len = reader.read_u64::()?; + + if len > 1_000_000 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Array length {} is unreasonably large", len), + )); + } + + if len > 24 { + skip_array_data(reader, elem_type, len)?; + return Ok(format!( + "", + elem_type, len + )); + } + + let mut elems = Vec::with_capacity(len as usize); + for _ in 0..len { + elems.push(read_gguf_value(reader, elem_type)?); + } + Ok(format!("[{}]", elems.join(", "))) + } + } +} + +fn skip_array_data( + reader: &mut R, + elem_type: GgufValueType, + len: u64, +) -> io::Result<()> { + match elem_type { + GgufValueType::Uint8 | GgufValueType::Int8 | GgufValueType::Bool => { + reader.seek(io::SeekFrom::Current(len as i64))?; + } + GgufValueType::Uint16 | GgufValueType::Int16 => { + reader.seek(io::SeekFrom::Current((len * 2) as i64))?; + } + GgufValueType::Uint32 | GgufValueType::Int32 | GgufValueType::Float32 => { + reader.seek(io::SeekFrom::Current((len * 4) as i64))?; + } + GgufValueType::Uint64 | GgufValueType::Int64 | GgufValueType::Float64 => { + reader.seek(io::SeekFrom::Current((len * 8) as i64))?; + } + GgufValueType::String => { + for _ in 0..len { + let str_len = reader.read_u64::()?; + reader.seek(io::SeekFrom::Current(str_len as i64))?; + } + } + GgufValueType::Array => { + for _ in 0..len { + read_gguf_value(reader, elem_type)?; + } + } + } + Ok(()) +} + +#[tauri::command] +pub async fn read_gguf_metadata(path: String) -> Result { + // run the blocking code in a separate thread pool + tauri::async_runtime::spawn_blocking(move || { + read_gguf_metadata_internal(path) + .map_err(|e| e.to_string()) + }) + .await + .unwrap() +} + diff --git a/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/mod.rs b/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/mod.rs index 35a24a4f9..2bb37b87f 100644 --- a/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/mod.rs +++ b/src-tauri/src/core/utils/extensions/inference_llamacpp_extension/mod.rs @@ -1,2 +1,3 @@ pub mod server; pub mod cleanup; +pub mod gguf; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index b0935206b..263f488cc 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -102,6 +102,7 @@ pub fn run() { core::utils::extensions::inference_llamacpp_extension::server::get_loaded_models, core::utils::extensions::inference_llamacpp_extension::server::generate_api_key, core::utils::extensions::inference_llamacpp_extension::server::is_process_running, + core::utils::extensions::inference_llamacpp_extension::gguf::read_gguf_metadata, ]) .manage(AppState { app_token: Some(generate_app_token()),