From d147bc3e3eec84a3f2b896ed1d38cfe6b9658c59 Mon Sep 17 00:00:00 2001 From: ph0llux Date: Fri, 6 Oct 2023 16:21:43 +0200 Subject: [PATCH] implemented chunkmap preload feature --- Cargo.toml | 3 ++- src/fs/mod.rs | 43 ++++++++++++++++++++++++++++++++++++++++++- src/main.rs | 49 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 86 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ceae528..c143f94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,4 +29,5 @@ env_logger = "0.9.0" serde = { version = "1.0", features = ["derive"] } hex = "0.4.3" toml = "0.5.8" -dialoguer = "0.10.4" \ No newline at end of file +dialoguer = "0.10.4" +redb = "1.0.5" \ No newline at end of file diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 7e992f0..6b7a52d 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -33,6 +33,14 @@ use libc::ENOENT; use time::{OffsetDateTime}; use dialoguer::{theme::ColorfulTheme, Password as PasswordDialog}; +#[derive(Debug)] +pub enum PreloadChunkmap { + None, + InMemory, + Redb(redb::Database) +} + + #[derive(Debug, Clone, Eq, PartialEq)] struct ZffFsCache { pub object_list: BTreeMap, @@ -65,7 +73,7 @@ pub struct ZffFs { } impl ZffFs { - pub fn new(inputfiles: Vec, decryption_passwords: &HashMap) -> Self { + pub fn new(inputfiles: Vec, decryption_passwords: &HashMap, preload_chunkmap: PreloadChunkmap) -> Self { info!("Reading segment files to create initial ZffReader."); let mut zffreader = match ZffReader::with_reader(inputfiles) { Ok(reader) => reader, @@ -165,6 +173,39 @@ impl ZffFs { } let cache = ZffFsCache::with_data(object_list, inode_reverse_map, filename_lookup_table, inode_attributes_map); + //preload chunkmap + match preload_chunkmap { + PreloadChunkmap::None => (), + PreloadChunkmap::InMemory => { + info!("Preload in memory chunkmap ..."); + if let Err(e) = zffreader.set_preload_chunkmap_mode_in_memory() { + error!("An error occurred while trying to create the in memory preload chunkmap."); + debug!("{e}"); + exit(EXIT_STATUS_ERROR); + }; + if let Err(e) = zffreader.preload_chunkmap_full() { + error!("An error occurred while trying to preload chunkmap."); + debug!("{e}"); + exit(EXIT_STATUS_ERROR); + }; + info!("In memory chunkmap successfully preloaded..."); + } + PreloadChunkmap::Redb(db) => { + info!("Preload redb chunkmap ..."); + if let Err(e) = zffreader.set_preload_chunkmap_mode_redb(db) { + error!("An error occurred while trying to create the redb preload chunkmap."); + debug!("{e}"); + exit(EXIT_STATUS_ERROR); + }; + if let Err(e) = zffreader.preload_chunkmap_full() { + error!("An error occurred while trying to preload chunkmap."); + debug!("{e}"); + exit(EXIT_STATUS_ERROR); + }; + info!("Redb chunkmap successfully preloaded ..."); + } + } + Self { zffreader, shift_value, diff --git a/src/main.rs b/src/main.rs index 39dec45..973fc9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,9 +7,6 @@ use std::process::exit; use std::path::PathBuf; use std::fs::{File}; - - - // - modules mod fs; mod constants; @@ -49,13 +46,29 @@ pub struct Cli { #[clap(short='p', long="decryption-passwords", value_parser = parse_key_val::)] decryption_passwords: Vec<(String, String)>, - /// The Loglevel #[clap(short='l', long="log-level", arg_enum, default_value="info")] - log_level: LogLevel + log_level: LogLevel, + + /// Preload the chunkmap (in memory or in redb database e.g. at a fast NVMe drive) to speed up the read operations. + /// None: saves memory but the read operations are slower (default) + /// In memory: needs 24 bytes per chunk (plus a lot of bytes for additional overhead) to store the chunkmap in memory. This is the fastest option, but you need to ensure that you have enough memory. + /// redb: use a fast redb database to cache the chunkmap. This could e.g. be useful, if your container is stored at a slow harddrive but the redb database can be cached at a fast nvme drive. + #[clap(short='c', long="preload-chunkmap", arg_enum, default_value="none")] + preload_chunkmap: PreloadChunkmapValue, + + #[clap(short='r', long="redb-path", required_if_eq("preload-chunkmap", "redb"))] + redb_path: Option, } -#[derive(ArgEnum, Clone)] +#[derive(ArgEnum, Clone, Debug)] +enum PreloadChunkmapValue { + None, + InMemory, + Redb, +} + +#[derive(ArgEnum, Clone, Debug)] enum LogLevel { Error, Warn, @@ -97,6 +110,9 @@ fn main() { .init(); let inputfiles = open_files(&args); + + let preload_chunkmap = gen_preload_chunkmap(&args); + let mut decryption_passwords = HashMap::new(); for (obj_no, pw) in args.decryption_passwords { let obj_no = match obj_no.parse::() { @@ -109,7 +125,7 @@ fn main() { decryption_passwords.insert(obj_no, pw); } - let fs = ZffFs::new(inputfiles, &decryption_passwords); + let fs = ZffFs::new(inputfiles, &decryption_passwords, preload_chunkmap); let mountoptions = vec![MountOption::RO, MountOption::FSName(String::from(ZFF_OVERLAY_FS_NAME))]; let session = match fuser::spawn_mount2(fs, &args.mount_point, &mountoptions) { Ok(session) => session, @@ -144,4 +160,23 @@ fn main() { exit(EXIT_STATUS_SUCCESS); } } +} + +fn gen_preload_chunkmap(args: &Cli) -> fs::PreloadChunkmap { + match args.preload_chunkmap { + PreloadChunkmapValue::None => fs::PreloadChunkmap::None, + PreloadChunkmapValue::InMemory => fs::PreloadChunkmap::InMemory, + PreloadChunkmapValue::Redb => { + //unwrap should safe here, because it is a required argument defined by clap. + let db = match redb::Database::create(args.redb_path.clone().unwrap()) { + Ok(db) => db, + Err(e) => { + error!("An error occurred while trying to create preload chunmap database."); + debug!("{e}"); + exit(EXIT_STATUS_ERROR); + } + }; + fs::PreloadChunkmap::Redb(db) + } + } } \ No newline at end of file