implemented chunkmap preload feature

Cargo.toml

@@ -29,4 +29,5 @@ env_logger = "0.9.0"
 serde = { version = "1.0", features = ["derive"] }
 hex = "0.4.3"
 toml = "0.5.8"
-dialoguer = "0.10.4"
+dialoguer = "0.10.4"
+redb = "1.0.5"

src/fs/mod.rs

@@ -33,6 +33,14 @@ use libc::ENOENT;
 use time::{OffsetDateTime};
 use dialoguer::{theme::ColorfulTheme, Password as PasswordDialog};
 
+#[derive(Debug)]
+pub enum PreloadChunkmap {
+    None,
+    InMemory,
+    Redb(redb::Database)
+}
+
+
 #[derive(Debug, Clone, Eq, PartialEq)]
 struct ZffFsCache {
     pub object_list: BTreeMap<u64, ZffReaderObjectType>,
@@ -65,7 +73,7 @@ pub struct ZffFs<R: Read + Seek> {
 }
 
 impl<R: Read + Seek> ZffFs<R> {
-    pub fn new(inputfiles: Vec<R>, decryption_passwords: &HashMap<u64, String>) -> Self {
+    pub fn new(inputfiles: Vec<R>, decryption_passwords: &HashMap<u64, String>, preload_chunkmap: PreloadChunkmap) -> Self {
         info!("Reading segment files to create initial ZffReader.");
         let mut zffreader = match ZffReader::with_reader(inputfiles) {
             Ok(reader) => reader,
@@ -165,6 +173,39 @@ impl<R: Read + Seek> ZffFs<R> {
         }
         let cache = ZffFsCache::with_data(object_list, inode_reverse_map, filename_lookup_table, inode_attributes_map);
 
+        //preload chunkmap
+        match preload_chunkmap {
+            PreloadChunkmap::None => (),
+            PreloadChunkmap::InMemory => {
+                info!("Preload in memory chunkmap ...");
+                if let Err(e) = zffreader.set_preload_chunkmap_mode_in_memory() {
+                    error!("An error occurred while trying to create the in memory preload chunkmap.");
+                    debug!("{e}");
+                    exit(EXIT_STATUS_ERROR);
+                };
+                if let Err(e) = zffreader.preload_chunkmap_full() {
+                    error!("An error occurred while trying to preload chunkmap.");
+                    debug!("{e}");
+                    exit(EXIT_STATUS_ERROR);
+                };
+                info!("In memory chunkmap successfully preloaded...");
+            }
+            PreloadChunkmap::Redb(db) => {
+                info!("Preload redb chunkmap ...");
+                if let Err(e) = zffreader.set_preload_chunkmap_mode_redb(db) {
+                    error!("An error occurred while trying to create the redb preload chunkmap.");
+                    debug!("{e}");
+                    exit(EXIT_STATUS_ERROR);
+                };
+                if let Err(e) = zffreader.preload_chunkmap_full() {
+                    error!("An error occurred while trying to preload chunkmap.");
+                    debug!("{e}");
+                    exit(EXIT_STATUS_ERROR);
+                };
+                info!("Redb chunkmap successfully preloaded ...");
+            }
+        }
+
         Self {
             zffreader,
             shift_value,

src/main.rs

@@ -7,9 +7,6 @@ use std::process::exit;
 use std::path::PathBuf;
 use std::fs::{File};
 
-
-
-
 // - modules
 mod fs;
 mod constants;
@@ -49,13 +46,29 @@ pub struct Cli {
     #[clap(short='p', long="decryption-passwords", value_parser = parse_key_val::<String, String>)]
     decryption_passwords: Vec<(String, String)>,
 
-
     /// The Loglevel
     #[clap(short='l', long="log-level", arg_enum, default_value="info")]
-    log_level: LogLevel
+    log_level: LogLevel,
+
+    /// Preload the chunkmap (in memory or in redb database e.g. at a fast NVMe drive) to speed up the read operations.  
+    /// None: saves memory but the read operations are slower (default)   
+    /// In memory: needs 24 bytes per chunk (plus a lot of bytes for additional overhead) to store the chunkmap in memory. This is the fastest option, but you need to ensure that you have enough memory.  
+    /// redb: use a fast redb database to cache the chunkmap. This could e.g. be useful, if your container is stored at a slow harddrive but the redb database can be cached at a fast nvme drive.  
+    #[clap(short='c', long="preload-chunkmap", arg_enum, default_value="none")]
+    preload_chunkmap: PreloadChunkmapValue,
+
+    #[clap(short='r', long="redb-path", required_if_eq("preload-chunkmap", "redb"))]
+    redb_path: Option<PathBuf>,
 }
 
-#[derive(ArgEnum, Clone)]
+#[derive(ArgEnum, Clone, Debug)]
+enum PreloadChunkmapValue {
+    None,
+    InMemory,
+    Redb,
+}
+
+#[derive(ArgEnum, Clone, Debug)]
 enum LogLevel {
     Error,
     Warn,
@@ -97,6 +110,9 @@ fn main() {
         .init();
 
     let inputfiles = open_files(&args);
+
+    let preload_chunkmap = gen_preload_chunkmap(&args);
+
     let mut decryption_passwords = HashMap::new();
     for (obj_no, pw) in args.decryption_passwords {
         let obj_no = match obj_no.parse::<u64>() {
@@ -109,7 +125,7 @@ fn main() {
         decryption_passwords.insert(obj_no, pw);
     }
 
-    let fs = ZffFs::new(inputfiles, &decryption_passwords);
+    let fs = ZffFs::new(inputfiles, &decryption_passwords, preload_chunkmap);
     let mountoptions = vec![MountOption::RO, MountOption::FSName(String::from(ZFF_OVERLAY_FS_NAME))];
     let session = match fuser::spawn_mount2(fs, &args.mount_point, &mountoptions) {
         Ok(session) => session,
@@ -144,4 +160,23 @@ fn main() {
             exit(EXIT_STATUS_SUCCESS);
         }
     }
+}
+
+fn gen_preload_chunkmap(args: &Cli) -> fs::PreloadChunkmap {
+    match args.preload_chunkmap {
+        PreloadChunkmapValue::None => fs::PreloadChunkmap::None,
+        PreloadChunkmapValue::InMemory => fs::PreloadChunkmap::InMemory,
+        PreloadChunkmapValue::Redb => {
+            //unwrap should safe here, because it is a required argument defined by clap.
+            let db = match redb::Database::create(args.redb_path.clone().unwrap()) {
+                Ok(db) => db,
+                Err(e) => {
+                    error!("An error occurred while trying to create preload chunmap database.");
+                    debug!("{e}");
+                    exit(EXIT_STATUS_ERROR);
+                }
+            };
+            fs::PreloadChunkmap::Redb(db)
+        }
+    }
 }