diff --git a/src/lib/compression.rs b/src/lib/compression.rs index f666eb9..20228b1 100644 --- a/src/lib/compression.rs +++ b/src/lib/compression.rs @@ -52,7 +52,7 @@ impl fmt::Display for CompressionAlgorithm { } } -/// decompresses a buffer with the given [CompressionAlgorithm]. +/// Decompresses a buffer with the given [CompressionAlgorithm]. pub fn decompress_buffer(buffer: &[u8], compression_algorithm: C) -> Result> where C: Borrow, @@ -74,6 +74,7 @@ where } } +/// Decompresses a reader with the given [CompressionAlgorithm]. pub fn decompress_reader(input: &mut R, compression_algorithm: C) -> Result> where C: Borrow, diff --git a/src/lib/constants.rs b/src/lib/constants.rs index 0b99b9a..537d895 100644 --- a/src/lib/constants.rs +++ b/src/lib/constants.rs @@ -89,6 +89,7 @@ pub(crate) const ERROR_ZFFREADER_MISSING_OBJECT: &str = "Missing object number i // Default values pub(crate) const DEFAULT_LENGTH_HEADER_IDENTIFIER: usize = 4; pub(crate) const DEFAULT_LENGTH_VALUE_HEADER_LENGTH: usize = 8; + /// The number of the first object in a zff container. pub const INITIAL_OBJECT_NUMBER: u64 = 1; @@ -174,7 +175,9 @@ pub(crate) const METADATA_BTIME: &str = "btime"; // - ChunkMap +/// Table name for the redb chunkmap pub const CHUNK_MAP_TABLE: TableDefinition<&[u8; 32], u64> = TableDefinition::new("map"); +/// Table name for the redb preloaded chunkmap pub const PRELOADED_CHUNK_MAP_TABLE: TableDefinition = TableDefinition::new("preloaded_map"); // - Encryption parameters diff --git a/src/lib/error.rs b/src/lib/error.rs index d2a4fd8..de4c6c0 100644 --- a/src/lib/error.rs +++ b/src/lib/error.rs @@ -29,7 +29,9 @@ use cbc::cipher::block_padding::UnpadError as AesCbcError; /// The main error-type of this crate. #[derive(Debug)] pub struct ZffError { + /// A detailed error message. pub details: String, + /// The appropriate [ZffErrorKind]. pub kind: ZffErrorKind, } @@ -283,6 +285,20 @@ impl ZffError { } } + /// Creates a new crate-related "value not in map" error. + /// # Example + /// ``` + /// use zff::{ZffError, Result}; + /// fn my_func() -> Result<()> { + /// let decode_error = ZffError::new_not_in_map_error("the appropriate value is not in map!"); + /// Err(decode_error) + /// } + /// fn main() { + /// match my_func() { + /// Err(x) => println!("It work's! Your custom error message is: {}", x), + /// _ => () + /// } + /// } pub fn new_not_in_map_error() -> ZffError { ZffError { kind: ZffErrorKind::ValueNotInMap, diff --git a/src/lib/footer/file_footer.rs b/src/lib/footer/file_footer.rs index fba46be..c731b1e 100644 --- a/src/lib/footer/file_footer.rs +++ b/src/lib/footer/file_footer.rs @@ -124,6 +124,7 @@ impl FileFooter { vec } + /// encrypts the file footer by the given encryption information and returns the encrypted file footer. pub fn encrypt_directly(&self, encryption_information: E) -> Result> where E: Borrow diff --git a/src/lib/footer/main_footer.rs b/src/lib/footer/main_footer.rs index b497679..5973973 100644 --- a/src/lib/footer/main_footer.rs +++ b/src/lib/footer/main_footer.rs @@ -104,6 +104,7 @@ impl MainFooter { Some(self.description_notes.as_ref()?) } + /// Returns a reference of the global chunkmap table. pub fn chunk_maps(&self) -> &BTreeMap { &self.chunk_maps } diff --git a/src/lib/footer/mod.rs b/src/lib/footer/mod.rs index d766997..9908c39 100644 --- a/src/lib/footer/mod.rs +++ b/src/lib/footer/mod.rs @@ -1,8 +1,8 @@ // - modules mod main_footer; mod segment_footer; -pub mod file_footer; -pub mod object_footer; +mod file_footer; +mod object_footer; // - re-exports -- this section contains the footer of the current zff version. pub use main_footer::*; diff --git a/src/lib/footer/object_footer.rs b/src/lib/footer/object_footer.rs index 5e7c7a6..f45115f 100644 --- a/src/lib/footer/object_footer.rs +++ b/src/lib/footer/object_footer.rs @@ -97,6 +97,7 @@ impl ObjectFooter { } } + /// Returns the appropriate object number. pub fn object_number(&self) -> u64 { match self { ObjectFooter::Physical(footer) => footer.object_number, @@ -104,6 +105,7 @@ impl ObjectFooter { } } + /// Returns the appropriate acquisition start timestamp. pub fn acquisition_start(&self) -> u64 { match self { ObjectFooter::Physical(footer) => footer.acquisition_start, @@ -111,6 +113,7 @@ impl ObjectFooter { } } + /// Returns the appropriate acquisition end timestamp. pub fn acquisition_end(&self) -> u64 { match self { ObjectFooter::Physical(footer) => footer.acquisition_end, @@ -248,17 +251,21 @@ impl EncryptedObjectFooter { -/// Encrypted footer. +/// Represents an encrypted object footer of a physical object. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(Serialize))] #[cfg_attr(feature = "serde", derive(Deserialize))] pub struct EncryptedObjectFooterPhysical { + /// The appropriate footer version. pub version: u8, + /// The appropriate object number. pub object_number: u64, + /// The underlying data in encrypted form. pub encrypted_data: Vec, } impl EncryptedObjectFooterPhysical { + /// Creates a new [EncryptedObjectFooterPhysical] by the given values. pub fn new(version: u8, object_number: u64, encrypted_data: Vec) -> Self { Self { version, @@ -267,7 +274,7 @@ impl EncryptedObjectFooterPhysical { } } - /// tries to decrypt the ObjectFooter. If an error occures, the EncryptedObjectFooterPhysical is still available. + /// Tries to decrypt the ObjectFooter. If an error occures, the EncryptedObjectFooterPhysical is still available. pub fn decrypt(&self, key: K, algorithm: A) -> Result where A: Borrow, @@ -292,7 +299,7 @@ impl EncryptedObjectFooterPhysical { hash_header)) } - /// tries to decrypt the ObjectFooter. Consumes the EncryptedObjectFooterPhysical, regardless of whether an error occurs or not. + /// Tries to decrypt the ObjectFooter. Consumes the EncryptedObjectFooterPhysical, regardless of whether an error occurs or not. pub fn decrypt_and_consume(self, key: K, algorithm: A) -> Result where A: Borrow, @@ -370,13 +377,21 @@ impl HeaderCoding for EncryptedObjectFooterPhysical { #[derive(Debug,Clone, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct ObjectFooterPhysical { + /// The version of the footer. pub version: u8, + /// The object number of the footer. pub object_number: u64, + /// The acquisition start timestamp of the footer. pub acquisition_start: u64, + /// The acquisition end timestamp of the footer. pub acquisition_end: u64, + /// The original length of the data. pub length_of_data: u64, + /// The first used chunk number in this object. pub first_chunk_number: u64, + /// The total number of chunks used in this object. pub number_of_chunks: u64, + /// The appropriate [crate::header::HashHeader]. pub hash_header: HashHeader, } @@ -414,6 +429,7 @@ impl ObjectFooterPhysical { vec } + /// encrypts the object footer by the given encryption information and returns the encrypted object footer. pub fn encrypt_directly(&self, encryption_information: E) -> Result> where E: Borrow @@ -515,17 +531,21 @@ impl HeaderCoding for ObjectFooterPhysical { } } -/// Encrypted footer. +/// An object footer for a logical object in encrypted form. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(Serialize))] #[cfg_attr(feature = "serde", derive(Deserialize))] pub struct EncryptedObjectFooterLogical { + /// The footer version. pub version: u8, + /// The appropriate object number. pub object_number: u64, + /// the encrypted data of this footer pub encrypted_data: Vec, } impl EncryptedObjectFooterLogical { + /// Creates a new [EncryptedObjectFooterLogical] by the given values. pub fn new(version: u8, object_number: u64, encrypted_data: Vec) -> Self { Self { version, @@ -534,7 +554,7 @@ impl EncryptedObjectFooterLogical { } } - /// tries to decrypt the ObjectFooter. If an error occures, the EncryptedObjectFooterPhysical is still available. + /// Tries to decrypt the ObjectFooter. If an error occures, the EncryptedObjectFooterPhysical is still available. pub fn decrypt(&self, key: K, algorithm: A) -> Result where A: Borrow, @@ -561,7 +581,7 @@ impl EncryptedObjectFooterLogical { file_footer_offsets)) } - /// tries to decrypt the ObjectFooter. Consumes the EncryptedObjectFooterPhysical, regardless of whether an error occurs or not. + /// Tries to decrypt the ObjectFooter. Consumes the EncryptedObjectFooterPhysical, regardless of whether an error occurs or not. pub fn decrypt_and_consume(self, key: K, algorithm: A) -> Result where A: Borrow, @@ -780,6 +800,7 @@ impl ObjectFooterLogical { vec } + /// encrypts the object footer by the given encryption information and returns the encrypted object footer. pub fn encrypt_directly(&self, encryption_information: E) -> Result> where E: Borrow @@ -886,5 +907,4 @@ impl HeaderCoding for ObjectFooterLogical { file_footer_segment_numbers, file_footer_offsets)) } - } \ No newline at end of file diff --git a/src/lib/footer/segment_footer.rs b/src/lib/footer/segment_footer.rs index 208ccbc..eff4be2 100644 --- a/src/lib/footer/segment_footer.rs +++ b/src/lib/footer/segment_footer.rs @@ -32,11 +32,17 @@ use serde::{ #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct SegmentFooter { + /// The footer version pub version: u8, + /// The total length of the segment. pub length_of_segment: u64, + /// A [HashMap] containing the object number and the appropriate offset of the [crate::header::ObjectHeader]. pub object_header_offsets: HashMap, //, + /// A [HashMap] containing the object number and the appropriate offset of the [crate::footer::ObjectFooter]. pub object_footer_offsets: HashMap, //, + /// [BTreeMap] containing the chunk number and the appropriate offset of the chunkmaps. pub chunk_map_table: BTreeMap, // + /// The first chunk number which was used in this segment. pub first_chunk_number: u64, /// The offset where the footer starts. pub footer_offset: u64, diff --git a/src/lib/header/chunk_header.rs b/src/lib/header/chunk_header.rs index 965a674..86be8f3 100644 --- a/src/lib/header/chunk_header.rs +++ b/src/lib/header/chunk_header.rs @@ -29,15 +29,22 @@ use serde::{ Serialize, }; +/// The appropriate flags for each chunk. #[derive(Debug,Clone,Default)] #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct ChunkHeaderFlags { + /// is set, if an read error is occured and the data in this chunk could be corrupted. pub error: bool, + /// is set, if the data in the chunk are compressed. pub compression: bool, + /// is set, if the chunk contains the same bytes. pub same_bytes: bool, + /// is set, if this chunk is a duplicate of an other chunk. pub duplicate: bool, + /// is set, if the chunk data is encrypted. pub encryption: bool, + /// is set, if this is a placeholder chunk of an empty file. pub empty_file: bool, } @@ -74,10 +81,15 @@ impl ChunkHeaderFlags { #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct ChunkHeader { + /// the header version. pub version: u8, + /// the appropriate chunk number. pub chunk_number: u64, + /// the appropriate size of the chunk data (in the final optionally compressed and encrypted form). pub chunk_size: u64, + /// the appropriate chunk header flags. pub flags: ChunkHeaderFlags, + /// the crc32 value to ensure (a bit) integrity. pub crc32: u32, } @@ -108,7 +120,7 @@ impl ChunkHeader { } } - /// tries to encrypt the ChunkHeader. If an error occures, the unencrypted ChunkHeader is still available. + /// Tries to encrypt the ChunkHeader. If an error occures, the unencrypted ChunkHeader is still available. pub fn encrypt(&self, key: K, algorithm: A) -> Result where A: Borrow, @@ -118,7 +130,7 @@ impl ChunkHeader { Ok(EncryptedChunkHeader::new(self.chunk_number, self.chunk_size, self.flags.clone(), crc32)) } - /// tries to encrypt the ChunkHeader. Consumes theunencrypted ChunkHeader, regardless of whether an error occurs or not. + /// Tries to encrypt the ChunkHeader. Consumes theunencrypted ChunkHeader, regardless of whether an error occurs or not. pub fn encrypt_and_consume(self, key: K, algorithm: A) -> Result where A: Borrow, @@ -197,15 +209,20 @@ impl ChunkHeader { } } -/// Header for chunk data (contains encrypted crc32 and ed25519 signature). +/// Header for (encrypted) chunk data (contains encrypted crc32 and ed25519 signature). #[derive(Debug,Clone)] #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct EncryptedChunkHeader { + /// the header version. pub version: u8, + /// the appropriate chunk number pub chunk_number: u64, + /// the appropriate size of the chunk data (in the final optionally compressed and encrypted form). pub chunk_size: u64, + /// the appropriate chunk header flags. pub flags: ChunkHeaderFlags, + /// the encrypted crc32 value. pub crc32: Vec, } diff --git a/src/lib/header/chunk_map.rs b/src/lib/header/chunk_map.rs index 00a81ac..a93dcd1 100644 --- a/src/lib/header/chunk_map.rs +++ b/src/lib/header/chunk_map.rs @@ -34,37 +34,37 @@ use hex; #[derive(Debug,Clone,PartialEq,Eq)] #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] -pub struct ChunkMap { - pub chunkmap: BTreeMap, // +pub(crate) struct ChunkMap { + pub(crate) chunkmap: BTreeMap, // target_size: usize, } impl ChunkMap { /// returns a new [ChunkMap] with the given values. - pub fn new(chunkmap: BTreeMap) -> Self { + pub(crate) fn new(chunkmap: BTreeMap) -> Self { Self { chunkmap, target_size: 0, } } - pub fn current_size(&self) -> usize { + pub(crate) fn current_size(&self) -> usize { self.chunkmap.len() * 16 + 8 } /// returns a new, empty [ChunkMap] with the given values. - pub fn new_empty() -> Self { + pub(crate) fn new_empty() -> Self { Self { chunkmap: BTreeMap::new(), target_size: 0, } } - pub fn set_target_size(&mut self, target_size: usize) { + pub(crate) fn set_target_size(&mut self, target_size: usize) { self.target_size = target_size } - pub fn add_chunk_entry(&mut self, chunk_no: u64, offset: u64) -> bool { + pub(crate) fn add_chunk_entry(&mut self, chunk_no: u64, offset: u64) -> bool { if self.target_size < self.current_size() + 24 { //24 -> 8bytes for next chunk_no, 8bytes for next offset, 8 bytes for the size of the encoded BTreeMap false } else { @@ -73,7 +73,7 @@ impl ChunkMap { } } - pub fn flush(&mut self) -> BTreeMap { + pub(crate) fn flush(&mut self) -> BTreeMap { std::mem::take(&mut self.chunkmap) } } @@ -122,9 +122,12 @@ impl ChunkMap { } } +/// A map which can be used to handle the chunk deduplication. #[derive(Debug)] pub enum DeduplicationChunkMap { + /// Use a in-memory deduplication map at cost of memory. InMemory(HashMap), // + /// Use a Redb based deduplication map at cost of I/O. Redb(Database), } @@ -135,19 +138,24 @@ impl Default for DeduplicationChunkMap { } impl DeduplicationChunkMap { + /// Creates a new [DeduplicationChunkMap] with a Redb by given path. + /// May fail if the Redb can not be created at the given Path. pub fn new_from_path>(path: P) -> Result { let db = Database::create(path.as_ref())?; Ok(Self::Redb(db)) } + /// Creates a new [DeduplicationChunkMap] with the given [Redb-Database](crate::redb::Database). pub fn new_from_db(database: Database) -> Self { Self::Redb(database) } + /// Creates a new in-memory [DeduplicationChunkMap]. pub fn new_in_memory_map() -> Self { DeduplicationChunkMap::InMemory(HashMap::new()) } + /// Adds an entry to the deduplication map. pub fn append_entry(&mut self, chunk_no: u64, blak3_hash: Blake3Hash) -> Result<()> { match self { DeduplicationChunkMap::InMemory(map) => { @@ -169,7 +177,7 @@ impl DeduplicationChunkMap { } } - #[allow(clippy::let_and_return)] + /// Returns the chunk number to the given hash. pub fn get_chunk_number(&mut self, blak3_hash: B) -> Result where B: Borrow diff --git a/src/lib/header/encryption_header.rs b/src/lib/header/encryption_header.rs index 6085569..3944e68 100644 --- a/src/lib/header/encryption_header.rs +++ b/src/lib/header/encryption_header.rs @@ -35,11 +35,14 @@ use serde::{ #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct EncryptionInformation { + /// The encryption key in **unencrypted** form. pub encryption_key: Vec, + /// The used [crate::encryption::EncryptionAlgorithm]. pub algorithm: EncryptionAlgorithm, } impl EncryptionInformation { + /// Creates a new [EncryptionInformation] by the given values. pub fn new(key: Vec, algorithm: EncryptionAlgorithm) -> Self { Self { encryption_key: key, diff --git a/src/lib/header/file_header.rs b/src/lib/header/file_header.rs index 91c45d7..89f79d1 100644 --- a/src/lib/header/file_header.rs +++ b/src/lib/header/file_header.rs @@ -128,10 +128,15 @@ impl fmt::Display for SpecialFileType { #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct FileHeader { + /// The appropriate filenumber. pub file_number: u64, + /// The appropriate filetype. pub file_type: FileType, + /// The appropriate filename. pub filename: String, + /// The parent file number of this file. Will be 0, if the parent is the root directory. pub parent_file_number: u64, + /// A [HashMap] of the metadata of this file. pub metadata_ext: HashMap, } diff --git a/src/lib/header/object_header.rs b/src/lib/header/object_header.rs index 4f7ef48..96b0bc6 100644 --- a/src/lib/header/object_header.rs +++ b/src/lib/header/object_header.rs @@ -51,8 +51,11 @@ use serde::{ #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct ObjectFlags { + /// this flag is set if the object is encrypted. pub encryption: bool, + /// this flag is set, if signatures are available for this object. pub sign_hash: bool, + /// this flag is set, if the object is passive and should not read directly, but via a virtual object. pub passive_object: bool, } @@ -78,12 +81,19 @@ impl From for ObjectFlags { #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct ObjectHeader { + /// the appropriate object number. pub object_number: u64, + /// the appropriate [ObjectFlags]. pub flags: ObjectFlags, + /// the [EncryptionHeader], if available. pub encryption_header: Option, + /// the target chunk size for chunks of this object. pub chunk_size: u64, + /// the [crate::header::CompressionHeader], containing all information about the used compression method for this object. pub compression_header: CompressionHeader, + /// the [crate::header::DescriptionHeader], containing some information about this object. pub description_header: DescriptionHeader, + /// the appropriate [ObjectType]. pub object_type: ObjectType, } @@ -367,14 +377,20 @@ impl fmt::Display for ObjectType { } - +/// An [EncryptedObjectHeader] contains the appropriate object number, the [ObjectFlags], +/// the [crate::header::EncryptionHeader] and +/// the encrypted blob containing the other header values (in encrypted form). #[derive(Debug,Clone,Eq,PartialEq)] #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct EncryptedObjectHeader { + /// the appropriate object number. pub object_number: u64, + /// the object flags. pub flags: ObjectFlags, + /// the [crate::header::EncryptionHeader]. pub encryption_header: EncryptionHeader, + /// the encrypted blob with the other header values. pub encrypted_content: Vec } @@ -394,15 +410,16 @@ impl EncryptedObjectHeader { } } - /// decodes the length of the header. - fn decode_header_length(data: &mut R) -> Result { + //todo: check if this method is needed in any way or could be deleted + /// Decodes the length of the header. + pub fn decode_header_length(data: &mut R) -> Result { match data.read_u64::() { Ok(value) => Ok(value), Err(_) => Err(ZffError::new_header_decode_error(ERROR_HEADER_DECODER_HEADER_LENGTH)), } } - /// decodes the encrypted header with the given password. + /// Decodes the encrypted header with the given password. pub fn decrypt_with_password

(&mut self, password: P) -> Result where P: AsRef<[u8]>, @@ -426,7 +443,7 @@ impl EncryptedObjectHeader { Ok(object_header) } - /// tries to decrypt the ObjectFooter. Consumes the EncryptedObjectFooterPhysical, regardless of whether an error occurs or not. + /// Tries to decrypt the ObjectHeader. Consumes the EncryptedObjectHeader, regardless of whether an error occurs or not. pub fn decrypt_and_consume_with_password

(mut self, password: P) -> Result where P: AsRef<[u8]>, diff --git a/src/lib/header/pbe_header.rs b/src/lib/header/pbe_header.rs index 4133c4c..36928dc 100644 --- a/src/lib/header/pbe_header.rs +++ b/src/lib/header/pbe_header.rs @@ -207,7 +207,9 @@ impl ValueDecoder for KDFParameters { #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct PBKDF2SHA256Parameters { + /// The iterations to use. iterations: u32, + /// The salt value. salt: [u8; 32], } @@ -266,9 +268,13 @@ impl HeaderCoding for PBKDF2SHA256Parameters { #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct ScryptParameters { + /// The log n parameter for Scrypt. logn: u8, + /// The r parameter for Scrypt. r: u32, + /// The p parameter for Scrypt. p: u32, + /// The used salt. salt: [u8; 32], } @@ -344,9 +350,13 @@ impl HeaderCoding for ScryptParameters { #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct Argon2idParameters { + /// The memory cost parameter for Argon2id. pub mem_cost: u32, + /// The used number of lanes for Argon2id. pub lanes: u32, + /// The iterations value for Argon2id. pub iterations: u32, + /// The used salt. pub salt: [u8; 32], } diff --git a/src/lib/header/segment_header.rs b/src/lib/header/segment_header.rs index 94830c5..28d5bd6 100644 --- a/src/lib/header/segment_header.rs +++ b/src/lib/header/segment_header.rs @@ -31,8 +31,11 @@ use serde::{ #[cfg_attr(feature = "serde", derive(Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize))] pub struct SegmentHeader { + /// the unique identifier. Segments at the same group (=same zff container) should have the same identifier. pub unique_identifier: u64, + /// the appropriate segment number. pub segment_number: u64, + /// the target size of a chunkmap. pub chunkmap_size: u64, } diff --git a/src/lib/io/zffreader.rs b/src/lib/io/zffreader.rs index a6724b5..644f83b 100644 --- a/src/lib/io/zffreader.rs +++ b/src/lib/io/zffreader.rs @@ -3,6 +3,7 @@ use std::fmt; use std::borrow::Borrow; use std::io::{Read, Seek, SeekFrom, Cursor}; use std::collections::{HashMap, BTreeMap}; +use std::ops::Range; // - internal use crate::{ @@ -42,10 +43,14 @@ use super::*; // - external use redb::{Database, ReadableTable}; +/// Defines the recognized object type (used by the [ZffReader]). #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum ObjectType { + /// Physical object Physical, + /// Logical object Logical, + /// Encrypted object (physical or logical) Encrypted, } @@ -60,13 +65,38 @@ impl fmt::Display for ObjectType { } } +/// Several types of a preloaded chunkmap. #[derive(Debug)] -enum PreloadedChunkMap { +pub enum PreloadedChunkMap { + /// No chunkmap is preloaded. None, + /// Contains a in-memory preloaded chunkmap. InMemory(HashMap), //, + /// Contains a chunkmap, cached in the given Redb. Redb(Database), } +/// The [ZffReader] can be used to read the data of a zff container in a proper way. +/// It implements [std::io::Read] and [std::io::Seek] to ensure a wide range of possible use. +/// # Example +/// ```rust +/// use std::fs::File; +/// +/// let segment_files = vec!["zff_segment.z01", "zff_segment.z02", "zff_segment.z03"]; +/// let mut files = Vec::new(); +/// for segment in segment_files { +/// files.push(File::open(segment).unwrap()); +/// } +/// +/// let zffreader = ZffReader::with_reader(files); +/// assert_eq!(zffreader.list_objects().unwrap().keys(), vec![1]); +/// // let us assume object 1 is a physical object. +/// zffreader.initialize_object(1).unwrap(); +/// zffreader.set_active_object(1).unwrap(); +/// +/// let buffer = vec![0u8; 32000]; +/// let _ = zffreader.read_exact(&mut buffer).unwrap(); +/// ``` #[derive(Debug)] pub struct ZffReader { segments: HashMap>, // @@ -77,6 +107,11 @@ pub struct ZffReader { } impl ZffReader { + /// This method will initialize the [ZffReader] in general. + /// This method will identify the appropriate [SegmentHeader](crate::header::SegmentHeader), + /// [SegmentFooter](crate::footer::SegmentFooter) and [MainFooter](crate::footer::MainFooter). + /// This method will **not** initizalize the objects itself! This has to be done by using the + /// initialize_object() or initialize_objects_all() methods. pub fn with_reader(reader_vec: Vec) -> Result { let mut segments = HashMap::new(); let mut main_footer = None; @@ -111,6 +146,13 @@ impl ZffReader { }) } + /// Lists all objects which are inside the zff container (even if they are uninitialized). + /// Returns a BTreeMap, which contains the appropriate object number and the object type. + /// # Error + /// Fails if + /// - a segment is missing which should contain the appropriate object header + /// - there is a error while reading the object header + /// - there is a decoding error (e.g. corrupted segment) pub fn list_objects(&mut self) -> Result> { let mut map = BTreeMap::new(); for (object_number, segment_number) in self.main_footer.object_header() { @@ -134,6 +176,9 @@ impl ZffReader { Ok(map) } + /// Sets an appropriate object as active to read or seek from this object. + /// # Error + /// This method fails, if the appropriate object number not exists in this zff container. pub fn set_active_object(&mut self, object_number: u64) -> Result<()> { if self.object_reader.get(&object_number).is_some() { self.active_object = object_number; @@ -143,6 +188,10 @@ impl ZffReader { } } + /// Sets an appropriate file as active to read or seek from this object. + /// # Error + /// This method fails, if the appropriate object type is not "logical" or if no file for the appropriate file number exists. + /// Will also fail, if no object was activated by using the set_active_object() method. pub fn set_active_file(&mut self, filenumber: u64) -> Result<()> { if let Some(object_reader) = self.object_reader.get_mut(&self.active_object) { match object_reader { @@ -154,23 +203,37 @@ impl ZffReader { } } + /// Will initialize the appropriate object. + /// # Error + /// May fail due to various conditions, e.g. corrupted or missing segments. pub fn initialize_object(&mut self, object_number: u64) -> Result<()> { let object_reader = initialize_object_reader(object_number, &mut self.segments, &self.main_footer)?; self.object_reader.insert(object_number, object_reader); Ok(()) } + /// Same as initialize_object(), but will initialize **all** objects of this zff container. + /// # Error + /// May fail due to various conditions, e.g. corrupted or missing segments. pub fn initialize_objects_all(&mut self) -> Result<()> { let object_reader_map = initialize_object_reader_all(&mut self.segments, &self.main_footer)?; self.object_reader = object_reader_map; Ok(()) } + /// Lists the number of chunks of this zff container. pub fn number_of_chunks(&self) -> u64 { let (chunk_number, _) = self.main_footer.chunk_maps().last_key_value().unwrap_or((&0, &0)); *chunk_number } + /// Decrypts an encrypted initialized object (and re-initialize/replaces the appropriate object directly). + /// # Error + /// May fail due to various conditions: + /// - The appropriate object number does not exist or is uninitialized. + /// - The appropriate object is not encrypted. + /// - The decryption password is incorrect. + /// - The decoding or reading of the data fails (e.g. corrupted or missing segments) pub fn decrypt_object>(&mut self, object_number: u64, decryption_password: P) -> Result { let object_reader = match self.object_reader.get_mut(&object_number) { Some(reader) => reader, @@ -189,7 +252,12 @@ impl ZffReader { Ok(o_type) } - + /// Defines a new preload chunkmap which will be held in memory, if none exists up to this point. + /// This method will (then) only "initialize" a new preload chunkmap. You have to fill this map by using + /// methods like self::preloaded_chunkmap() or self::preload_chunkmap_full(). + /// If there is already a preloaded chunkmap, then this method will + /// - do nothing, if the existing preloaded chunkmap is an in-memory map. + /// - convert the existing preloaded chunkmap to an in-memory chunkmap, if the existing preloaded chunkmap is a redb-based preloaded chunkmap. pub fn set_preload_chunkmap_mode_in_memory(&mut self) -> Result<()> { match &mut self.chunk_map { PreloadedChunkMap::None => self.chunk_map = PreloadedChunkMap::InMemory(HashMap::new()), @@ -199,9 +267,16 @@ impl ZffReader { Ok(()) } + /// Defines a new preload chunkmap which will be cached to a external Redb, if none exists up to this point. + /// This method will (then) only "initialize" a new preload chunkmap. You have to fill this map by using + /// methods like self::preloaded_chunkmap() or self::preload_chunkmap_full(). + /// If there is already a preloaded chunkmap, then this method will + /// - Copy the content of the existing Redb to the new given Redb and use the given Redb as the new one. + /// - Initialize a empty Redb and use this. + /// - convert the existing preloaded (in-memory) chunkmap to the Redb (copy the content) and use the Redb as the appropriate preloaded chunkmap. pub fn set_preload_chunkmap_mode_redb(&mut self, mut db: Database) -> Result<()> { match &self.chunk_map { - PreloadedChunkMap::Redb(_) => return Ok(()), + PreloadedChunkMap::Redb(old_db) => copy_redb_map(&old_db, &mut db)?, PreloadedChunkMap::None => initialize_redb_chunkmap(&mut db, &HashMap::new())?, PreloadedChunkMap::InMemory(map) => initialize_redb_chunkmap(&mut db, map)?, } @@ -209,18 +284,20 @@ impl ZffReader { Ok(()) } - pub fn preload_chunkmap(&mut self, first: u64, last: u64) -> Result<()> { + /// Preloads the offsets of the given [Range](std::ops::Range) of chunks. + /// If no chunkmap was initialized, a new in-memory map will be initialized by using this method. + pub fn preload_chunkmap(&mut self, chunk_numbers: &Range) -> Result<()> { // check if chunk numbers are valid. - if first == 0 { - return Err(ZffError::new(ZffErrorKind::InvalidChunkNumber, first.to_string())); - } else if first > last { + if chunk_numbers.start == 0 { + return Err(ZffError::new(ZffErrorKind::InvalidChunkNumber, chunk_numbers.start.to_string())); + } else if chunk_numbers.start > chunk_numbers.end { return Err(ZffError::new(ZffErrorKind::InvalidChunkNumber, ERROR_LAST_GREATER_FIRST)); - } else if last > self.number_of_chunks() { - return Err(ZffError::new(ZffErrorKind::InvalidChunkNumber, last.to_string())); + } else if chunk_numbers.end > self.number_of_chunks() { + return Err(ZffError::new(ZffErrorKind::InvalidChunkNumber, chunk_numbers.end.to_string())); } //try to reserve the additional size, if PreloadedChunkMap::InMemory is used. - let mut size = last - first; + let mut size = chunk_numbers.end - chunk_numbers.start; match &mut self.chunk_map { PreloadedChunkMap::None => { let mut map = HashMap::new(); @@ -229,7 +306,7 @@ impl ZffReader { }, PreloadedChunkMap::Redb(_) => (), PreloadedChunkMap::InMemory(map) => { - for chunk_no in first..=last { + for chunk_no in chunk_numbers.start..=chunk_numbers.end { if map.contains_key(&chunk_no) { size -= 1; } @@ -238,7 +315,7 @@ impl ZffReader { } } - for chunk_no in first..=last { + for chunk_no in chunk_numbers.start..=chunk_numbers.end { let segment = match get_segment_of_chunk_no(chunk_no, self.main_footer.chunk_maps()) { Some(segment_no) => match self.segments.get_mut(&segment_no) { Some(segment) => segment, @@ -258,12 +335,19 @@ impl ZffReader { Ok(()) } + /// Preloads all offsets of the chunks of the appropriate zff container. pub fn preload_chunkmap_full(&mut self) -> Result<()> { let first = 1; let last = self.number_of_chunks(); - self.preload_chunkmap(first, last) + self.preload_chunkmap(&Range{start: first, end: last}) } + /// Returns the [FileMetadata] of the appropriate active file. + /// # Error + /// May fail if + /// - the active object is not a "logical" object. + /// - the active file number was not set. + /// - no object was set as active. pub fn current_filemetadata(&self) -> Result<&FileMetadata> { match self.object_reader.get(&self.active_object) { Some(ZffObjectReader::Logical(reader)) => { @@ -275,6 +359,12 @@ impl ZffReader { } } + /// Returns the [FileHeader](crate::header::FileHeader) of the appropriate active file. + /// # Error + /// May fail if + /// - the active object is not a "logical" object. + /// - the active file number was not set. + /// - no object was set as active. pub fn current_fileheader(&mut self) -> Result { match self.object_reader.get(&self.active_object) { Some(ZffObjectReader::Logical(reader)) => { @@ -286,6 +376,12 @@ impl ZffReader { } } + /// Returns the [FileFooter](crate::footer::FileFooter) of the appropriate active file. + /// # Error + /// May fail if + /// - the active object is not a "logical" object. + /// - the active file number was not set. + /// - no object was set as active. pub fn current_filefooter(&mut self) -> Result { match self.object_reader.get(&self.active_object) { Some(ZffObjectReader::Logical(reader)) => { @@ -297,19 +393,29 @@ impl ZffReader { } } + /// Returns a reference to the [ObjectHeader](crate::header::ObjectHeader) of the appropriate active object. + /// # Error + /// May fail if + /// - no object was set as active. + /// - the object was not decrypted. pub fn active_object_header_ref(&self) -> Result<&ObjectHeader> { match self.get_active_reader()? { ZffObjectReader::Physical(reader) => Ok(reader.object_header_ref()), ZffObjectReader::Logical(reader) => Ok(reader.object_header_ref()), - ZffObjectReader::Encrypted(_) => Err(ZffError::new(ZffErrorKind::InvalidOption, "")) //TODO: Error msg + ZffObjectReader::Encrypted(_) => Err(ZffError::new(ZffErrorKind::InvalidOption, "")) } } + /// Returns the [ObjectFooter](crate::footer::ObjectFooter) of the appropriate active object. + /// # Error + /// May fail if + /// - no object was set as active. + /// - the object was not decrypted. pub fn active_object_footer(&self) -> Result { match self.get_active_reader()? { ZffObjectReader::Physical(reader) => Ok(reader.object_footer()), ZffObjectReader::Logical(reader) => Ok(reader.object_footer()), - ZffObjectReader::Encrypted(_) => Err(ZffError::new(ZffErrorKind::InvalidOption, "")) //TODO: Error msg + ZffObjectReader::Encrypted(_) => Err(ZffError::new(ZffErrorKind::InvalidOption, "")) } } @@ -320,6 +426,11 @@ impl ZffReader { } } + /// Returns a reference to the [Segment](crate::segment::Segment) of the appropriate segment number. + /// # Error + /// May fail if + /// - the appropriate segment number does not exist. + /// - a decoding error occurs while trying to read the appropriate metadata of the segment. pub fn segment_mut_ref(&mut self, segment_number: u64) -> Result<&mut Segment> { match self.segments.get_mut(&segment_number) { Some(segment) => Ok(segment), @@ -353,10 +464,14 @@ impl Seek for ZffReader { } } +/// An enum, which provides an appropriate object reader. #[derive(Debug)] pub(crate) enum ZffObjectReader { + /// Contains a [ZffObjectReaderPhysical]. Physical(Box), + /// Contains a [ZffObjectReaderLogical]. Logical(Box), + /// Contains a [ZffObjectReaderEncrypted]. Encrypted(Box), } @@ -370,8 +485,10 @@ impl Seek for ZffObjectReader { } } +/// A reader which contains the appropriate metadata of a physical object +/// (e.g. the appropriate [ObjectHeader](crate::header::ObjectHeader) and [ObjectFooter](crate::footer::ObjectFooter)). #[derive(Debug)] -pub(crate) struct ZffObjectReaderPhysical { +pub struct ZffObjectReaderPhysical { object_header: ObjectHeader, object_footer: ObjectFooterPhysical, global_chunkmap: BTreeMap, @@ -379,7 +496,8 @@ pub(crate) struct ZffObjectReaderPhysical { } impl ZffObjectReaderPhysical { - pub(crate) fn with_obj_metadata( + /// creates a new [ZffObjectReaderPhysical] with the given metadata. + pub fn with_obj_metadata( object_header: ObjectHeader, object_footer: ObjectFooterPhysical, global_chunkmap: BTreeMap, //TODO: only used in read_with_segments. Could also be a &-paramter for the specific method? @@ -392,15 +510,18 @@ impl ZffObjectReaderPhysical { } } - pub(crate) fn object_header_ref(&self) -> &ObjectHeader { + /// Returns a reference of the appropriate [ObjectHeader](crate::header::ObjectHeader). + pub fn object_header_ref(&self) -> &ObjectHeader { &self.object_header } - pub(crate) fn object_footer(&self) -> ObjectFooter { + /// Returns the appropriate [ObjectFooter](crate::footer::ObjectFooter). + pub fn object_footer(&self) -> ObjectFooter { ObjectFooter::Physical(self.object_footer.clone()) } - fn read_with_segments( + /// Works like [std::io::Read] for the underlying data, but needs also the segments and the optional preloaded chunkmap. + pub fn read_with_segments( &mut self, buffer: &mut [u8], segments: &mut HashMap>, @@ -469,6 +590,8 @@ impl Seek for ZffObjectReaderPhysical { } } +/// A reader which contains the appropriate metadata of a logical object +/// (e.g. the appropriate [ObjectHeader](crate::header::ObjectHeader) and [ObjectFooter](crate::footer::ObjectFooter)). #[derive(Debug)] pub struct ZffObjectReaderLogical { object_header: ObjectHeader, @@ -479,6 +602,7 @@ pub struct ZffObjectReaderLogical { } impl ZffObjectReaderLogical { + /// Initialize the [ZffObjectReaderLogical] with a minimal set of (the absolutly required) metadata which will be stored in memory. pub fn with_obj_metadata_minimal( object_header: ObjectHeader, object_footer: ObjectFooterLogical, @@ -488,6 +612,7 @@ impl ZffObjectReaderLogical { Self::with_obj_metadata(object_header, object_footer, segments, global_chunkmap, PreloadDegree::Minimal) } + /// Initialize the [ZffObjectReaderLogical] with the recommended set of metadata which will be stored in memory. pub fn with_obj_metadata_recommended( object_header: ObjectHeader, object_footer: ObjectFooterLogical, @@ -497,6 +622,7 @@ impl ZffObjectReaderLogical { Self::with_obj_metadata(object_header, object_footer, segments, global_chunkmap, PreloadDegree::Recommended) } + /// Initialize the [ZffObjectReaderLogical] which will store all metadata in memory. pub fn with_obj_metadata_all( object_header: ObjectHeader, object_footer: ObjectFooterLogical, @@ -506,14 +632,17 @@ impl ZffObjectReaderLogical { Self::with_obj_metadata(object_header, object_footer, segments, global_chunkmap, PreloadDegree::All) } + /// Returns a reference of the appropriate [ObjectHeader](crate::header::ObjectHeader). pub(crate) fn object_header_ref(&self) -> &ObjectHeader { &self.object_header } + /// Returns the appropriate [ObjectFooter](crate::footer::ObjectFooter). pub(crate) fn object_footer(&self) -> ObjectFooter { ObjectFooter::Logical(self.object_footer.clone()) } + /// Returns the appropriate [FileHeader](crate::header::FileHeader) of the current active file. pub fn current_fileheader(&self, segments: &mut HashMap>) -> Result { let header_segment_number = match self.object_footer.file_header_segment_numbers().get(&self.active_file) { Some(no) => no, @@ -546,6 +675,7 @@ impl ZffObjectReaderLogical { } } + /// Returns the appropriate [FileFooter](crate::footer::FileFooter) of the current active file. pub fn current_filefooter(&self, segments: &mut HashMap>) -> Result { let footer_segment_number = match self.object_footer.file_footer_segment_numbers().get(&self.active_file) { Some(no) => no, @@ -651,6 +781,7 @@ impl ZffObjectReaderLogical { }) } + /// Sets the file of the appropriate filenumber active. /// # Error /// fails if no appropriate file for the given filenumber exists. pub fn set_active_file(&mut self, filenumber: u64) -> Result<()> { @@ -668,7 +799,8 @@ impl ZffObjectReaderLogical { } } - fn read_with_segments( + /// Works like [std::io::Read] for the underlying data, but needs also the segments and the optional preloaded chunkmap. + pub fn read_with_segments( &mut self, buffer: &mut [u8], segments: &mut HashMap>, @@ -762,6 +894,8 @@ enum PreloadDegree { All, } +/// A reader which contains the appropriate metadata of a encrypted object +/// (e.g. the appropriate [ObjectHeader](crate::header::ObjectHeader) and [ObjectFooter](crate::footer::ObjectFooter)). #[derive(Debug)] pub(crate) struct ZffObjectReaderEncrypted { encrypted_header: EncryptedObjectHeader, @@ -800,22 +934,44 @@ impl ZffObjectReaderEncrypted { } } +/// The Metadata of a [File](crate::file::File). #[derive(Debug, Clone, Eq, PartialEq)] pub struct FileMetadata { + /// The file number of the parent directory (0 if the parent directory is the root directory). pub parent_file_number: u64, + /// The length of the file in bytes. pub length_of_data: u64, + /// The first chunk number used by this file. pub first_chunk_number: u64, + /// The number of all chunks which are used for this file. pub number_of_chunks: u64, + /// Position of the internal reader. This is mostly internally used. pub position: u64, + /// The appropriate type of the file. pub file_type: FileType, + /// The appropriate filename. pub filename: Option, + /// The metadata of the appropriate file. pub metadata_ext: HashMap, + /// The timestamp when the acquisition has started. pub acquisition_start: Option, + /// The timestamp when the acquisition has ended. pub acquisition_end: Option, + /// The appropriate hash header of the file. pub hash_header: Option, } impl FileMetadata { + /// Creates the [FileMetadata] with minimum amount of data. Most optional fields will be "None" and have to + /// read directly from zff container. + /// This Method will reduce the memory usage in the most possible way. + /// This Option will provide: + /// - the parent file number + /// - the length (or size) of the file + /// - the first chunk number + /// - the number of chunks + /// - the internally used reader position + /// - the filetype pub fn with_header_minimal(fileheader: &FileHeader, filefooter: &FileFooter) -> Self { Self { parent_file_number: fileheader.parent_file_number, @@ -832,6 +988,18 @@ impl FileMetadata { } } + /// Creates the [FileMetadata] with recommended amount of data. Most optional fields will be "None" and have to + /// read directly from zff container. + /// This Method will reduce the memory usage a bit. + /// This Option will provide: + /// - the parent file number + /// - the length (or size) of the file + /// - the first chunk number + /// - the number of chunks + /// - the internally used reader position + /// - the filetype + /// - the filename + /// - the metadata of the file pub fn with_header_recommended(fileheader: &FileHeader, filefooter: &FileFooter) -> Self { Self { parent_file_number: fileheader.parent_file_number, @@ -848,6 +1016,20 @@ impl FileMetadata { } } + /// Creates the [FileMetadata] with recommended amount of data. Most optional fields will be "None" and have to + /// read directly from zff container. + /// This Method will reduce the need of I/O access in the most possible way. + /// This Option will provide: + /// - the parent file number + /// - the length (or size) of the file + /// - the first chunk number + /// - the number of chunks + /// - the internally used reader position + /// - the filetype + /// - the filename + /// - the metadata of the file + /// - the timestamps of start and end of the acquisition + /// - the appropriate hash header pub fn with_header_all(fileheader: &FileHeader, filefooter: &FileFooter) -> Self { Self { parent_file_number: fileheader.parent_file_number, @@ -1089,6 +1271,24 @@ fn extract_redb_map(db: &mut Database) -> Result> { Ok(new_map) } +// Will copy a redb to another redb. +fn copy_redb_map(input_db: &Database, output_db: &mut Database) -> Result<()> { + // prepare read context of input_db + let read_txn = input_db.begin_read()?; + let read_table = read_txn.open_table(PRELOADED_CHUNK_MAP_TABLE)?; + let mut table_iterator = read_table.iter()?; + + // prepare write context of output_db + let write_txn = output_db.begin_write()?; + let mut write_table = write_txn.open_table(PRELOADED_CHUNK_MAP_TABLE)?; + + while let Some(data) = table_iterator.next_back() { + let (key, value) = data?; + write_table.insert(key.value(), value.value())?; + } + Ok(()) +} + fn initialize_redb_chunkmap(db: &mut Database, map: &HashMap) -> Result<()> { let write_txn = db.begin_write()?; { diff --git a/src/lib/io/zffwriter.rs b/src/lib/io/zffwriter.rs index df45227..1d5c78c 100644 --- a/src/lib/io/zffwriter.rs +++ b/src/lib/io/zffwriter.rs @@ -47,23 +47,36 @@ use super::{ // - external use ed25519_dalek::{SigningKey}; +/// Defines the output for a [ZffWriter]. +/// This enum determine, that the [ZffWriter] will extend or build a new Zff container. pub enum ZffWriterOutput { + /// Build a new container by using the appropriate Path-prefix + /// (e.g. if "/home/user/zff_container" is given, "/home/user/zff_container.z??" will be used). NewContainer(PathBuf), + /// Determine an extension of the given zff container (path). ExtendContainer(Vec), } -/// struct contains optional, additional parameter. -#[derive(Default)] +/// This struct contains optional, additional parameter for the [ZffWriter]. +#[derive(Default, Debug)] pub struct ZffWriterOptionalParameter { + /// If given, the appropriate data will be signed by the given [SigningKey](crate::ed25519_dalek::SigningKey). pub signature_key: Option, - pub target_segment_size: Option, //if None, the container will not be segmentized. + /// If None, the container will not be segmentized. Otherwise, [ZffWriter] ensure that no segment will be larger than this size. + pub target_segment_size: Option, + /// An optional description for the container + /// (note: you can describe every object with custom descriptions by using the [DescriptionHeader](crate::header::DescriptionHeader)). pub description_notes: Option, + /// If set, the chunkmaps will not grow larger than the given size. Otherwise, the default size 32k will be used. pub chunkmap_size: Option, //default is 32k + /// Optional [DeduplicationChunkMap](crate::header::DeduplicationChunkMap) to ensure a chunk deduplication (and safe some disk space). pub deduplication_chunkmap: Option, - pub unique_identifier: u64 // TODO: set a random number, if zero? + /// Will be used as a unique identifier, to assign each segment to the appropriate zff container. + /// If the [ZffWriter] will be extend an existing Zff container, this value will be ignored. + pub unique_identifier: u64 } -pub struct ZffExtenderParameter { +struct ZffExtenderParameter { pub main_footer: MainFooter, pub current_segment: PathBuf, pub next_object_no: u64, @@ -71,7 +84,7 @@ pub struct ZffExtenderParameter { } impl ZffExtenderParameter { - pub fn with_data( + fn with_data( main_footer: MainFooter, current_segment: PathBuf, next_object_no: u64, @@ -132,6 +145,7 @@ impl ZffWriter { ZffWriterOutput::NewContainer(_) => return Err(ZffError::new(ZffErrorKind::InvalidOption, ERROR_INVALID_OPTION_ZFFCREATE)), //TODO, ZffWriterOutput::ExtendContainer(ref files_to_extend) => files_to_extend.clone() }; + let mut params = params; for ext_file in &files_to_extend { let mut raw_segment = File::open(ext_file)?; if let Ok(mf) = decode_main_footer(&mut raw_segment) { @@ -152,6 +166,8 @@ impl ZffWriter { Some(x) => *x + 1, None => return Err(ZffError::new(ZffErrorKind::NoObjectsLeft, "")), }; + let unique_identifier = segment.header().unique_identifier; + params.unique_identifier = unique_identifier; let extension_parameter = ZffExtenderParameter::with_data( mf, diff --git a/src/lib/mod.rs b/src/lib/mod.rs index 0407db9..b8fd0f7 100644 --- a/src/lib/mod.rs +++ b/src/lib/mod.rs @@ -1,5 +1,6 @@ #![forbid(unsafe_code)] -//#![deny(missing_docs)] +#![deny(missing_docs)] +#![deny(warnings)] //! This crate provides the reference implementation of the forensic file format Zff. //! Zff is a new file format for forensic images, as an alternative to EWF and AFF. //! Zff is focused on speed and security. If you want to learn more about ZFF, visit [https://github.com/ph0llux/zff](https://github.com/ph0llux/zff). @@ -11,6 +12,8 @@ pub mod constants; pub mod header; /// This module contains all footer, could be found in the zff specification (footer version 1 and footer version 2). pub mod footer; +/// Contains several stuff to handle zff container (e.g. create, extend or read zff container). +pub mod io; mod hashing; mod compression; mod encryption; @@ -22,7 +25,6 @@ mod object; mod file; mod segment; mod chunk; -pub mod io; // - re-exports pub use hashing::*; diff --git a/src/lib/object/encoder.rs b/src/lib/object/encoder.rs index 7bbbed6..6c6dc75 100644 --- a/src/lib/object/encoder.rs +++ b/src/lib/object/encoder.rs @@ -117,17 +117,10 @@ impl ObjectEncoder { pub struct PhysicalObjectEncoder { /// The appropriate object header obj_header: ObjectHeader, - /// remaining bytes of the encoded header to read. This is only (internally) used, if you will use the [Read] implementation of [PhysicalObjectEncoder]. - encoded_header_remaining_bytes: usize, underlying_data: R, read_bytes_underlying_data: u64, - /// data of current chunk (only used in Read implementation) - current_chunked_data: Option>, - current_chunked_data_remaining_bytes: usize, current_chunk_number: u64, initial_chunk_number: u64, - encoded_footer: Vec, - encoded_footer_remaining_bytes: usize, hasher_map: HashMap>, signing_key: Option, has_hash_signatures: bool, @@ -150,7 +143,7 @@ impl PhysicalObjectEncoder { None => None }; - let (encoded_header, encryption_key) = if let Some(encryption_header) = &obj_header.encryption_header { + let (_, encryption_key) = if let Some(encryption_header) = &obj_header.encryption_header { match encryption_header.get_encryption_key() { Some(key) => (obj_header.encode_encrypted_header_directly(&key)?, Some(key)), None => return Err(ZffError::new(ZffErrorKind::MissingEncryptionKey, obj_header.object_number.to_string())) @@ -167,15 +160,10 @@ impl PhysicalObjectEncoder { Ok(Self { has_hash_signatures: obj_header.has_hash_signatures(), obj_header, - encoded_header_remaining_bytes: encoded_header.len(), underlying_data: reader, read_bytes_underlying_data: 0, - current_chunked_data: None, - current_chunked_data_remaining_bytes: 0, current_chunk_number, initial_chunk_number: current_chunk_number, - encoded_footer: Vec::new(), - encoded_footer_remaining_bytes: 0, hasher_map, encryption_key, signing_key, @@ -200,7 +188,8 @@ impl PhysicalObjectEncoder { self.current_chunk_number } - /// Returns the encoded object header. A call of this method sets the acquisition start time to the current time. + /// Returns the encoded object header. + /// Note: **A call of this method sets the acquisition start time to the current time**. pub fn get_encoded_header(&mut self) -> Vec { if self.acquisition_start == 0 { self.acquisition_start = OffsetDateTime::from(SystemTime::now()).unix_timestamp() as u64; @@ -340,79 +329,6 @@ impl PhysicalObjectEncoder { } } -/*/// This implement Read for [PhysicalObjectEncoder]. This implementation should only used for a single zff segment file (e.g. in http streams). -impl Read for PhysicalObjectEncoder { - fn read(&mut self, buf: &mut [u8]) -> std::result::Result { - let mut read_bytes = 0; - //read encoded header, if there are remaining bytes to read. - if let remaining_bytes @ 1.. = self.encoded_header_remaining_bytes { - let mut inner_read_bytes = 0; - let mut inner_cursor = Cursor::new(self.get_encoded_header()); - inner_cursor.seek(SeekFrom::End(-(remaining_bytes as i64)))?; - inner_read_bytes += inner_cursor.read(&mut buf[read_bytes..])?; - self.encoded_header_remaining_bytes -= inner_read_bytes; - read_bytes += inner_read_bytes; - } - loop { - if read_bytes == buf.len() { - self.encoded_footer = match self.get_encoded_footer() { - Ok(footer) => footer, - Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())), - }; - self.encoded_footer_remaining_bytes = self.encoded_footer.len(); - break; - }; - match &self.current_chunked_data { - Some(data) => { - let mut inner_read_bytes = 0; - let mut inner_cursor = Cursor::new(&data); - inner_cursor.seek(SeekFrom::End(-(self.current_chunked_data_remaining_bytes as i64)))?; - inner_read_bytes += inner_cursor.read(&mut buf[read_bytes..])?; - self.current_chunked_data_remaining_bytes -= inner_read_bytes; - if self.current_chunked_data_remaining_bytes < 1 { - self.current_chunked_data = None; - } - read_bytes += inner_read_bytes; - }, - None => { - match self.get_next_chunk() { - Ok(chunk) => { - self.current_chunked_data_remaining_bytes = chunk.len(); - self.current_chunked_data = Some(chunk); - }, - Err(e) => match e.unwrap_kind() { - ZffErrorKind::ReadEOF => break, - ZffErrorKind::IoError(ioe) => return Err(ioe), - e => return Err(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())), - } - } - } - } - } - //read encoded footer, if there are remaining bytes to read. - if let remaining_bytes @ 1.. = self.encoded_footer_remaining_bytes { - let mut inner_read_bytes = 0; - let mut inner_cursor = Cursor::new(&self.encoded_footer); - inner_cursor.seek(SeekFrom::End(-(remaining_bytes as i64)))?; - inner_read_bytes += inner_cursor.read(&mut buf[read_bytes..])?; - self.encoded_footer_remaining_bytes -= inner_read_bytes; - read_bytes += inner_read_bytes; - } - Ok(read_bytes) - } -}*/ - -pub enum FileRessource { - ByPath(PathBuf), - ByReader(Box) -} - -impl From for FileRessource { - fn from(path: PathBuf) -> Self { - FileRessource::ByPath(path) - } -} - /// The [LogicalObjectEncoder] can be used to encode a logical object. pub struct LogicalObjectEncoder { /// The appropriate original object header @@ -454,7 +370,7 @@ impl LogicalObjectEncoder { } - pub fn add_unaccessable_file(&mut self, file_path: F) + pub(crate) fn add_unaccessable_file(&mut self, file_path: F) where F: AsRef { @@ -474,7 +390,7 @@ impl LogicalObjectEncoder { directory_children: HashMap>, current_chunk_number: u64) -> Result { - //test if the encryption is successful. + // ensures that the encryption key is available in decrypted form. let (_encoded_header, encryption_key) = if let Some(encryption_header) = &obj_header.encryption_header { match encryption_header.get_encryption_key() { Some(key) => (obj_header.encode_encrypted_header_directly(&key)?, Some(key)), @@ -484,7 +400,7 @@ impl LogicalObjectEncoder { (obj_header.encode_directly(), None) }; - let mut files = files; //TODO: ???? + let mut files = files; let (reader, current_file_header) = match files.pop() { Some((file, header)) => (file, header), None => return Err(ZffError::new(ZffErrorKind::NoFilesLeft, "There is no input file")) @@ -496,12 +412,7 @@ impl LogicalObjectEncoder { Some(children) => children.to_owned(), None => Vec::new() }; - let signing_key = match &signing_key_bytes { - Some(bytes) => Some(Signature::bytes_to_signingkey(bytes)?), - None => None - }; - let encryption_information = if let Some(encryption_key) = &encryption_key { obj_header.encryption_header.clone().map(|enc_header| EncryptionInformation::new(encryption_key.to_vec(), enc_header.algorithm().clone())) } else { @@ -516,7 +427,6 @@ impl LogicalObjectEncoder { reader, hash_types.clone(), encryption_information, - signing_key, current_chunk_number, symlink_real_path, hardlink_filenumber, @@ -640,10 +550,6 @@ impl LogicalObjectEncoder { Some(children) => children.to_owned(), None => Vec::new(), }; - let signing_key = match &self.signing_key_bytes { - Some(bytes) => Some(Signature::bytes_to_signingkey(bytes)?), - None => None - }; let encryption_information = if let Some(encryption_key) = &self.encryption_key { self.obj_header.encryption_header.as_ref().map(|enc_header| EncryptionInformation::new(encryption_key.to_vec(), enc_header.algorithm().clone())) @@ -658,7 +564,6 @@ impl LogicalObjectEncoder { reader, self.hash_types.clone(), encryption_information, - signing_key, self.current_chunk_number, symlink_real_path, hardlink_filenumber, diff --git a/src/lib/segment.rs b/src/lib/segment.rs index aaa2bec..c200dbc 100644 --- a/src/lib/segment.rs +++ b/src/lib/segment.rs @@ -23,8 +23,8 @@ use crate::{ }; /// Represents a full [Segment], containing a [crate::header::SegmentHeader], -/// a [crate::footer::SegmentFooter], a [Reader](std::io::Read) to the appropriate -/// segmented data and a position marker for this [Reader](std::io::Read). +/// a [crate::footer::SegmentFooter], a [Read](std::io::Read)er to the appropriate +/// segmented data and a position marker for this [Read](std::io::Read)er. #[derive(Debug)] pub struct Segment { header: SegmentHeader, @@ -44,7 +44,7 @@ impl Segment { } } - /// Creates a new [Segment] from the given [Reader](std::io::Read). + /// Creates a new [Segment] from the given [Read](std::io::Read)er. pub fn new_from_reader(mut data: R) -> Result> { let segment_header = SegmentHeader::decode_directly(&mut data)?; @@ -75,13 +75,14 @@ impl Segment { &self.footer } - /// Returns the raw chunk, if so present, then also encrypted and/or compressed. + /// Returns the raw chunk (and if so present, then also in encrypted and/or compressed form). pub fn raw_chunk(&mut self, chunk_number: u64) -> Result { let chunk_offset = self.get_chunk_offset(&chunk_number)?; self.data.seek(SeekFrom::Start(chunk_offset))?; Chunk::new_from_reader(&mut self.data) } + /// Returns the offset of the appropriate chunk (number). pub fn get_chunk_offset(&mut self, chunk_number: &u64) -> Result { let chunkmap_offset = get_chunkmap_offset(&self.footer.chunk_map_table, *chunk_number)?; //get the first chunk number of the specific chunk map @@ -104,9 +105,9 @@ impl Segment { Ok(offset) } - /// Returns the chunked data, uncompressed and unencrypted - /// The chunk offset could be optionally assigned directly, e.g. from a preloaded chunk map - pub fn chunk_data(&mut self, + /// Returns the chunked data, uncompressed and unencrypted. + /// The chunk offset could be optionally assigned directly, e.g. from a preloaded chunk map. + pub(crate) fn chunk_data(&mut self, chunk_number: u64, encryption_information: &Option, compression_algorithm: C, @@ -180,6 +181,7 @@ impl Segment { Ok(object_header) } + /// Returns the [EncryptedObjectHeader] of the given object number (if available in this [Segment]). Otherwise, returns an error. pub fn read_encrypted_object_header(&mut self, object_number: u64) -> Result { let offset = match self.footer.object_header_offsets().get(&object_number) { Some(value) => value, @@ -216,6 +218,7 @@ impl Segment { ObjectFooter::decode_directly(&mut self.data) } + /// Returns the [crate::footer::EncryptedObjectFooter] of the given object number, if available in this [Segment]. Otherwise, returns an error. pub fn read_encrypted_object_footer(&mut self, object_number: u64) -> Result { let offset = match self.footer.object_footer_offsets().get(&object_number) { Some(value) => value, @@ -264,11 +267,14 @@ impl Seek for Segment { } } - -pub enum ChunkContent { - Raw(Vec), // contains original data, - SameBytes(u8), // contains the appropriate byte, - Duplicate(u64), //contains the appropriate chunk with the original data +/// The data of the chunk. +pub(crate) enum ChunkContent { + /// The unencrypted and uncompressed original data of the chunk. + Raw(Vec), + /// The appropriate byte, if the same byte flag is set. + SameBytes(u8), + /// The appropriate chunk, if this chunk is a duplication. + Duplicate(u64), } diff --git a/src/lib/signatures.rs b/src/lib/signatures.rs index 72d1979..f8f1e0b 100644 --- a/src/lib/signatures.rs +++ b/src/lib/signatures.rs @@ -11,6 +11,7 @@ use ed25519_dalek::{ }; use rand::rngs::OsRng; use rand::RngCore; +use base64::{Engine, engine::general_purpose::STANDARD as base64engine}; // - internal use crate::{ @@ -20,6 +21,7 @@ use crate::{ ED25519_DALEK_SIGNATURE_LEN, }; + /// structure contains serveral methods to handle signing of chunked data. pub struct Signature; @@ -36,7 +38,7 @@ impl Signature { /// Input data can be a secret key (32 bytes) or a secret/public keypair (64 bytes). pub fn new_signingkey_from_base64>(key: K) -> Result { //decodes the base64 content. - let key = base64::decode(key.into())?; + let key = base64engine.decode(key.into())?; // check if the content is a keypair or a secret key if key.len() == KEYPAIR_LENGTH { let mut key_slice = [0u8; KEYPAIR_LENGTH];