New syncing logic (#63)

* Use millisecond resolution when comparing timestamps

* Use nanos

* Round down scan time

* Use micros

* Update syncing logic with some api changes

* Abstract loading from fs logic

* Add full syncing logic

* Fix need syncing logic

* Address comments

* Fix lints

* add a 1sec sleep

Signed-off-by: pushkarm029 <[email protected]>

* check

Signed-off-by: pushkarm029 <[email protected]>

* fmt

Signed-off-by: pushkarm029 <[email protected]>

* Update fs-storage/src/file_storage.rs

Co-authored-by: Kirill Taran <[email protected]>

* Update fs-storage/src/file_storage.rs

Co-authored-by: Kirill Taran <[email protected]>

* Update fs-storage/src/file_storage.rs

Co-authored-by: Kirill Taran <[email protected]>

* Add doc comments on FileStorage fields

* Weird bug in file metadata modified timestamp returned by OS (#71)

* File modified metadata bug

* Add even smaller test

* fix(fs-storage): flush the buffer in write_fs()

Signed-off-by: Tarek <[email protected]>

---------

Signed-off-by: Tarek <[email protected]>
Co-authored-by: Tarek <[email protected]>

* Use file writer and update test

* cargo fix

* Weird bug in file metadata again

* Update fs-storage/src/file_storage.rs

Co-authored-by: Tarek Elsayed <[email protected]>

* Update fs-storage/src/base_storage.rs

Co-authored-by: Tarek Elsayed <[email protected]>

* Refactor sync_status to cleaner match case

Co-authored-by: Tarek <[email protected]>

* Add MRE for file metadata timestamp not updated on write_fs

* fmt fix

* fix(fs-storage): add delay in tests after file write

Signed-off-by: Tarek <[email protected]>

* Set file modified timestamp from application layer

* Fmt fix

* Update fs-storage/src/file_storage.rs

Co-authored-by: Tarek Elsayed <[email protected]>

* Resolve comments

* Fix fmt

---------

Signed-off-by: pushkarm029 <[email protected]>
Signed-off-by: Tarek <[email protected]>
Co-authored-by: pushkarm029 <[email protected]>
Co-authored-by: Kirill Taran <[email protected]>
Co-authored-by: Tarek <[email protected]>
Co-authored-by: Tarek Elsayed <[email protected]>

fs-storage/examples/cli.rs

@@ -44,18 +44,19 @@ fn read_command(args: &[String], path: &str) -> Result<()> {
     };
 
     let mut fs: FileStorage<String, String> =
-        FileStorage::new("cli".to_string(), Path::new(path));
+        FileStorage::new("cli".to_string(), Path::new(path))
+            .context("Failed to create FileStorage")?;
 
     let map = fs
         .read_fs()
         .expect("No Data is present on this path");
     if keys.is_empty() {
-        for (key, value) in &map {
+        for (key, value) in map {
             println!("{}: {}", key, value);
         }
     }
     for key in &keys {
-        if let Some(value) = &map.get(key) {
+        if let Some(value) = map.get(key) {
             println!("{}: {}", key, value);
         } else {
             eprintln!("Key '{}' not found", key);
@@ -78,7 +79,8 @@ fn write_command(args: &[String], path: &str) -> Result<()> {
         .map_or(false, |ext| ext == "json");
 
     let mut fs: FileStorage<String, String> =
-        FileStorage::new("cli".to_string(), Path::new(path));
+        FileStorage::new("cli".to_string(), Path::new(path))
+            .context("Failed to create FileStorage")?;
     if content_json {
         let content =
             fs::read_to_string(content).context("Failed to read JSON file")?;

fs-storage/src/base_storage.rs

@@ -1,37 +1,64 @@
 use data_error::Result;
 use std::collections::BTreeMap;
 
+#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Clone)]
+/// Represents the synchronization status of the storage.
+pub enum SyncStatus {
+    /// No synchronization needed.
+    InSync,
+    /// In-memory key-value mapping is stale.
+    MappingStale,
+    /// External file system storage is stale.
+    StorageStale,
+    /// In-memory key-value mapping and external file system storage diverge.
+    Diverge,
+}
+
+impl std::fmt::Display for SyncStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SyncStatus::InSync => write!(f, "InSync"),
+            SyncStatus::MappingStale => write!(f, "MappingStale"),
+            SyncStatus::StorageStale => write!(f, "StorageStale"),
+            SyncStatus::Diverge => write!(f, "Diverge"),
+        }
+    }
+}
+
+/// The `BaseStorage` trait represents a key-value mapping that is written to the file system.
+///
+/// This trait provides methods to create or update entries in the internal mapping, remove entries from the internal mapping,
+/// determine if the in-memory model or the underlying storage requires syncing, scan and load the mapping from the filesystem,
+/// write the mapping to the filesystem, and remove all stored data.
+///
+/// The trait also includes a method to merge values from another key-value mapping.
+///
+/// Note: The trait does not write to storage by default. It is up to the implementor to decide when to read or write to storage
+/// based on `SyncStatus`. This is to allow for trading off between performance and consistency.
 pub trait BaseStorage<K, V>: AsRef<BTreeMap<K, V>> {
     /// Create or update an entry in the internal mapping.
     fn set(&mut self, id: K, value: V);
 
     /// Remove an entry from the internal mapping.
     fn remove(&mut self, id: &K) -> Result<()>;
 
-    /// Determine if in-memory model
-    /// or the underlying storage requires syncing.
-    /// This is a quick method checking timestamps
-    /// of modification of both model and storage.
-    ///
-    /// Returns:
-    /// - `Ok(true)` if the on-disk data and in-memory data are not in sync.
-    /// - `Ok(false)` if the on-disk data and in-memory data are in sync.
-    /// - `Err(ArklibError::Storage)` in case of any error retrieving the file metadata.
-    fn needs_syncing(&self) -> Result<bool>;
+    /// Get [`SyncStatus`] of the storage
+    fn sync_status(&self) -> Result<SyncStatus>;
+
+    /// Sync the in-memory storage with the storage on disk
+    fn sync(&mut self) -> Result<()>;
 
     /// Scan and load the key-value mapping
     /// from pre-configured location in the filesystem.
-    fn read_fs(&mut self) -> Result<BTreeMap<K, V>>;
+    fn read_fs(&mut self) -> Result<&BTreeMap<K, V>>;
 
-    /// Persist the internal key-value mapping
+    /// Write the internal key-value mapping
     /// to pre-configured location in the filesystem.
     fn write_fs(&mut self) -> Result<()>;
 
-    /// Remove all persisted data
-    /// by pre-configured location in the file-system.
+    /// Erase data stored on the filesystem
     fn erase(&self) -> Result<()>;
 
-    /// Merge two storages instances
-    /// and write the result to the filesystem.
+    /// Merge values from another key-value mapping.
     fn merge_from(&mut self, other: impl AsRef<BTreeMap<K, V>>) -> Result<()>;
 }