Merge 27d8c19 into ddec4cf

.github/workflows/build.yml

@@ -22,6 +22,7 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
+          toolchain: nightly # nightly is required for fmt
           components: rustfmt, clippy
 
       - name: Check
@@ -36,7 +37,9 @@ jobs:
         run: cargo build --verbose
 
       - name: Run tests
-        run: cargo test --verbose
+        run: |
+          cargo test --verbose
+          cargo test --verbose
 
       - name: Build Release
         run: cargo build --verbose --release
@@ -68,7 +71,9 @@ jobs:
         run: cargo build --verbose --release
 
       - name: Run tests
-        run: cargo test --workspace --verbose
+        run: |
+          cargo test --workspace --verbose
+          cargo test --workspace --verbose
 
       - name: Install JDK
         uses: actions/[email protected]
@@ -97,7 +102,9 @@ jobs:
         run: cargo build --verbose --release
 
       - name: Run tests
-        run: cargo test --workspace --verbose
+        run: |
+          cargo test --workspace --verbose
+          cargo test --workspace --verbose
 
       - name: Install JDK
         uses: actions/[email protected]

.gitignore

@@ -2,4 +2,5 @@
 target
 Cargo.lock
 **/app_id
-*.class
+*.class
+**/.ark

README.md

@@ -76,7 +76,7 @@ cargo bench
 This command runs all benchmarks and generates a report in HTML format located at `target/criterion/report`. If you wish to run a specific benchmark, you can specify its name as an argument as in:
 
 ```bash
-cargo bench index_build
+cargo bench resource_index
 ```
 
 ### Benchmarking Local Files
@@ -97,10 +97,10 @@ To install `flamegraph`, run:
 cargo install flamegraph
 ```
 
-To generate a flame graph for `index_build_benchmark`, use the following command:
+To generate a flame graph for `resource_index_benchmark`, use the following command:
 
 ```bash
-cargo flamegraph --bench index_build_benchmark -o index_build_benchmark.svg -- --bench
+cargo flamegraph --bench resource_index_benchmark -o resource_index_benchmark.svg -- --bench
 ```
 
 > [!NOTE]

ark-cli/src/commands/backup.rs

@@ -1,5 +1,4 @@
-use std::io::Write;
-use std::path::PathBuf;
+use std::{io::Write, path::PathBuf};
 
 use crate::{
     create_dir_all, dir, discover_roots, home_dir, storages_exists, timestamp,

ark-cli/src/commands/file/append.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-use std::str::FromStr;
+use std::{path::PathBuf, str::FromStr};
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError, Format, ResourceId,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError, Format, ResourceId,
 };
 
 use data_error::ArklibError;

ark-cli/src/commands/file/insert.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-use std::str::FromStr;
+use std::{path::PathBuf, str::FromStr};
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError, Format, ResourceId,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError, Format, ResourceId,
 };
 
 use data_error::ArklibError;

ark-cli/src/commands/file/read.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-use std::str::FromStr;
+use std::{path::PathBuf, str::FromStr};
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError, ResourceId,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError, ResourceId,
 };
 
 use data_error::ArklibError;

ark-cli/src/commands/file/utils.rs

@@ -1,6 +1,7 @@
-use crate::error::AppError;
-use crate::models::key_value_to_str;
-use crate::models::Format;
+use crate::{
+    error::AppError,
+    models::{key_value_to_str, Format},
+};
 use data_error::Result as ArklibResult;
 use fs_atomic_versions::atomic::{modify, modify_json, AtomicFile};
 

ark-cli/src/commands/link/utils.rs

@@ -3,8 +3,7 @@ use data_link::Link;
 use std::path::PathBuf;
 use url::Url;
 
-use crate::error::AppError;
-use crate::util::provide_index; // Import your custom AppError type
+use crate::{error::AppError, util::provide_index}; // Import your custom AppError type
 
 pub async fn create_link(
     root: &PathBuf,
@@ -28,8 +27,17 @@ pub fn load_link(
 ) -> Result<Link<ResourceId>, AppError> {
     let path_from_index = id.clone().map(|id| {
         let index = provide_index(root);
-        index.id2path[&id].as_path().to_path_buf()
+        index
+            .get_resources_by_id(&id)
+            .map(|r| r[0].path().to_owned())
+            .ok_or_else(|| {
+                AppError::IndexError(format!(
+                    "Resource with id {} not found",
+                    id
+                ))
+            })
     });
+    let path_from_index = path_from_index.transpose()?;
     let path_from_user = file_path;
 
     let path = match (path_from_user, path_from_index) {
@@ -46,7 +54,7 @@ pub fn load_link(
             }
         }
         (Some(path), None) => Ok(path.to_path_buf()),
-        (None, Some(path)) => Ok(path),
+        (None, Some(path)) => Ok(path.to_path_buf()),
         (None, None) => Err(AppError::LinkLoadError(
             "Provide a path or id for request.".to_owned(),
         ))?,

ark-cli/src/commands/list.rs

@@ -1,5 +1,4 @@
-use std::io::Read;
-use std::path::PathBuf;
+use std::{io::Read, path::PathBuf};
 
 use crate::{
     provide_index, provide_root, read_storage_value, AppError, DateTime,
@@ -76,15 +75,17 @@ impl List {
             .map_err(|_| {
                 AppError::IndexError("Could not read index".to_owned())
             })?
-            .path2id
+            .resources()
             .iter()
-            .filter_map(|(path, resource)| {
+            .filter_map(|indexed_resource| {
+                let path = indexed_resource.path();
+                let id = indexed_resource.id();
                 let tags = if self.tags {
                     Some(
                         read_storage_value(
                             &root,
                             "tags",
-                            &resource.id.to_string(),
+                            &id.to_string(),
                             &None,
                         )
                         .map_or(vec![], |s| {
@@ -102,7 +103,7 @@ impl List {
                         read_storage_value(
                             &root,
                             "scores",
-                            &resource.id.to_string(),
+                            &id.to_string(),
                             &None,
                         )
                         .map_or(0, |s| s.parse::<u32>().unwrap_or(0)),
@@ -114,7 +115,7 @@ impl List {
                 let datetime = if self.modified {
                     let format = "%b %e %H:%M %Y";
                     Some(
-                        DateTime::<Utc>::from(resource.modified)
+                        DateTime::<Utc>::from(indexed_resource.last_modified())
                             .format(format)
                             .to_string(),
                     )
@@ -123,21 +124,18 @@ impl List {
                 };
 
                 let (path, resource, content) = match entry_output {
-                    EntryOutput::Both => (
-                        Some(path.to_owned().into_path_buf()),
-                        Some(resource.clone().id),
-                        None,
-                    ),
-                    EntryOutput::Path => {
-                        (Some(path.to_owned().into_path_buf()), None, None)
+                    EntryOutput::Both => {
+                        (Some(path.to_owned()), Some(id.to_owned()), None)
                     }
-                    EntryOutput::Id => (None, Some(resource.clone().id), None),
+                    EntryOutput::Path => (Some(path.to_owned()), None, None),
+                    EntryOutput::Id => (None, Some(id.to_owned()), None),
                     EntryOutput::Link => match File::open(path) {
                         Ok(mut file) => {
                             let mut contents = String::new();
                             match file.read_to_string(&mut contents) {
                                 Ok(_) => {
-                                    // Check if the content of the file is a valid url
+                                    // Check if the content
+                                    // of the file is a valid url
                                     let url = contents.trim();
                                     let url = url::Url::parse(url);
                                     match url {

ark-cli/src/commands/render.rs

@@ -26,7 +26,8 @@ impl Render {
         let dest_path = filepath.with_file_name(
             filepath
                 .file_stem()
-                // SAFETY: we know that the file stem is valid UTF-8 because it is a file name
+                // SAFETY: we know that the file stem is valid UTF-8
+                // because it is a file name
                 .unwrap()
                 .to_str()
                 .unwrap()

ark-cli/src/commands/storage/list.rs

@@ -1,8 +1,8 @@
 use std::path::PathBuf;
 
 use crate::{
-    models::storage::Storage, models::storage::StorageType, translate_storage,
-    AppError,
+    models::storage::{Storage, StorageType},
+    translate_storage, AppError,
 };
 
 #[derive(Clone, Debug, clap::Args)]

ark-cli/src/index_registrar.rs

@@ -2,11 +2,13 @@ use lazy_static::lazy_static;
 extern crate canonical_path;
 
 use data_error::{ArklibError, Result};
-use fs_index::ResourceIndex;
+use fs_index::{load_or_build_index, ResourceIndex};
 
-use std::collections::HashMap;
-use std::path::Path;
-use std::sync::{Arc, RwLock};
+use std::{
+    collections::HashMap,
+    path::Path,
+    sync::{Arc, RwLock},
+};
 
 use crate::ResourceId;
 
@@ -34,7 +36,9 @@ pub fn provide_index<P: AsRef<Path>>(
     }
 
     log::info!("Index has not been registered before");
-    match ResourceIndex::provide(&root_path) {
+    // If the index has not been registered before,
+    // we need to load it, update it and register it
+    match load_or_build_index(&root_path, true) {
         Ok(index) => {
             let mut registrar = REGISTRAR.write().map_err(|_| {
                 ArklibError::Other(anyhow::anyhow!("Failed to lock registrar"))

ark-cli/src/main.rs

@@ -1,5 +1,7 @@
-use std::fs::{create_dir_all, File};
-use std::path::PathBuf;
+use std::{
+    fs::{create_dir_all, File},
+    path::PathBuf,
+};
 
 use crate::index_registrar::provide_index;
 use data_pdf::{render_preview_page, PDFQuality};
@@ -15,25 +17,23 @@ use fs_storage::ARK_FOLDER;
 
 use anyhow::Result;
 
-use chrono::prelude::DateTime;
-use chrono::Utc;
+use chrono::{prelude::DateTime, Utc};
 
-use clap::CommandFactory;
-use clap::FromArgMatches;
+use clap::{CommandFactory, FromArgMatches};
 
 use fs_extra::dir::{self, CopyOptions};
 
 use home::home_dir;
 
-use crate::cli::Cli;
-use crate::commands::file::File::{Append, Insert, Read};
-use crate::commands::link::Link::{Create, Load};
-use crate::commands::Commands::Link;
-use crate::commands::Commands::Storage;
-use crate::commands::Commands::*;
-use crate::models::EntryOutput;
-use crate::models::Format;
-use crate::models::Sort;
+use crate::{
+    cli::Cli,
+    commands::{
+        file::File::{Append, Insert, Read},
+        link::Link::{Create, Load},
+        Commands::{Link, Storage, *},
+    },
+    models::{EntryOutput, Format, Sort},
+};
 
 use crate::error::AppError;
 

ark-cli/src/models/storage.rs

@@ -1,7 +1,6 @@
 use crate::ResourceId;
 use fs_atomic_versions::atomic::AtomicFile;
-use std::fmt::Write;
-use std::path::PathBuf;
+use std::{fmt::Write, path::PathBuf};
 
 use crate::{
     commands::{file_append, file_insert, format_file, format_line},