From c1cc958e2016986ed8228fffe453518fdb608487 Mon Sep 17 00:00:00 2001 From: zamu-flowerpot <106848396+zamu-flowerpot@users.noreply.github.com> Date: Wed, 9 Nov 2022 17:29:57 -0500 Subject: [PATCH 1/4] Add 'WalkDir' dependency --- Cargo.lock | 1 + Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index e4892ec52..67e32fde7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1604,6 +1604,7 @@ dependencies = [ "tar", "thiserror", "url", + "walkdir", "yansi", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index a9af8e3d7..153bad68a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ socket2 = "0.4" strum = { version = "0.24", features = ["derive"] } tar = "0.4" thiserror = "1" +walkdir = "2.3.2" yansi = "0.5" zip = { version = "0.6.2", default-features = false } From 041adb7f6c0f5f256c0cf32d3766aea8a8bda7a2 Mon Sep 17 00:00:00 2001 From: zamu-flowerpot <106848396+zamu-flowerpot@users.noreply.github.com> Date: Wed, 9 Nov 2022 17:29:57 -0500 Subject: [PATCH 2/4] feat/issue-306-pr: `src/listing.rs` === * Add `EntrySize` struct * Impl Display for EntrySize * Change `Option` to `EntrySize` in `Entry` * Add Entry::Directory size calculation utilizing WalkDir * Sort directory listings by always putting directories at the top and sorting files and directories based on their own semantics (ie. entry counts for directories and bytes for files) `src/renderer.rs` === * Change `entry_raw` from using `if-let` to just operating on `entry.size` --- src/listing.rs | 44 ++++++++++++++++++++++++++++++++++++-------- src/renderer.rs | 18 +++++++++++------- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/listing.rs b/src/listing.rs index bca531a81..85baa563b 100644 --- a/src/listing.rs +++ b/src/listing.rs @@ -1,4 +1,5 @@ #![allow(clippy::format_push_string)] +use std::fmt::Display; use std::io; use std::path::{Component, Path, PathBuf}; use std::time::SystemTime; @@ -10,6 +11,7 @@ use percent_encoding::{percent_decode_str, utf8_percent_encode}; use regex::Regex; use serde::Deserialize; use strum::{Display, EnumString}; +use walkdir::WalkDir; use crate::archive::ArchiveMethod; use crate::auth::CurrentUser; @@ -78,6 +80,24 @@ pub enum EntryType { File, } +#[derive(Clone, Copy, PartialEq, Eq)] +/// Possible entry size types +pub enum EntrySize { + /// EntryCount is number of entries in a directory + EntryCount(usize), + /// Bytes is number of bytes in a file + Bytes(bytesize::ByteSize), +} + +impl Display for EntrySize { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EntrySize::EntryCount(count) => write!(f, "{}", count), + EntrySize::Bytes(bytes) => write!(f, "{}", bytes), + } + } +} + /// Entry pub struct Entry { /// Name of the entry @@ -89,8 +109,8 @@ pub struct Entry { /// URL of the entry pub link: String, - /// Size in byte of the entry. Only available for EntryType::File - pub size: Option, + /// Size of the entry + pub size: EntrySize, /// Last modification date pub last_modification_date: Option, @@ -104,7 +124,7 @@ impl Entry { name: String, entry_type: EntryType, link: String, - size: Option, + size: EntrySize, last_modification_date: Option, symlink_info: Option, ) -> Self { @@ -256,12 +276,14 @@ pub fn directory_listing( Err(_) => None, }; + let size = WalkDir::new(entry.path()).into_iter().count(); + if metadata.is_dir() { entries.push(Entry::new( file_name, EntryType::Directory, file_url, - None, + EntrySize::EntryCount(size), last_modification_date, symlink_dest, )); @@ -270,7 +292,7 @@ pub fn directory_listing( file_name.clone(), EntryType::File, file_url, - Some(ByteSize::b(metadata.len())), + EntrySize::Bytes(ByteSize::b(metadata.len())), last_modification_date, symlink_dest, )); @@ -302,9 +324,15 @@ pub fn directory_listing( SortingMethod::Size => entries.sort_by(|e1, e2| { // If we can't get the size of the entry (directory for instance) // let's consider it's 0b - e2.size - .unwrap_or_else(|| ByteSize::b(0)) - .cmp(&e1.size.unwrap_or_else(|| ByteSize::b(0))) + match (e1.size, e2.size) { + (EntrySize::EntryCount(ref e1_count), EntrySize::EntryCount(ref e2_count)) => e2_count.cmp(e1_count), + (EntrySize::Bytes(ref e1_bytes), EntrySize::Bytes(ref e2_bytes)) => e2_bytes.cmp(e1_bytes), + (EntrySize::EntryCount(_), EntrySize::Bytes(_)) => std::cmp::Ordering::Greater, + (EntrySize::Bytes(_), EntrySize::EntryCount(_)) => std::cmp::Ordering::Less, + } + // e2.size + // .unwrap_or_else(|| ByteSize::b(0)) + // .cmp(&e1.size.unwrap_or_else(|| ByteSize::b(0))) }), SortingMethod::Date => entries.sort_by(|e1, e2| { // If, for some reason, we can't get the last modification date of an entry diff --git a/src/renderer.rs b/src/renderer.rs index f53620f03..f4f59dfd5 100644 --- a/src/renderer.rs +++ b/src/renderer.rs @@ -511,19 +511,23 @@ fn entry_row( } @if !raw { - @if let Some(size) = entry.size { - span.mobile-info.size { - (maud::display(size)) - } + span.mobile-info.size { + (maud::display(&entry.size)) } + // @if let size = entry.size { + // span.mobile-info.size { + // (maud::display(size)) + // } + // } } } } } td.size-cell { - @if let Some(size) = entry.size { - (maud::display(size)) - } + (maud::display(&entry.size)) + // @if let size = entry.size { + // (maud::display(size)) + // } } td.date-cell { @if let Some(modification_date) = convert_to_utc(entry.last_modification_date) { From 1c71e93977ae256fa6d4b056220935e587c84943 Mon Sep 17 00:00:00 2001 From: zamu-flowerpot <106848396+zamu-flowerpot@users.noreply.github.com> Date: Sun, 20 Nov 2022 20:17:39 -0500 Subject: [PATCH 3/4] feat/issue-306-pr: `Cargo.lock` & `Cargo.toml` === * Remove `WalkDir` * Add `fs_extra` * Add `OnceCell` `src/listing.rs` === * remove `use walkdir::WalkDir` * add HashMap, Arc, Mutex, OnceCell, log::warn, and `FILE_SIZE_CACHE` * Cargo fmt --- Cargo.lock | 9 ++++++++- Cargo.toml | 3 ++- src/listing.rs | 10 ++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67e32fde7..270307a4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -904,6 +904,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + [[package]] name = "futf" version = "0.1.5" @@ -1575,6 +1581,7 @@ dependencies = [ "comrak", "fake-tty", "fast_qr", + "fs_extra", "futures", "get_if_addrs", "grass", @@ -1586,6 +1593,7 @@ dependencies = [ "maud", "mime", "nanoid", + "once_cell", "percent-encoding", "port_check", "predicates", @@ -1604,7 +1612,6 @@ dependencies = [ "tar", "thiserror", "url", - "walkdir", "yansi", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 153bad68a..7a7365ad1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ clap_complete = "4" clap_mangen = "0.2" comrak = { version = "0.14", default-features = false } fast_qr = "0.5.1" +fs_extra = "1.2.0" futures = "0.3" get_if_addrs = "0.5" hex = "0.4" @@ -43,6 +44,7 @@ log = "0.4" maud = "0.24" mime = "0.3" nanoid = "0.4" +once_cell = "1.16.0" percent-encoding = "2" port_check = "0.1" regex = "1" @@ -55,7 +57,6 @@ socket2 = "0.4" strum = { version = "0.24", features = ["derive"] } tar = "0.4" thiserror = "1" -walkdir = "2.3.2" yansi = "0.5" zip = { version = "0.6.2", default-features = false } diff --git a/src/listing.rs b/src/listing.rs index 85baa563b..cc4ada60b 100644 --- a/src/listing.rs +++ b/src/listing.rs @@ -1,17 +1,20 @@ #![allow(clippy::format_push_string)] +use std::collections::HashMap; use std::fmt::Display; use std::io; use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::SystemTime; use actix_web::{dev::ServiceResponse, web::Query, HttpMessage, HttpRequest, HttpResponse}; use bytesize::ByteSize; use comrak::{markdown_to_html, ComrakOptions}; +use log::warn; +use once_cell::sync::OnceCell; use percent_encoding::{percent_decode_str, utf8_percent_encode}; use regex::Regex; use serde::Deserialize; use strum::{Display, EnumString}; -use walkdir::WalkDir; use crate::archive::ArchiveMethod; use crate::auth::CurrentUser; @@ -20,6 +23,9 @@ use crate::renderer; use self::percent_encode_sets::PATH_SEGMENT; + +static FILE_SIZE_CACHE: OnceCell>>> = OnceCell::new(); + /// "percent-encode sets" as defined by WHATWG specs: /// https://url.spec.whatwg.org/#percent-encoded-bytes mod percent_encode_sets { @@ -93,7 +99,7 @@ impl Display for EntrySize { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { EntrySize::EntryCount(count) => write!(f, "{}", count), - EntrySize::Bytes(bytes) => write!(f, "{}", bytes), + EntrySize::Bytes(bytes) => write!(f, "{}", bytes), } } } From dc734c4ef252039f4a2d37113e8485f8b3115501 Mon Sep 17 00:00:00 2001 From: zamu-flowerpot <106848396+zamu-flowerpot@users.noreply.github.com> Date: Sun, 20 Nov 2022 20:20:42 -0500 Subject: [PATCH 4/4] feat/issue-306-pr: `src/listing.rs` === * Cargo fmt * Replace `WalkDir::new(entry.path()).into_iter().count()` with calculating directory size using `fs_extra::dir::get_size` * If `get_size` fails, fall back to `std::fs::read_dir` counting --- src/listing.rs | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/listing.rs b/src/listing.rs index cc4ada60b..ad4194dd2 100644 --- a/src/listing.rs +++ b/src/listing.rs @@ -23,7 +23,6 @@ use crate::renderer; use self::percent_encode_sets::PATH_SEGMENT; - static FILE_SIZE_CACHE: OnceCell>>> = OnceCell::new(); /// "percent-encode sets" as defined by WHATWG specs: @@ -282,14 +281,36 @@ pub fn directory_listing( Err(_) => None, }; - let size = WalkDir::new(entry.path()).into_iter().count(); - + let cache_map = FILE_SIZE_CACHE + .get_or_init(|| { + println!("init cache_map"); + Arc::default() + }) + .clone(); + let size = { + match fs_extra::dir::get_size(&entry.path()) { + Err(_) => EntrySize::EntryCount({ + std::fs::read_dir(entry.path()) + .into_iter() + .take(500_000) + .count() + }), + Ok(result) => { + if let Ok(mut lock) = cache_map.lock() { + lock.insert(entry.path().to_path_buf(), result); + } else { + warn!("Failed to write to cache"); + }; + EntrySize::Bytes(ByteSize::b(result)) + } + } + }; if metadata.is_dir() { entries.push(Entry::new( file_name, EntryType::Directory, file_url, - EntrySize::EntryCount(size), + size, last_modification_date, symlink_dest, )); @@ -331,8 +352,12 @@ pub fn directory_listing( // If we can't get the size of the entry (directory for instance) // let's consider it's 0b match (e1.size, e2.size) { - (EntrySize::EntryCount(ref e1_count), EntrySize::EntryCount(ref e2_count)) => e2_count.cmp(e1_count), - (EntrySize::Bytes(ref e1_bytes), EntrySize::Bytes(ref e2_bytes)) => e2_bytes.cmp(e1_bytes), + (EntrySize::EntryCount(ref e1_count), EntrySize::EntryCount(ref e2_count)) => { + e2_count.cmp(e1_count) + } + (EntrySize::Bytes(ref e1_bytes), EntrySize::Bytes(ref e2_bytes)) => { + e2_bytes.cmp(e1_bytes) + } (EntrySize::EntryCount(_), EntrySize::Bytes(_)) => std::cmp::Ordering::Greater, (EntrySize::Bytes(_), EntrySize::EntryCount(_)) => std::cmp::Ordering::Less, }