From 118a334c86f2a511d11d10d4a8ca46d468da4514 Mon Sep 17 00:00:00 2001 From: figsoda Date: Mon, 30 Jan 2023 18:54:40 -0500 Subject: [PATCH 1/4] bump dependencies --- Cargo.lock | 116 ++++++++++++++++++++++++----------------------------- Cargo.toml | 16 ++++---- 2 files changed, 61 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f83831e33..fdcaa8d36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,11 +19,11 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa3d466004a8b4cb1bc34044240a2fd29d17607e2e3bd613eb44fd48e8100da3" +checksum = "9834fcc22e0874394a010230586367d4a3e9f11b560f469262678547e1d2575e" dependencies = [ - "bstr 1.1.0", + "bstr", "doc-comment", "predicates", "predicates-core", @@ -69,15 +69,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "memchr", -] - [[package]] name = "bstr" version = "1.1.0" @@ -119,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" dependencies = [ "jobserver", ] @@ -145,9 +136,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.0.32" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" dependencies = [ "bitflags", "clap_derive", @@ -160,18 +151,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.0.7" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10861370d2ba66b0f5989f83ebf35db6421713fd92351790e7fdd6c36774c56b" +checksum = "3d6540eedc41f8a5a76cf3d8d458057dcdf817be4158a55b5f861f7a5483de75" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", "proc-macro-error", @@ -182,18 +173,18 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" dependencies = [ "os_str_bytes", ] [[package]] name = "clap_mangen" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904eb24d05ad587557e0f484ddce5c737c30cf81372badb16d13e41c4b8340b1" +checksum = "eb258c6232b4d728d13d6072656627924c16707aae6267cd5a1ea05abff9a25c" dependencies = [ "clap", "roff", @@ -265,9 +256,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "errno" @@ -347,12 +338,12 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" +checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" dependencies = [ "aho-corasick", - "bstr 0.2.17", + "bstr", "fnv", "log", "regex", @@ -384,11 +375,10 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.18" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d" +checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" dependencies = [ - "crossbeam-utils", "globset", "lazy_static", "log", @@ -420,9 +410,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", "windows-sys", @@ -598,7 +588,7 @@ version = "0.4.1" dependencies = [ "assert_cmd", "atty", - "bstr 1.1.0", + "bstr", "bzip2", "clap", "clap_complete", @@ -721,9 +711,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -820,9 +810,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -841,9 +831,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -879,9 +869,9 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ "bitflags", "errno", @@ -997,9 +987,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -1012,9 +1002,9 @@ checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" [[package]] name = "test-strategy" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62d6408d1406657be2f9d1701fbae379331d30d2f6e92050710edb0d34eeb480" +checksum = "61348cf95c0dfa12f0b6155f5b564d2806f518620a28263280ae357b41c96b4a" dependencies = [ "proc-macro2", "quote", @@ -1162,45 +1152,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "xattr" @@ -1235,9 +1225,9 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.1+zstd.1.5.2" +version = "0.12.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c947d2adc84ff9a59f2e3c03b81aa4128acf28d6ad7d56273f7e8af14e47bea" +checksum = "e9262a83dc741c0b0ffec209881b45dbc232c21b02a2b9cb1adb93266e41303d" dependencies = [ "zstd-safe", ] diff --git a/Cargo.toml b/Cargo.toml index dab4eec5e..3ffc54d74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,11 +14,11 @@ description = "A command-line utility for easily compressing and decompressing f atty = "0.2.14" bstr = { version = "1.1.0", default-features = false, features = ["std"] } bzip2 = "0.4.4" -clap = { version = "4.0.32", features = ["derive", "env"] } +clap = { version = "4.1.4", features = ["derive", "env"] } filetime = "0.2.19" flate2 = { version = "1.0.25", default-features = false } fs-err = "2.9.0" -ignore = "0.4.18" +ignore = "0.4.20" libc = "0.2.139" linked-hash-map = "0.5.6" lzzzz = "1.0.4" @@ -31,7 +31,7 @@ tempfile = "3.3.0" ubyte = { version = "0.10.3", default-features = false } xz2 = "0.1.7" zip = { version = "0.6.3", default-features = false, features = ["time"] } -zstd = { version = "0.12.1", default-features = false } +zstd = { version = "0.12.2", default-features = false } # zstd-sys > 2.0.1 unconditionally enables thin LTO and causes CI to fail # https://github.com/gyscos/zstd-rs/pull/155 zstd-sys = "=2.0.1" @@ -43,17 +43,17 @@ time = { version = "0.3.17", default-features = false } is_executable = "1.0.1" [build-dependencies] -clap = { version = "4.0.32", features = ["derive", "env", "string"] } -clap_complete = "4.0.7" -clap_mangen = "0.2.6" +clap = { version = "4.1.4", features = ["derive", "env", "string"] } +clap_complete = "4.1.1" +clap_mangen = "0.2.7" [dev-dependencies] -assert_cmd = "2.0.7" +assert_cmd = "2.0.8" infer = "0.12.0" parse-display = "0.8.0" proptest = "1.0.0" rand = { version = "0.8.5", default-features = false, features = ["small_rng", "std"] } -test-strategy = "0.2.1" +test-strategy = "0.3.0" [features] default = ["flate2/zlib", "zip/deflate-zlib", "zstd/thin"] From 5abdd5fc96e1eee012765c77f8fec717adf338dd Mon Sep 17 00:00:00 2001 From: figsoda Date: Mon, 30 Jan 2023 19:10:12 -0500 Subject: [PATCH 2/4] multi-threaded gzip compression using gzp --- Cargo.lock | 199 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 +- src/commands/compress.rs | 20 ++-- 3 files changed, 212 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdcaa8d36..3bb84a57f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,12 +81,24 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + [[package]] name = "bzip2" version = "0.4.4" @@ -190,6 +202,17 @@ dependencies = [ "roff", ] +[[package]] +name = "core_affinity" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4436406e93f52cce33bfba4be067a9f7229da44a634c385e4b22cdfaca5f84cc" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -313,6 +336,19 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -325,6 +361,18 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" +[[package]] +name = "futures-core" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" + +[[package]] +name = "futures-sink" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" + [[package]] name = "getrandom" version = "0.2.8" @@ -332,8 +380,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -349,6 +399,22 @@ dependencies = [ "regex", ] +[[package]] +name = "gzp" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c65d1899521a11810501b50b898464d133e1afc96703cff57726964cfa7baf" +dependencies = [ + "byteorder", + "bytes", + "core_affinity", + "flate2", + "flume", + "libz-sys", + "num_cpus", + "thiserror", +] + [[package]] name = "heck" version = "0.4.0" @@ -463,6 +529,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -482,6 +557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -498,6 +574,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -551,6 +637,15 @@ dependencies = [ "adler", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -596,6 +691,7 @@ dependencies = [ "filetime", "flate2", "fs-err", + "gzp", "ignore", "infer", "is_executable", @@ -646,6 +742,26 @@ dependencies = [ "syn", ] +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pkg-config" version = "0.3.26" @@ -920,6 +1036,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +dependencies = [ + "lock_api", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1012,6 +1137,26 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.4" @@ -1104,6 +1249,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 3ffc54d74..c123e4974 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ clap = { version = "4.1.4", features = ["derive", "env"] } filetime = "0.2.19" flate2 = { version = "1.0.25", default-features = false } fs-err = "2.9.0" +gzp = { version = "0.11.3", default-features = false } ignore = "0.4.20" libc = "0.2.139" linked-hash-map = "0.5.6" @@ -56,7 +57,7 @@ rand = { version = "0.8.5", default-features = false, features = ["small_rng", " test-strategy = "0.3.0" [features] -default = ["flate2/zlib", "zip/deflate-zlib", "zstd/thin"] +default = ["flate2/zlib", "gzp/deflate_zlib", "zip/deflate-zlib", "zstd/thin"] [profile.release] lto = true diff --git a/src/commands/compress.rs b/src/commands/compress.rs index 12f6eef98..50836eb73 100644 --- a/src/commands/compress.rs +++ b/src/commands/compress.rs @@ -8,11 +8,7 @@ use fs_err as fs; use crate::{ archive, commands::warn_user_about_loading_zip_in_memory, - extension::{ - split_first_compression_format, - CompressionFormat::{self, *}, - Extension, - }, + extension::{split_first_compression_format, CompressionFormat::*, Extension}, utils::{user_wants_to_continue, FileVisibilityPolicy}, QuestionAction, QuestionPolicy, BUFFER_CAPACITY, }; @@ -39,12 +35,18 @@ pub fn compress_files( // If the input files contain a directory, then the total size will be underestimated let file_writer = BufWriter::with_capacity(BUFFER_CAPACITY, output_file); - let mut writer: Box = Box::new(file_writer); + let mut writer: Box = Box::new(file_writer); // Grab previous encoder and wrap it inside of a new one - let chain_writer_encoder = |format: &CompressionFormat, encoder: Box| -> crate::Result> { - let encoder: Box = match format { - Gzip => Box::new(flate2::write::GzEncoder::new(encoder, Default::default())), + let chain_writer_encoder = |format: &_, encoder| -> crate::Result<_> { + let encoder: Box = match format { + Gzip => Box::new( + // by default, ParCompress uses a default compression level of 3 + // instead of the regular default that flate2 uses + gzp::par::compress::ParCompress::::builder() + .compression_level(Default::default()) + .from_writer(encoder), + ), Bzip => Box::new(bzip2::write::BzEncoder::new(encoder, Default::default())), Lz4 => Box::new(lzzzz::lz4f::WriteCompressor::new(encoder, Default::default())?), Lzma => Box::new(xz2::write::XzEncoder::new(encoder, 6)), From 2d5dc06a4fdbd0504af5a03a32accfd8ae137b78 Mon Sep 17 00:00:00 2001 From: figsoda Date: Mon, 30 Jan 2023 19:13:00 -0500 Subject: [PATCH 3/4] multi-threaded snap compression using gzp --- Cargo.lock | 1 + Cargo.toml | 2 +- src/commands/compress.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bb84a57f..a408cb7e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -412,6 +412,7 @@ dependencies = [ "flume", "libz-sys", "num_cpus", + "snap", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index c123e4974..6723e1985 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ clap = { version = "4.1.4", features = ["derive", "env"] } filetime = "0.2.19" flate2 = { version = "1.0.25", default-features = false } fs-err = "2.9.0" -gzp = { version = "0.11.3", default-features = false } +gzp = { version = "0.11.3", default-features = false, features = ["snappy_default"] } ignore = "0.4.20" libc = "0.2.139" linked-hash-map = "0.5.6" diff --git a/src/commands/compress.rs b/src/commands/compress.rs index 50836eb73..fb69a7534 100644 --- a/src/commands/compress.rs +++ b/src/commands/compress.rs @@ -50,7 +50,7 @@ pub fn compress_files( Bzip => Box::new(bzip2::write::BzEncoder::new(encoder, Default::default())), Lz4 => Box::new(lzzzz::lz4f::WriteCompressor::new(encoder, Default::default())?), Lzma => Box::new(xz2::write::XzEncoder::new(encoder, 6)), - Snappy => Box::new(snap::write::FrameEncoder::new(encoder)), + Snappy => Box::new(gzp::par::compress::ParCompress::::builder().from_writer(encoder)), Zstd => { let zstd_encoder = zstd::stream::write::Encoder::new(encoder, Default::default()); // Safety: From ce7246acab5211205336607fafbc33bc60f4a64a Mon Sep 17 00:00:00 2001 From: figsoda Date: Mon, 30 Jan 2023 21:02:32 -0500 Subject: [PATCH 4/4] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dacd571b5..e058e618c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ Categories Used: ## [Unreleased](https://github.com/ouch-org/ouch/compare/0.4.1...HEAD) +### Improvements + +- Multi-threaded compression for gzip and snappy using gzp [\#348](https://github.com/ouch-org/ouch/pull/348) ([figsoda](https://github.com/figsoda)) + ## [0.4.1](https://github.com/ouch-org/ouch/compare/0.4.0...0.4.1) ### New Features