From 339ee3fa5420a6d8c070dc65676f4ea88b69a60c Mon Sep 17 00:00:00 2001 From: wcampbell Date: Sat, 13 Jan 2024 22:59:30 -0500 Subject: [PATCH] Add gzip-zune-inflate feature This crate features an optimized inflate algorithm supporting whole buffer decompression. --- .github/workflows/coverage.yml | 2 +- .github/workflows/main.yml | 1 + BENCHMARK.md | 38 +++++++------ Cargo.lock | 17 +++++- backhand-cli/Cargo.toml | 2 + backhand-test/Cargo.toml | 12 ++--- backhand-test/tests/test.rs | 97 +++++++++++++++++++++++++++------- backhand/Cargo.toml | 10 ++-- backhand/src/compressor.rs | 9 +++- bench.bash | 19 ++++--- 10 files changed, 153 insertions(+), 54 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a3dde019..44f49c14 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -36,7 +36,7 @@ jobs: - run: cargo llvm-cov run --bin unsquashfs-backhand --no-clean --release || true # run coverage on tests - - run: cargo llvm-cov --workspace --codecov --output-path codecov.json --all-features --release --no-clean -- --skip slow + - run: cargo llvm-cov --workspace --codecov --output-path codecov.json --features __test_unsquashfs --release --no-clean -- --skip slow - name: Upload coverage to Codecov uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ecaa6bb2..9e5b21f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,6 +68,7 @@ jobs: - --no-default-features --features gzip - --no-default-features --features gzip,xz - --no-default-features --features xz-static + - --no-default-features --features gzip-zune-inflate # default features - diff --git a/BENCHMARK.md b/BENCHMARK.md index eff3a9a3..54bc768b 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -36,35 +36,41 @@ Vendor ID: GenuineIntel ## Wall time: `backhand/unsquashfs-v0.14.0` vs `squashfs-tools/unsquashfs-4.6.1` -| Command | Mean [s] | Min [s] | Max [s] | Relative | -|:---|---:|---:|---:|---:| -| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHnU0 -o 0 backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 1.052 ± 0.033 | 1.025 | 1.194 | 1.13 ± 0.04 | -| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BH6PE -f -o 0 -ignore-errors backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 0.934 ± 0.014 | 0.912 | 0.985 | 1.00 | - | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHXbp -o 0 backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 486.1 ± 19.9 | 462.0 | 542.5 | 1.07 ± 0.06 | -| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHCdV -f -o 0 -ignore-errors backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 454.6 ± 14.9 | 420.1 | 502.1 | 1.00 | +| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHgGC -o 2252752 backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 199.8 ± 5.4 | 192.9 | 219.4 | 1.18 ± 0.07 | +| `./target/dist/unsquashfs-backhand --quiet -f -d /tmp/BHGD0 -o 2252752 backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 180.0 ± 4.9 | 170.1 | 190.6 | 1.06 ± 0.06 | +| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BH7Ww -f -o 2252752 -ignore-errors backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 169.0 ± 8.5 | 159.1 | 194.9 | 1.00 | | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BH3Uj -o 2252752 backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 198.2 ± 4.6 | 192.7 | 217.3 | 1.18 ± 0.07 | -| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHGE9 -f -o 2252752 -ignore-errors backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 167.7 ± 8.7 | 155.4 | 196.3 | 1.00 | +| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHi7s -o 2883712 backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 202.6 ± 4.1 | 196.1 | 212.0 | 1.25 ± 0.06 | +| `./target/dist/unsquashfs-backhand --quiet -f -d /tmp/BHxh9 -o 2883712 backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 183.4 ± 8.5 | 169.2 | 219.8 | 1.13 ± 0.07 | +| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHpvG -f -o 2883712 -ignore-errors backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 162.1 ± 6.8 | 152.1 | 185.0 | 1.00 | -| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| -| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHu2n -o 2883712 backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 200.7 ± 4.6 | 194.5 | 222.6 | 1.24 ± 0.07 | -| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHidr -f -o 2883712 -ignore-errors backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 162.3 ± 8.2 | 147.9 | 180.3 | 1.00 | +| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BH04Q -o 0 backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 1.005 ± 0.042 | 0.949 | 1.221 | 1.44 ± 0.08 | +| `./target/dist/unsquashfs-backhand --quiet -f -d /tmp/BHNDa -o 0 backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 0.870 ± 0.023 | 0.820 | 0.911 | 1.25 ± 0.05 | +| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHlK0 -f -o 0 -ignore-errors backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 0.698 ± 0.024 | 0.684 | 0.815 | 1.00 | | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| -| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHqtd -o 188392 backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 1.314 ± 0.011 | 1.294 | 1.349 | 2.55 ± 0.03 | -| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHXUc -f -o 188392 -ignore-errors backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 0.516 ± 0.003 | 0.510 | 0.524 | 1.00 | +| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHkLA -o 0 backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 1.070 ± 0.041 | 1.041 | 1.243 | 1.12 ± 0.05 | +| `./target/dist/unsquashfs-backhand --quiet -f -d /tmp/BHxKt -o 0 backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 0.964 ± 0.048 | 0.923 | 1.087 | 1.01 ± 0.05 | +| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHmjP -f -o 0 -ignore-errors backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 0.953 ± 0.019 | 0.925 | 1.037 | 1.00 | + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +|:---|---:|---:|---:|---:| +| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHyeJ -o 0 backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 489.4 ± 20.7 | 465.6 | 542.5 | 1.10 ± 0.06 | +| `./target/dist/unsquashfs-backhand --quiet -f -d /tmp/BHKaH -o 0 backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 444.6 ± 17.0 | 418.4 | 485.0 | 1.00 | +| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHxoF -f -o 0 -ignore-errors backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 456.1 ± 12.9 | 429.4 | 482.8 | 1.03 ± 0.05 | | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHjpe -o 0 backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 983.0 ± 25.0 | 930.6 | 1026.3 | 1.46 ± 0.04 | -| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BH6IH -f -o 0 -ignore-errors backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 674.4 ± 5.4 | 667.8 | 697.8 | 1.00 | +| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHQl3 -o 188392 backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 646.1 ± 6.4 | 634.5 | 661.8 | 1.21 ± 0.02 | +| `./target/dist/unsquashfs-backhand --quiet -f -d /tmp/BHm5w -o 188392 backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 635.6 ± 8.2 | 618.4 | 663.0 | 1.19 ± 0.02 | +| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHUlC -f -o 188392 -ignore-errors backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 533.3 ± 6.6 | 527.6 | 571.7 | 1.00 | ## Heap Usage: `backhand/unsquashfs-v0.14.0` vs `squashfs-tools/unsquashfs-4.6.1` | Command | Peak Heap Memory Consumption | diff --git a/Cargo.lock b/Cargo.lock index d60bf94b..45aab023 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,6 +138,7 @@ dependencies = [ "tracing", "xz2", "zstd", + "zune-inflate", ] [[package]] @@ -1285,6 +1286,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "smallvec" version = "1.11.2" @@ -1406,7 +1413,6 @@ version = "0.0.0" dependencies = [ "assert_cmd", "backhand", - "backhand-cli", "dir-diff", "env_logger", "libdeflater", @@ -2013,3 +2019,12 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] diff --git a/backhand-cli/Cargo.toml b/backhand-cli/Cargo.toml index 491ca6c8..548a91da 100644 --- a/backhand-cli/Cargo.toml +++ b/backhand-cli/Cargo.toml @@ -38,6 +38,8 @@ xz = ["backhand/xz"] xz-static = ["backhand/xz-static"] ## Enables gzip compression inside library and binaries gzip = ["backhand/gzip"] +## Enables faster gzip de-compression only inside library and binaries +gzip-zune-inflate = ["backhand/gzip-zune-inflate"] ## This library is licensed GPL and thus disabled by default lzo = ["backhand/lzo"] ## Enables zstd compression inside library and binaries diff --git a/backhand-test/Cargo.toml b/backhand-test/Cargo.toml index 3478fc2e..7d998b72 100644 --- a/backhand-test/Cargo.toml +++ b/backhand-test/Cargo.toml @@ -6,7 +6,6 @@ publish = false [dev-dependencies] backhand = { path = "../backhand", default-features = false } -backhand-cli = { path = "../backhand-cli", default-features = false } assert_cmd = { version = "2.0.12", features = ["color", "color-auto"] } dir-diff = { git = "https://github.com/wcampbell0x2a/dir-diff", branch = "add-checking-permissions" } tempfile = "3.9.0" @@ -25,12 +24,13 @@ bench = false # testing only feature for testing vs squashfs-tools/unsquashfs __test_unsquashfs = [] default = ["xz", "gzip", "zstd"] -xz = ["backhand-cli/xz"] -xz-static = ["backhand-cli/xz-static"] -gzip = ["backhand-cli/gzip"] +xz = ["backhand/xz"] +xz-static = ["backhand/xz-static"] +gzip = ["backhand/gzip"] +gzip-zune-inflate = ["backhand/gzip-zune-inflate"] # this library is licensed GPL and thus disabled by default -lzo = ["backhand-cli/lzo"] -zstd = ["backhand-cli/zstd"] +lzo = ["backhand/lzo"] +zstd = ["backhand/zstd"] [[test]] name = "add" diff --git a/backhand-test/tests/test.rs b/backhand-test/tests/test.rs index b9f6ec2c..fe5f27c6 100644 --- a/backhand-test/tests/test.rs +++ b/backhand-test/tests/test.rs @@ -11,9 +11,29 @@ use test_assets::TestAssetDef; use test_log::test; use tracing::info; +#[cfg(feature = "gzip")] +fn has_gzip_feature() -> bool { + true +} + +#[cfg(not(feature = "gzip"))] +fn has_gzip_feature() -> bool { + false +} + enum Verify { Extract, } + +fn only_read(assets_defs: &[TestAssetDef], filepath: &str, test_path: &str, offset: u64) { + test_assets::download_test_files(assets_defs, test_path, true).unwrap(); + + let og_path = format!("{test_path}/{filepath}"); + let file = BufReader::new(File::open(&og_path).unwrap()); + info!("calling from_reader"); + let _ = FilesystemReader::from_reader_with_offset(file, offset).unwrap(); +} + /// - Download file /// - Read into Squashfs /// - Into Filesystem @@ -74,7 +94,7 @@ fn full_test( /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip -Xcompression-level 2 -always-use-fragments #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_00() { const FILE_NAME: &str = "out.squashfs"; let asset_defs = [TestAssetDef { @@ -83,12 +103,17 @@ fn test_00() { url: format!("https://wcampbell.dev/squashfs/testing/test_00/{FILE_NAME}"), }]; const TEST_PATH: &str = "test-assets/test_00"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip -Xcompression-level 2 #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_01() { const FILE_NAME: &str = "out.squashfs"; let asset_defs = [TestAssetDef { @@ -97,7 +122,11 @@ fn test_01() { url: format!("https://wcampbell.dev/squashfs/testing/test_01/{FILE_NAME}"), }]; const TEST_PATH: &str = "test-assets/test_01"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp xz @@ -156,7 +185,7 @@ fn test_05() { /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip -always-use-fragments #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_06() { const FILE_NAME: &str = "out.squashfs"; let asset_defs = [TestAssetDef { @@ -165,12 +194,16 @@ fn test_06() { url: format!("https://wcampbell.dev/squashfs/testing/test_06/{FILE_NAME}"), }]; const TEST_PATH: &str = "test-assets/test_06"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_07() { const FILE_NAME: &str = "out.squashfs"; let asset_defs = [TestAssetDef { @@ -179,7 +212,12 @@ fn test_07() { url: format!("https://wcampbell.dev/squashfs/testing/test_07/{FILE_NAME}"), }]; const TEST_PATH: &str = "test-assets/test_07"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } // mksquashfs ./target/release/squashfs-deku out.squashfs -comp xz -Xbcj arm @@ -255,7 +293,7 @@ fn test_openwrt_netgear_ex6100v2() { } #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_appimage_plexamp() { const FILE_NAME: &str = "Plexamp-4.6.1.AppImage"; let asset_defs = [TestAssetDef { @@ -264,11 +302,16 @@ fn test_appimage_plexamp() { url: format!("https://plexamp.plex.tv/plexamp.plex.tv/desktop/{FILE_NAME}"), }]; const TEST_PATH: &str = "test-assets/test_appimage_plexamp"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2dfe8, Verify::Extract, true); + + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2dfe8, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0x2dfe8); + } } #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_appimage_firefox() { const FILE_NAME: &str = "firefox-108.0.r20221215175817-x86_64.AppImage"; let asset_defs = [TestAssetDef { @@ -277,7 +320,12 @@ fn test_appimage_firefox() { url: "https://github.com/srevinsaju/Firefox-Appimage/releases/download/firefox-v108.0.r20221215175817/firefox-108.0.r20221215175817-x86_64.AppImage".to_string(), }]; const TEST_PATH: &str = "test-assets/test_appimage_firefox"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2f4c0, Verify::Extract, true); + + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2f4c0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0x2f4c0); + } } /// Archer\ AX1800\(US\)_V3_221016.zip from https://www.tp-link.com/us/support/download/archer-ax1800/#Firmware @@ -350,7 +398,7 @@ fn test_slow_archlinux_iso_rootfs() { } #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_many_files() { const FILE_NAME: &str = "many_files.squashfs"; let asset_defs = [TestAssetDef { @@ -360,11 +408,15 @@ fn test_many_files() { }]; const TEST_PATH: &str = "test-assets/test_many_files"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_many_dirs() { const FILE_NAME: &str = "many_dirs.squashfs"; let asset_defs = [TestAssetDef { @@ -374,11 +426,15 @@ fn test_many_dirs() { }]; const TEST_PATH: &str = "test-assets/test_many_dirs"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } #[test] -#[cfg(feature = "gzip")] +#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))] fn test_few_dirs_many_files() { const FILE_NAME: &str = "few_dirs_many_files.squashfs"; let asset_defs = [TestAssetDef { @@ -388,5 +444,10 @@ fn test_few_dirs_many_files() { }]; const TEST_PATH: &str = "test-assets/test_few_dirs_many_files"; - full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + + if has_gzip_feature() { + full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true); + } else { + only_read(&asset_defs, FILE_NAME, TEST_PATH, 0); + } } diff --git a/backhand/Cargo.toml b/backhand/Cargo.toml index c978a746..9b41170a 100644 --- a/backhand/Cargo.toml +++ b/backhand/Cargo.toml @@ -11,11 +11,12 @@ rust-version = "1.67.1" description = "Library for the reading, creating, and modification of SquashFS file systems" [dependencies] -# for lib deku = "0.16.0" -tracing = "0.1.40" +# only error and warn traces in release mode +tracing = { version = "0.1.40"} thiserror = "1.0.53" flate2 = { version = "1.0.28", optional = true } +zune-inflate = { version = "0.2.54", optional = true, default-features = false, features = ["zlib"] } xz2 = { version = "0.1.7", optional = true } rust-lzo = { version = "0.6.2", optional = true } zstd = { version = "0.13.0", optional = true } @@ -30,6 +31,9 @@ xz = ["dep:xz2"] xz-static = ["dep:xz2", "xz2?/static"] ## Enables gzip compression inside library and binaries gzip = ["dep:flate2"] +## Enables faster gzip (de-compression only) inside library and binaries +## Cannot be used with the `gzip` feature +gzip-zune-inflate = ["dep:zune-inflate"] ## This library is licensed GPL and thus disabled by default lzo = ["dep:rust-lzo"] ## Enables zstd compression inside library and binaries @@ -53,4 +57,4 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [lib] -bench = false \ No newline at end of file +bench = false diff --git a/backhand/src/compressor.rs b/backhand/src/compressor.rs index e7d96fa3..ad18699c 100644 --- a/backhand/src/compressor.rs +++ b/backhand/src/compressor.rs @@ -191,11 +191,18 @@ impl CompressionAction for DefaultCompressor { ) -> Result<(), BackhandError> { match compressor { Compressor::None => out.extend_from_slice(bytes), - #[cfg(feature = "gzip")] + #[cfg(all(feature = "gzip", not(feature = "gzip-zune-inflate")))] Compressor::Gzip => { let mut decoder = flate2::read::ZlibDecoder::new(bytes); decoder.read_to_end(out)?; } + #[cfg(all(feature = "gzip-zune-inflate", not(feature = "gzip")))] + Compressor::Gzip => { + use std::io::Write; + let mut decoder = zune_inflate::DeflateDecoder::new(&bytes); + let mut decompressed = decoder.decode_zlib().unwrap(); + out.write_all(&mut decompressed)?; + } #[cfg(feature = "xz")] Compressor::Xz => { let mut decoder = XzDecoder::new(bytes); diff --git a/bench.bash b/bench.bash index 52e0ebb1..c4e6f859 100755 --- a/bench.bash +++ b/bench.bash @@ -1,31 +1,34 @@ #!/bin/bash -BACKHAND="./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand" +BACKHAND="./target/dist/unsquashfs-backhand" +BACKHAND_MUSL="./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand" UNSQUASHFS="/usr/bin/unsquashfs" bench () { file $1 hyperfine --runs 50 --warmup 10 --export-markdown bench-results/$3.md -i \ + "$BACKHAND_MUSL --quiet -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \ "$BACKHAND --quiet -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \ "$UNSQUASHFS -quiet -no-progress -d $(mktemp -d /tmp/BHXXX) -f -o $(rz-ax $2) -ignore-errors $1" } -cross +stable build -p backhand-cli --bins --locked --target x86_64-unknown-linux-musl --profile=dist +cross +stable build -p backhand-cli --bins --locked --target x86_64-unknown-linux-musl --profile=dist --no-default-features --features xz --features gzip-zune-inflate +cargo +stable build -p backhand-cli --bins --locked --profile=dist --no-default-features --features xz --features gzip-zune-inflate mkdir -p bench-results # xz -bench "backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin" 0x225fd0 openwrt1 +bench "backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin" 0x225fd0 0_openwrt1 # xz -bench "backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img" 0x2c0080 openwrt2 +bench "backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img" 0x2c0080 1_openwrt2 # xz -bench "backhand-test/test-assets/test_re815_xev160/870D97.squashfs" 0x0 re815 +bench "backhand-test/test-assets/test_re815_xev160/870D97.squashfs" 0x0 2_re815 # xz -bench "backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs" 0x0 ax18000 +bench "backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs" 0x0 3_ax18000 # xz #bench "test-assets/test_archlinux_iso_rootfs/airootfs.sfs" 0x0 # xz -bench "backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs" 0x0 er605 +bench "backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs" 0x0 4_er605 # gzip -bench "backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage" 0x2dfe8 plexamp +bench "backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage" 0x2dfe8 5_plexamp rm -rf /tmp/BH*