Merge 2f9f467 into d75f8ba

.github/workflows/main.yml

@@ -68,6 +68,7 @@ jobs:
           - --no-default-features --features gzip
           - --no-default-features --features gzip,xz
           - --no-default-features --features xz-static
+          - --no-default-features --features gzip-zune-inflate
           # default features
           -
 

BENCHMARK.md

@@ -38,33 +38,33 @@ Vendor ID:               GenuineIntel
 ## Wall time: `backhand/unsquashfs-v0.14.0` vs `squashfs-tools/unsquashfs-4.6.1`
 | Command | Mean [s] | Min [s] | Max [s] | Relative |
 |:---|---:|---:|---:|---:|
-| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHnU0 -o 0 backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 1.052 ± 0.033 | 1.025 | 1.194 | 1.13 ± 0.04 |
-| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BH6PE      -f -o 0 -ignore-errors backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 0.934 ± 0.014 | 0.912 | 0.985 | 1.00 |
+| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHWfc -o 0 backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 1.109 ± 0.049 | 1.074 | 1.280 | 1.15 ± 0.05 |
+| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BH9KN      -f -o 0 -ignore-errors backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 0.962 ± 0.011 | 0.944 | 0.996 | 1.00 |
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHXbp -o 0 backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 486.1 ± 19.9 | 462.0 | 542.5 | 1.07 ± 0.06 |
-| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHCdV      -f -o 0 -ignore-errors backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 454.6 ± 14.9 | 420.1 | 502.1 | 1.00 |
+| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHqLx -o 0 backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 505.0 ± 18.4 | 481.6 | 564.3 | 1.13 ± 0.05 |
+| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHV1O      -f -o 0 -ignore-errors backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs` | 448.8 ± 8.0 | 434.8 | 475.4 | 1.00 |
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BH3Uj -o 2252752 backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 198.2 ± 4.6 | 192.7 | 217.3 | 1.18 ± 0.07 |
-| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHGE9      -f -o 2252752 -ignore-errors backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 167.7 ± 8.7 | 155.4 | 196.3 | 1.00 |
+| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHqvW -o 2252752 backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 207.8 ± 6.5 | 196.4 | 232.2 | 1.17 ± 0.14 |
+| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHcfE      -f -o 2252752 -ignore-errors backhand-test/test-assets/test_openwrt_tplink_archera7v5/openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin` | 177.1 ± 20.6 | 160.0 | 251.3 | 1.00 |
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHu2n -o 2883712 backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 200.7 ± 4.6 | 194.5 | 222.6 | 1.24 ± 0.07 |
-| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHidr      -f -o 2883712 -ignore-errors backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 162.3 ± 8.2 | 147.9 | 180.3 | 1.00 |
+| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHc6m -o 2883712 backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 208.3 ± 3.8 | 201.4 | 217.2 | 1.26 ± 0.07 |
+| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHzpO      -f -o 2883712 -ignore-errors backhand-test/test-assets/test_openwrt_netgear_ex6100v2/openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img` | 165.1 ± 9.3 | 153.7 | 203.5 | 1.00 |
 
-| Command | Mean [s] | Min [s] | Max [s] | Relative |
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHqtd -o 188392 backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 1.314 ± 0.011 | 1.294 | 1.349 | 2.55 ± 0.03 |
-| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHXUc      -f -o 188392 -ignore-errors backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 0.516 ± 0.003 | 0.510 | 0.524 | 1.00 |
+| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHR8s -o 188392 backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 657.5 ± 7.8 | 645.7 | 693.8 | 1.22 ± 0.02 |
+| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHqJW      -f -o 188392 -ignore-errors backhand-test/test-assets/test_appimage_plexamp/Plexamp-4.6.1.AppImage` | 540.5 ± 6.9 | 532.3 | 576.6 | 1.00 |
 
-| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+| Command | Mean [s] | Min [s] | Max [s] | Relative |
 |:---|---:|---:|---:|---:|
-| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHjpe -o 0 backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 983.0 ± 25.0 | 930.6 | 1026.3 | 1.46 ± 0.04 |
-| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BH6IH      -f -o 0 -ignore-errors backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 674.4 ± 5.4 | 667.8 | 697.8 | 1.00 |
+| `./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand --quiet -f -d /tmp/BHgBm -o 0 backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 1.018 ± 0.035 | 0.955 | 1.125 | 1.44 ± 0.06 |
+| `/usr/bin/unsquashfs -quiet -no-progress -d /tmp/BHNVr      -f -o 0 -ignore-errors backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 0.709 ± 0.018 | 0.699 | 0.812 | 1.00 |
 
 ## Heap Usage: `backhand/unsquashfs-v0.14.0` vs `squashfs-tools/unsquashfs-4.6.1`
 | Command | Peak Heap Memory Consumption |

backhand-cli/Cargo.toml

@@ -38,6 +38,8 @@ xz = ["backhand/xz"]
 xz-static = ["backhand/xz-static"]
 ## Enables gzip compression inside library and binaries
 gzip = ["backhand/gzip"]
+## Enables faster gzip de-compression only inside library and binaries
+gzip-zune-inflate = ["backhand/gzip-zune-inflate"]
 ## This library is licensed GPL and thus disabled by default
 lzo = ["backhand/lzo"]
 ## Enables zstd compression inside library and binaries

backhand-test/Cargo.toml

@@ -6,7 +6,6 @@ publish = false
 
 [dev-dependencies]
 backhand = { path = "../backhand", default-features = false }
-backhand-cli = { path = "../backhand-cli", default-features = false }
 assert_cmd = { version = "2.0.12", features = ["color", "color-auto"] }
 dir-diff = { git  = "https://github.com/wcampbell0x2a/dir-diff", branch = "add-checking-permissions" }
 tempfile = "3.9.0"
@@ -25,12 +24,13 @@ bench = false
 # testing only feature for testing vs squashfs-tools/unsquashfs
 __test_unsquashfs = []
 default = ["xz", "gzip", "zstd"]
-xz = ["backhand-cli/xz"]
-xz-static = ["backhand-cli/xz-static"]
-gzip = ["backhand-cli/gzip"]
+xz = ["backhand/xz"]
+xz-static = ["backhand/xz-static"]
+gzip = ["backhand/gzip"]
+gzip-zune-inflate = ["backhand/gzip-zune-inflate"]
 # this library is licensed GPL and thus disabled by default
-lzo = ["backhand-cli/lzo"]
-zstd = ["backhand-cli/zstd"]
+lzo = ["backhand/lzo"]
+zstd = ["backhand/zstd"]
 
 [[test]]
 name = "add"

backhand-test/tests/test.rs

@@ -11,9 +11,29 @@ use test_assets::TestAssetDef;
 use test_log::test;
 use tracing::info;
 
+#[cfg(feature = "gzip")]
+fn has_gzip_feature() -> bool {
+    true
+}
+
+#[cfg(not(feature = "gzip"))]
+fn has_gzip_feature() -> bool {
+    false
+}
+
 enum Verify {
     Extract,
 }
+
+fn only_read(assets_defs: &[TestAssetDef], filepath: &str, test_path: &str, offset: u64) {
+    test_assets::download_test_files(assets_defs, test_path, true).unwrap();
+
+    let og_path = format!("{test_path}/{filepath}");
+    let file = BufReader::new(File::open(&og_path).unwrap());
+    info!("calling from_reader");
+    let _ = FilesystemReader::from_reader_with_offset(file, offset).unwrap();
+}
+
 /// - Download file
 /// - Read into Squashfs
 /// - Into Filesystem
@@ -74,7 +94,7 @@ fn full_test(
 
 /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip -Xcompression-level 2 -always-use-fragments
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_00() {
     const FILE_NAME: &str = "out.squashfs";
     let asset_defs = [TestAssetDef {
@@ -83,12 +103,17 @@ fn test_00() {
         url: format!("https://wcampbell.dev/squashfs/testing/test_00/{FILE_NAME}"),
     }];
     const TEST_PATH: &str = "test-assets/test_00";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }
 
 /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip -Xcompression-level 2
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_01() {
     const FILE_NAME: &str = "out.squashfs";
     let asset_defs = [TestAssetDef {
@@ -97,7 +122,11 @@ fn test_01() {
         url: format!("https://wcampbell.dev/squashfs/testing/test_01/{FILE_NAME}"),
     }];
     const TEST_PATH: &str = "test-assets/test_01";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }
 
 /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp xz
@@ -156,7 +185,7 @@ fn test_05() {
 
 /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip -always-use-fragments
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_06() {
     const FILE_NAME: &str = "out.squashfs";
     let asset_defs = [TestAssetDef {
@@ -165,12 +194,16 @@ fn test_06() {
         url: format!("https://wcampbell.dev/squashfs/testing/test_06/{FILE_NAME}"),
     }];
     const TEST_PATH: &str = "test-assets/test_06";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }
 
 /// mksquashfs ./target/release/squashfs-deku out.squashfs -comp gzip
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_07() {
     const FILE_NAME: &str = "out.squashfs";
     let asset_defs = [TestAssetDef {
@@ -179,7 +212,12 @@ fn test_07() {
         url: format!("https://wcampbell.dev/squashfs/testing/test_07/{FILE_NAME}"),
     }];
     const TEST_PATH: &str = "test-assets/test_07";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }
 
 // mksquashfs ./target/release/squashfs-deku out.squashfs -comp xz -Xbcj arm
@@ -255,7 +293,7 @@ fn test_openwrt_netgear_ex6100v2() {
 }
 
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_appimage_plexamp() {
     const FILE_NAME: &str = "Plexamp-4.6.1.AppImage";
     let asset_defs = [TestAssetDef {
@@ -264,11 +302,16 @@ fn test_appimage_plexamp() {
         url: format!("https://plexamp.plex.tv/plexamp.plex.tv/desktop/{FILE_NAME}"),
     }];
     const TEST_PATH: &str = "test-assets/test_appimage_plexamp";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2dfe8, Verify::Extract, true);
+
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2dfe8, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0x2dfe8);
+    }
 }
 
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_appimage_firefox() {
     const FILE_NAME: &str = "firefox-108.0.r20221215175817-x86_64.AppImage";
     let asset_defs = [TestAssetDef {
@@ -277,7 +320,12 @@ fn test_appimage_firefox() {
         url: "https://github.com/srevinsaju/Firefox-Appimage/releases/download/firefox-v108.0.r20221215175817/firefox-108.0.r20221215175817-x86_64.AppImage".to_string(),
     }];
     const TEST_PATH: &str = "test-assets/test_appimage_firefox";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2f4c0, Verify::Extract, true);
+
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0x2f4c0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0x2f4c0);
+    }
 }
 
 /// Archer\ AX1800\(US\)_V3_221016.zip from https://www.tp-link.com/us/support/download/archer-ax1800/#Firmware
@@ -350,7 +398,7 @@ fn test_slow_archlinux_iso_rootfs() {
 }
 
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_many_files() {
     const FILE_NAME: &str = "many_files.squashfs";
     let asset_defs = [TestAssetDef {
@@ -360,11 +408,15 @@ fn test_many_files() {
     }];
 
     const TEST_PATH: &str = "test-assets/test_many_files";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }
 
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_many_dirs() {
     const FILE_NAME: &str = "many_dirs.squashfs";
     let asset_defs = [TestAssetDef {
@@ -374,11 +426,15 @@ fn test_many_dirs() {
     }];
 
     const TEST_PATH: &str = "test-assets/test_many_dirs";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }
 
 #[test]
-#[cfg(feature = "gzip")]
+#[cfg(any(feature = "gzip", feature = "gzip-zune-inflate"))]
 fn test_few_dirs_many_files() {
     const FILE_NAME: &str = "few_dirs_many_files.squashfs";
     let asset_defs = [TestAssetDef {
@@ -388,5 +444,10 @@ fn test_few_dirs_many_files() {
     }];
 
     const TEST_PATH: &str = "test-assets/test_few_dirs_many_files";
-    full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+
+    if has_gzip_feature() {
+        full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, Verify::Extract, true);
+    } else {
+        only_read(&asset_defs, FILE_NAME, TEST_PATH, 0);
+    }
 }

backhand/Cargo.toml

@@ -11,11 +11,12 @@ rust-version = "1.67.1"
 description = "Library for the reading, creating, and modification of SquashFS file systems"
 
 [dependencies]
-# for lib
 deku = "0.16.0"
-tracing = "0.1.40"
+# only error and warn traces in release mode
+tracing = { version = "0.1.40"}
 thiserror = "1.0.53"
 flate2 = { version = "1.0.28", optional = true }
+zune-inflate = { version = "0.2.54", optional = true, default-features = false, features = ["zlib"] }
 xz2 = { version = "0.1.7", optional = true }
 rust-lzo = { version = "0.6.2", optional = true }
 zstd = { version = "0.13.0", optional = true }
@@ -30,6 +31,9 @@ xz = ["dep:xz2"]
 xz-static = ["dep:xz2", "xz2?/static"]
 ## Enables gzip compression inside library and binaries
 gzip = ["dep:flate2"]
+## Enables faster gzip (de-compression only) inside library and binaries
+## Cannot be used with the `gzip` feature
+gzip-zune-inflate = ["dep:zune-inflate"]
 ## This library is licensed GPL and thus disabled by default
 lzo = ["dep:rust-lzo"]
 ## Enables zstd compression inside library and binaries
@@ -53,4 +57,4 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
-bench = false
+bench = false

backhand/src/compressor.rs

@@ -191,11 +191,18 @@ impl CompressionAction for DefaultCompressor {
     ) -> Result<(), BackhandError> {
         match compressor {
             Compressor::None => out.extend_from_slice(bytes),
-            #[cfg(feature = "gzip")]
+            #[cfg(all(feature = "gzip", not(feature = "gzip-zune-inflate")))]
             Compressor::Gzip => {
                 let mut decoder = flate2::read::ZlibDecoder::new(bytes);
                 decoder.read_to_end(out)?;
             }
+            #[cfg(all(feature = "gzip-zune-inflate", not(feature = "gzip")))]
+            Compressor::Gzip => {
+                use std::io::Write;
+                let mut decoder = zune_inflate::DeflateDecoder::new(&bytes);
+                let mut decompressed = decoder.decode_zlib().unwrap();
+                out.write_all(&mut decompressed)?;
+            }
             #[cfg(feature = "xz")]
             Compressor::Xz => {
                 let mut decoder = XzDecoder::new(bytes);