Merge 5df3fec into c43e5a6

wcampbell0x2a · May 27, 2024 · a78d411 · a78d411
2 parents c43e5a6 + 5df3fec
commit a78d411
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 46 deletions.
diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml
@@ -45,8 +45,7 @@ jobs:
           target: ${{ matrix.job.target }}
 
       - run: $BUILD_CMD build -p backhand-cli --bin add-backhand --bin replace-backhand --features xz-static --locked --target ${{ matrix.job.target }} --profile=dist
-      # default features, but replace gzip with gzip-zune-inflate
-      - run: $BUILD_CMD build -p backhand-cli --bin unsquashfs-backhand --locked --target ${{ matrix.job.target }} --profile=dist --no-default-features --features zstd,xz-static,gzip-zune-inflate
+      - run: $BUILD_CMD build -p backhand-cli --bin unsquashfs-backhand --locked --target ${{ matrix.job.target }} --profile=dist --no-default-features --features zstd,xz-static,gzip
       - name: archive
         run: |
           tar -czvf backhand-${{ matrix.job.target }}.tar.gz \

diff --git a/BENCHMARK.md b/BENCHMARK.md
@@ -63,61 +63,61 @@ $ ./bench.bash
 ### `openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 50.2 ± 3.1 | 43.7 | 58.3 | 1.02 ± 0.09 |
-| `backhand-dist` | 49.5 ± 3.2 | 44.2 | 57.8 | 1.00 |
-| `backhand-dist-musl` | 78.7 ± 5.1 | 66.0 | 90.2 | 1.59 ± 0.15 |
-| `squashfs-tools` | 111.8 ± 11.9 | 84.2 | 133.1 | 2.26 ± 0.28 |
+| `backhand-dist` | 52.2 ± 4.4 | 42.5 | 71.3 | 1.00 |
+| `backhand-dist-musl-v0.18.0` | 77.6 ± 4.4 | 70.5 | 89.9 | 1.49 ± 0.15 |
+| `backhand-dist-musl` | 77.4 ± 4.9 | 66.3 | 89.4 | 1.48 ± 0.16 |
+| `squashfs-tools` | 108.7 ± 13.0 | 84.0 | 137.6 | 2.08 ± 0.31 |
 ### `openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 51.0 ± 3.3 | 44.1 | 58.8 | 1.01 ± 0.08 |
-| `backhand-dist` | 50.3 ± 2.6 | 45.1 | 58.9 | 1.00 |
-| `backhand-dist-musl` | 78.0 ± 4.3 | 71.3 | 89.8 | 1.55 ± 0.12 |
-| `squashfs-tools` | 112.9 ± 16.7 | 90.1 | 159.0 | 2.24 ± 0.35 |
+| `backhand-dist` | 50.8 ± 3.6 | 44.3 | 65.2 | 1.00 |
+| `backhand-dist-musl-v0.18.0` | 78.6 ± 4.2 | 69.0 | 88.6 | 1.55 ± 0.14 |
+| `backhand-dist-musl` | 79.2 ± 4.6 | 69.6 | 89.4 | 1.56 ± 0.14 |
+| `squashfs-tools` | 112.1 ± 13.3 | 88.5 | 137.1 | 2.21 ± 0.31 |
 ### `870D97.squashfs`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 258.9 ± 12.5 | 228.4 | 275.5 | 1.35 ± 0.11 |
-| `backhand-dist` | 252.8 ± 14.9 | 220.2 | 273.7 | 1.32 ± 0.12 |
-| `backhand-dist-musl` | 429.0 ± 27.9 | 367.4 | 462.5 | 2.24 ± 0.20 |
-| `squashfs-tools` | 191.7 ± 12.4 | 168.9 | 215.0 | 1.00 |
+| `backhand-dist` | 255.4 ± 13.5 | 226.7 | 273.7 | 1.36 ± 0.11 |
+| `backhand-dist-musl-v0.18.0` | 433.4 ± 23.6 | 371.0 | 462.9 | 2.30 ± 0.19 |
+| `backhand-dist-musl` | 427.5 ± 26.5 | 369.0 | 463.1 | 2.27 ± 0.19 |
+| `squashfs-tools` | 188.3 ± 11.2 | 165.5 | 218.7 | 1.00 |
 ### `img-1571203182_vol-ubi_rootfs.ubifs`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 226.1 ± 7.9 | 214.7 | 240.6 | 1.00 |
-| `backhand-dist` | 226.4 ± 6.4 | 216.6 | 240.0 | 1.00 ± 0.04 |
-| `backhand-dist-musl` | 360.7 ± 10.8 | 342.2 | 384.5 | 1.60 ± 0.07 |
-| `squashfs-tools` | 288.9 ± 11.9 | 269.6 | 320.4 | 1.28 ± 0.07 |
+| `backhand-dist` | 227.1 ± 6.9 | 214.6 | 245.2 | 1.00 |
+| `backhand-dist-musl-v0.18.0` | 362.7 ± 12.0 | 341.7 | 385.4 | 1.60 ± 0.07 |
+| `backhand-dist-musl` | 361.0 ± 13.3 | 342.3 | 398.0 | 1.59 ± 0.08 |
+| `squashfs-tools` | 289.0 ± 12.5 | 262.1 | 322.1 | 1.27 ± 0.07 |
 ### `2611E3.squashfs`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 106.4 ± 4.3 | 97.9 | 117.6 | 1.00 ± 0.06 |
-| `backhand-dist` | 106.0 ± 4.7 | 97.5 | 119.3 | 1.00 |
-| `backhand-dist-musl` | 171.5 ± 6.3 | 151.2 | 185.2 | 1.62 ± 0.09 |
-| `squashfs-tools` | 187.3 ± 15.6 | 160.7 | 223.4 | 1.77 ± 0.17 |
+| `backhand-dist` | 104.7 ± 4.3 | 97.7 | 119.7 | 1.00 |
+| `backhand-dist-musl-v0.18.0` | 172.7 ± 8.2 | 153.5 | 187.8 | 1.65 ± 0.10 |
+| `backhand-dist-musl` | 172.5 ± 8.3 | 155.4 | 195.1 | 1.65 ± 0.10 |
+| `squashfs-tools` | 186.7 ± 18.0 | 152.5 | 223.5 | 1.78 ± 0.19 |
 ### `Plexamp-4.6.1.AppImage`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 490.1 ± 6.8 | 477.9 | 506.7 | 2.43 ± 0.10 |
-| `backhand-dist` | 511.3 ± 6.8 | 498.6 | 527.0 | 2.54 ± 0.10 |
-| `backhand-dist-musl` | 870.0 ± 8.2 | 856.6 | 890.6 | 4.32 ± 0.17 |
-| `squashfs-tools` | 201.4 ± 7.7 | 188.2 | 222.9 | 1.00 |
+| `backhand-dist` | 440.7 ± 4.7 | 430.8 | 454.3 | 2.19 ± 0.08 |
+| `backhand-dist-musl-v0.18.0` | 879.6 ± 9.0 | 862.1 | 901.0 | 4.36 ± 0.17 |
+| `backhand-dist-musl` | 489.9 ± 4.2 | 483.0 | 501.3 | 2.43 ± 0.09 |
+| `squashfs-tools` | 201.6 ± 7.4 | 188.1 | 217.4 | 1.00 |
 ### `crates-io.squashfs`
 | Command | Mean [s] | Min [s] | Max [s] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 1.423 ± 0.009 | 1.406 | 1.451 | 1.20 ± 0.01 |
-| `backhand-dist` | 1.392 ± 0.009 | 1.375 | 1.411 | 1.18 ± 0.01 |
-| `backhand-dist-musl` | 1.181 ± 0.004 | 1.174 | 1.190 | 1.00 |
-| `squashfs-tools` | 1.819 ± 0.042 | 1.706 | 1.908 | 1.54 ± 0.04 |
+| `backhand-dist` | 1.404 ± 0.009 | 1.388 | 1.430 | 5.64 ± 0.45 |
+| `backhand-dist-musl-v0.18.0` | 1.186 ± 0.005 | 1.175 | 1.197 | 4.76 ± 0.38 |
+| `backhand-dist-musl` | 1.182 ± 0.006 | 1.172 | 1.198 | 4.75 ± 0.38 |
+| `squashfs-tools` | 0.249 ± 0.020 | 0.208 | 0.289 | 1.00 |
 
 ## Heap Usage: `backhand/unsquashfs-master` vs `squashfs-tools/unsquashfs-4.6.1`
 ```
-$ cargo +stable build -p backhand-cli --bins --locked --profile=dist --no-default-features --features xz --features gzip-zune-inflate
+$ cargo +stable build -p backhand-cli --bins --locked --profile=dist --no-default-features --features xz --features gzip
 ```
 
 | Command | Peak Heap Memory Consumption |
 | :------ | ---------------------------: |
-| `heaptrack ./target/dist/unsquashfs-backhand --quiet -f -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 36.5MB |
+| `heaptrack ./target/dist/unsquashfs-backhand --quiet -f -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 41.7MB |
 | `heaptrack unsquashfs -quiet -no-progress -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 75.7MB |
 
 | Command | Peak Heap Memory Consumption |

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/backhand/Cargo.toml b/backhand/Cargo.toml
@@ -19,14 +19,16 @@ rustdoc-args = ["--cfg", "docsrs"]
 deku = "0.17.0"
 tracing = { version = "0.1.40" }
 thiserror = "1.0.59"
-flate2 = { version = "1.0.30", optional = true }
+flate2 = { version = "1.0.30", optional = true, features = ["zlib-ng"] }
 zune-inflate = { version = "0.2.54", optional = true, default-features = false, features = ["zlib"] }
 xz2 = { version = "0.1.7", optional = true }
 rust-lzo = { version = "0.6.2", optional = true }
 zstd = { version = "0.13.1", optional = true }
 zstd-safe = { version = "7.0.0", optional = true }
 rustc-hash = "1.1.0"
 document-features = { version = "0.2.8", optional = true }
+# Use the fastest implementation (libz-ng) for flate2 but remove dependence on CMake
+libz-sys = { version = "1.1.18", features = ["zlib-ng-no-cmake-experimental-community-maintained"], default-features = false }
 
 [features]
 default = ["xz", "gzip", "zstd"]

diff --git a/bench.bash b/bench.bash
@@ -1,24 +1,25 @@
 #!/bin/bash
 set -ex
 
-LAST_RELEASE="v0.17.0"
+LAST_RELEASE="v0.18.0"
 
-BACKHAND_LAST_RELEASE="./last-release/bin/unsquashfs-backhand"
+BACKHAND_LAST_RELEASE_MUSL_DIR="./last-release-musl/"
+BACKHAND_LAST_RELEASE_MUSL="./last-release-musl/unsquashfs-backhand"
 BACKHAND="./target/dist/unsquashfs-backhand"
 BACKHAND_MUSL="./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand"
 UNSQUASHFS="/usr/bin/unsquashfs"
 
 # Using dynamic linked xz for perf reasons and matching unsquashfs in this testing
-FLAGS="--bins --locked --profile=dist --no-default-features --features xz --features gzip-zune-inflate --features zstd"
+FLAGS="--bins --locked --profile=dist --no-default-features --features xz --features zstd --features gzip"
 
 bench () {
     echo ""
-    file $1
+    binwalk $1 --include squashfs
     hyperfine --sort command --runs 50 --warmup 10 \
-        --command-name backhand-dist-${LAST_RELEASE} \
-        "$BACKHAND_LAST_RELEASE --quiet -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \
         --command-name backhand-dist \
         "$BACKHAND --quiet -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \
+        --command-name backhand-dist-musl-${LAST_RELEASE} \
+        "$BACKHAND_LAST_RELEASE_MUSL --quiet -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \
         --command-name backhand-dist-musl \
         "$BACKHAND_MUSL --quiet -f -d $(mktemp -d /tmp/BHXXX) -o $(rz-ax $2) $1" \
         --command-name squashfs-tools \
@@ -28,8 +29,10 @@ bench () {
 }
 
 rm -rf bench-results
-cross +stable build -p backhand-cli $FLAGS --target x86_64-unknown-linux-musl
-cargo +stable install backhand-cli --git https://github.com/wcampbell0x2a/backhand.git --root last-release --tag "$LAST_RELEASE" $FLAGS
+mkdir -vp $BACKHAND_LAST_RELEASE_MUSL_DIR
+curl -L https://github.com/wcampbell0x2a/backhand/releases/download/$LAST_RELEASE/backhand-$LAST_RELEASE-x86_64-unknown-linux-musl.tar.gz --output backhand-$LAST_RELEASE-x86_64-unknown-linux-musl.tar.gz
+tar -xvf backhand-$LAST_RELEASE-x86_64-unknown-linux-musl.tar.gz  -C $BACKHAND_LAST_RELEASE_MUSL_DIR
+cargo +stable build -p backhand-cli $FLAGS --target x86_64-unknown-linux-musl
 cargo +stable build -p backhand-cli $FLAGS
 mkdir -p bench-results
 
@@ -42,7 +45,7 @@ bench "backhand-test/test-assets/test_re815_xev160/870D97.squashfs" 0x0 2_re815
 # xz
 bench "backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs" 0x0 3_ax18000
 # xz
-#bench "test-assets/test_archlinux_iso_rootfs/airootfs.sfs" 0x0
+# bench "test-assets/test_archlinux_iso_rootfs/airootfs.sfs" 0x0
 # xz
 bench "backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs" 0x0 4_er605
 # gzip