diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c1fdf8..59fea33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,9 +63,9 @@ jobs: command: sdist args: -o dist - run: | - python -m venv /tmp/fastbz2-sdist-test - /tmp/fastbz2-sdist-test/bin/pip install pytest dist/*.tar.gz - /tmp/fastbz2-sdist-test/bin/pytest tests/test_install.py + python -m venv /tmp/fbz-sdist-test + /tmp/fbz-sdist-test/bin/pip install pytest dist/*.tar.gz + /tmp/fbz-sdist-test/bin/pytest tests/test_install.py - uses: actions/upload-artifact@v7 with: name: wheels-sdist diff --git a/Cargo.toml b/Cargo.toml index d14001f..ffcdad0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,21 @@ [package] -name = "fastbz2" +name = "fbz" version = "0.1.7" edition = "2024" rust-version = "1.91" license = "Apache-2.0" -description = "Compression-format research workbench with fast bzip2, gzip, and ZIP decompression" -repository = "https://github.com/AnswerDotAI/fastbz2" -homepage = "https://github.com/AnswerDotAI/fastbz2" -documentation = "https://github.com/AnswerDotAI/fastbz2" +description = "Compression-format research workbench with fast bzip2, gzip, LZ4, and ZIP decompression" +repository = "https://github.com/AnswerDotAI/fbz" +homepage = "https://github.com/AnswerDotAI/fbz" +documentation = "https://github.com/AnswerDotAI/fbz" [lib] -name = "fastbz2" +name = "fbz" crate-type = ["cdylib", "rlib"] [[bin]] -name = "fastbz2" -path = "src/bin/fastbz2.rs" +name = "fbz" +path = "src/bin/fbz.rs" test = false [profile.release] @@ -33,6 +33,7 @@ rayon = "1.12.0" serde_json = "1.0.151" tempfile = "3.27.0" tar = { version = "0.4.46", default-features = false } +twox-hash = { version = "2.0.0", default-features = false, features = ["xxhash32"] } zip = { version = "8.6.0", default-features = false } [dev-dependencies] @@ -40,6 +41,7 @@ crabz2 = { version = "0.4.0", features = ["parallel"] } flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } libc = "0.2.175" libbz2-rs-sys = { version = "0.2.5", default-features = false, features = ["std"] } +lz4_flex = "0.14.0" [features] python = ["dep:pyo3"] diff --git a/DEV.md b/DEV.md index 7f082a6..113a32b 100644 --- a/DEV.md +++ b/DEV.md @@ -1,6 +1,6 @@ # Development -`fastbz2` is a mixed Rust/PyO3 project. The Rust crate is the implementation and public Rust API; `python/fastbz2/` is the public Python package over the private `fastbz2._core` extension. +`fbz` is a mixed Rust/PyO3 project. The Rust crate is the implementation and public Rust API; `python/fbz/` is the public Python package over the private `fbz._core` extension. ## Architecture @@ -13,16 +13,18 @@ src/decoder.rs bzip2 block machinery and 12-bit Huffman fast tables src/format.rs cheap structural scan for header and marker candidates src/gzip.rs gzip framing, LSB-first DEFLATE implementation, CRC32, and reports src/deflate.rs format-neutral raw-DEFLATE API shared by gzip and ZIP +src/lz4.rs safe LZ4 frame/block decoder and independent-block scheduling +src/history.rs shared overlapping LZ back-reference expansion src/output.rs owned/borrowed decoded-output sink abstraction src/pipeline.rs shared ordered, byte-budgeted, staged worker scheduler src/index.rs stable persistent index format src/indexed.rs seekable decoded view and block cache src/lib.rs public Rust API and private PyO3 binding src/source.rs owned and memory-mapped compressed sources -src/bin/fastbz2/archive_extract.rs shared same-filesystem staging and atomic commit -src/bin/fastbz2/tar_extract.rs bounded decode-to-tar bridge -src/bin/fastbz2/zip_extract.rs ZIP parsing policy and adaptive entry extraction -python/fastbz2/ thin Python I/O wrapper over fastbz2._core +src/bin/fbz/archive_extract.rs shared same-filesystem staging and atomic commit +src/bin/fbz/tar_extract.rs bounded decode-to-tar bridge +src/bin/fbz/zip_extract.rs ZIP parsing policy and adaptive entry extraction +python/fbz/ thin Python I/O wrapper over fbz._core build_backend.py stage the native CLI for PEP 517 wheel builds tests/corpus/ selected upstream conformance and corruption fixtures tests/ Rust CLI/corpus and Python API integration tests @@ -33,19 +35,21 @@ The current bzip2 scanner deliberately does not treat 48-bit marker matches or l The gzip decoder is an in-repo RFC 1952/RFC 1951 implementation rather than a wrapper around a production codec. It parses optional headers and concatenated members, decodes stored/fixed/dynamic blocks, maintains the 32 KiB LZ77 history, and validates FHCRC, CRC32, and ISIZE. For large inputs, independently discovered dynamic-block boundaries seed unknown history with compact markers. Primary jobs compute the CRC of each known clean suffix before ordered resolution. Resolution workers resolve the marker prefix, hash that prefix, and combine the two CRCs without rescanning the clean bytes. A marker-free history switches the same decoder to byte output. Reports retain member boundaries, DEFLATE block ranges, and accepted/fallback chunk counts. `crc32fast` is the sole production helper; `flate2` is dev-only. -The decoder remains independent of files, threads, Python, and the CLI. Parallel scanning/decoding and indexed seeking are layered over it. Native workers never call Python. Large offsets use explicit 64-bit bit/byte types, and speculative block-marker hits are accepted only when they form an exact stream chain with valid block and combined stream CRCs. +The decoder remains independent of files, Python, and the CLI. Parallel scanning/decoding and indexed seeking are layered over it. Native workers never call Python. Large offsets use explicit 64-bit bit/byte types, and speculative block-marker hits are accepted only when they form an exact stream chain with valid block and combined stream CRCs. -Core decode APIs report completed compressed and decoded byte counts without knowing anything about terminals. The CLI selects bzip2, gzip, or ZIP by a recognised extension and falls back to magic for stdin or unknown names. It layers delayed, rate-limited TTY progress rendering over the shared callbacks; redirected stderr and `--quiet` produce no progress output. Decoded files use same-directory temporary files and atomic persistence, then inherit the compressed input's modification time and permissions. `--rm` removes an input only after decode, persistence, and metadata copying all succeed. An `OutputSink` wrapper enforces output-size limits, so each decoder has one code path for files, stdout, validation, listing, and archive extraction. +Core decode APIs report completed compressed and decoded byte counts without knowing anything about terminals. The CLI selects bzip2, gzip, LZ4, or ZIP by a recognised extension and falls back to magic for stdin or unknown names. It layers delayed, rate-limited TTY progress rendering over the shared callbacks; redirected stderr and `--quiet` produce no progress output. Decoded files use same-directory temporary files and atomic persistence, then inherit the compressed input's modification time and permissions. `--rm` removes an input only after decode, persistence, and metadata copying all succeed. An `OutputSink` wrapper enforces output-size limits, so each decoder has one code path for files, stdout, validation, listing, and archive extraction. + +The LZ4 decoder parses current frames, concatenation, and skippable frames itself. It validates descriptor bits and XXH32 header, block, and content checksums, and bounds every literal and match before writing. Independent compressed blocks become ordinary `pipeline::Job`s, reserving the frame's declared maximum decoded block size; stored blocks borrow their source bytes and reserve no decoded allocation. Frames containing only stored blocks without block checksums bypass the worker pool because their only remaining work is ordered output and optional content hashing. Retained results remain charged at their allocation capacity rather than logical length, so highly compressible blocks cannot understate memory use. The coordinator commits results in source order and updates the content checksum. Linked frames use the same parser, block decoder, output sink, progress, and report path, but decode serially with a rolling 64 KiB history. This is one code path with a scheduling branch, not separate serial and parallel implementations. Concatenated frames currently run in frame order; parallelising across small independent frames is a possible measured optimization, not pre-built machinery. Tar format semantics use the mature `tar` crate, pinned from 0.4.46 and built without its optional xattr feature. It handles streaming GNU/PAX/long-name/link entries and confines extracted paths to the destination. A zero-capacity rendezvous channel transfers each owned decoder chunk and its live suffix offset to `tar::Archive`. The channel queues no chunks and applies backpressure. `tar::Archive` pulls data through `Read`, which copies once from the current chunk into its request buffer. Extraction writes immediately into a same-filesystem temporary directory, drains all trailing tar padding so codec validation completes, then preflights every destination conflict and moves entries into place with renames. Multiple inputs remain sequential so their per-codec worker pools cannot oversubscribe the global thread budget. -ZIP structure semantics use `zip` 8.6.0 with all codec features disabled. The crate parses the central directory, Zip64 fields, data descriptors, names, modes, symlink kinds, and timestamp extra fields; fastbz2 reads each raw stored/DEFLATE range and sends it through its own validated codec path. Because the crate intentionally collapses duplicate raw names into its index map, a small bounds-checked central-record count detects and rejects that ambiguity before using its metadata. Parsing also rejects encryption, unsupported methods, escaping or equivalent paths, non-directory ancestors, overlapping data ranges, and ranges crossing the central directory. An aggregate declared-size check runs before extraction, and a per-entry sink prevents output exceeding its declaration before size and CRC32 are checked. ZIP and tar share the same staging/preflight/rename implementation. +ZIP structure semantics use `zip` 8.6.0 with all codec features disabled. The crate parses the central directory, Zip64 fields, data descriptors, names, modes, symlink kinds, and timestamp extra fields; fbz reads each raw stored/DEFLATE range and sends it through its own validated codec path. Because the crate intentionally collapses duplicate raw names into its index map, a small bounds-checked central-record count detects and rejects that ambiguity before using its metadata. Parsing also rejects encryption, unsupported methods, escaping or equivalent paths, non-directory ancestors, overlapping data ranges, and ranges crossing the central directory. An aggregate declared-size check runs before extraction, and a per-entry sink prevents output exceeding its declaration before size and CRC32 are checked. ZIP and tar share the same staging/preflight/rename implementation. ZIP scheduling deliberately uses one parallelism level at a time. A sole DEFLATE entry at least 16 MiB compressed, or an entry at least 64 MiB in a multi-entry archive, uses all requested workers inside the raw DEFLATE decoder. Remaining entries run serial inner decoders concurrently on one Rayon pool. This keeps thread ownership and memory behaviour obvious, avoids nested oversubscription, and lets many ordinary entries naturally absorb stragglers through work stealing. -The shared `pipeline.rs` scheduler provides ordered results, byte-budgeted admission, cancellation, and a staged priority queue. Bzip2 uses the rolling candidate path: workers reserve the maximum possible decoded block size, then shrink that reservation to actual retained output until ordered validation consumes or rejects it. Gzip uses the staged path: native workers alternate speculative DEFLATE decoding with higher-priority marker resolution, while the coordinator advances only the 32 KiB dependency windows and emits resolved chunks in order. Decode results and outstanding resolution results have separate bounded horizons, preventing either dependency stalls or unbounded memory. +The shared `pipeline.rs` scheduler provides ordered results, byte-budgeted admission, cancellation, and a staged priority queue. Bzip2 uses the rolling candidate path: workers reserve the maximum possible decoded block size, then shrink that reservation to actual retained output until ordered validation consumes or rejects it. Gzip uses the staged path: native workers alternate speculative DEFLATE decoding with higher-priority marker resolution, while the coordinator advances only the 32 KiB dependency windows and emits resolved chunks in order. LZ4 independent blocks use the simpler ordered-job path and charge each retained allocation against the budget. Decode results and outstanding resolution results have bounded horizons, preventing dependency stalls from causing unbounded memory. -The bzip2 decoder is safe scalar Rust designed for LLVM auto-vectorisation. Huffman decoding uses a 4096-entry direct table for codes up to 12 bits and canonical fallback for longer codes. Gzip uses full canonical lookup tables packed into `u16`, a branch-free 64 KiB marker-resolution lookup for large chunks, and `crc32fast::Hasher::combine` so CRC scanning runs with the resolution workers rather than serially in the coordinator. The only unsafe codec operation marks a just-initialized `Vec` result as initialized after writing every spare-capacity byte. Add architecture-specific SIMD or further cross-codec abstraction only after profiling; `libbz2-rs-sys` and `flate2` remain dev-only differential oracles. +The bzip2 decoder is safe scalar Rust designed for LLVM auto-vectorisation. Huffman decoding uses a 4096-entry direct table for codes up to 12 bits and canonical fallback for longer codes. Gzip uses full canonical lookup tables packed into `u16`, a branch-free 64 KiB marker-resolution lookup for large chunks, and `crc32fast::Hasher::combine` so CRC scanning runs with the resolution workers rather than serially in the coordinator. Gzip byte/marker output and LZ4 share `history::extend_match`, whose doubling copies handle overlapping matches in logarithmically many operations; format-specific history validation remains at each call site. The only unsafe codec operation marks a just-initialized `Vec` result as initialized after writing every spare-capacity byte. Add architecture-specific SIMD or further cross-codec abstraction only after profiling; `libbz2-rs-sys`, `flate2`, and `lz4_flex` remain dev-only differential oracles. ## Commands @@ -55,7 +59,7 @@ cargo test --release cargo check --all-features cargo build --release --bins python tools/stage_binaries.py -maturin develop --release +uv pip install --reinstall --no-deps -e . pytest -q ship-rs-build ``` @@ -64,68 +68,94 @@ Run `cargo fmt --check` after Rust edits and `chkstyle` after Python edits once ## Correctness and performance acceptance -The normal release test path decodes selected valid and corrupt cases from the maintained upstream `bzip2-testfiles` collection. Generated byte distributions add differential coverage. Valid bzip2 outputs are compared byte-for-byte with `libbz2-rs-sys`. Gzip and raw-DEFLATE tests cover stored, fixed-Huffman, and dynamic-Huffman blocks; optional headers and FHCRC; concatenated members; exact end-of-stream boundaries; truncation; and trailer corruption across varied inputs and compression levels generated by `flate2`. Both oracles are dev-only and never part of production decoding. CLI tests generate tar and ZIP archives and cover wrappers and extension dispatch, stored/DEFLATE entries, Zip64 and streaming descriptors, metadata and safe symlinks, stdin, raw-tar output, output limits, overwrite preflight, late checksum failure, atomicity, and traversal confinement. +User-facing comparisons between installable CLIs are recorded only in [README Performance](README.md#performance). This file documents fixture reproduction, regression gates, and implementation-oriented diagnostics. + +The normal release test path decodes selected valid and corrupt cases from the maintained upstream `bzip2-testfiles` collection. Generated byte distributions add differential coverage. Valid bzip2 outputs are compared byte-for-byte with `libbz2-rs-sys`. Gzip and raw-DEFLATE tests cover stored, fixed-Huffman, and dynamic-Huffman blocks; optional headers and FHCRC; concatenated members; exact end-of-stream boundaries; truncation; and trailer corruption across varied inputs and compression levels generated by `flate2`. LZ4 tests use `lz4_flex` to generate a matrix spanning empty, repetitive, byte-distribution, and pseudorandom inputs; all four standard block sizes; independent and linked blocks; and every checksum/content-size combination. The production decoders do not depend on any oracle. CLI tests additionally cover LZ4 extension/magic dispatch, reports, limits, corruption, and `.tar.lz4` streaming extraction alongside the existing bzip2, gzip, tar, and ZIP policy cases. The normal release path contains warmed end-to-end performance regression gates capped at 1.3 times each oracle, allowing for noise on shared runners. The gzip gates independently exercise a highly compressible LZ77-heavy shape and an incompressible literal-heavy shape against `flate2`; the bzip2 gate uses `libbz2-rs-sys`. Representative local acceptance remains 1.2 times the corresponding oracle. The ignored full-wiki gzip test applies that threshold to rapidgzip-rust. Keep the whole release test suite below five seconds on the primary development laptop; individual timed workloads should normally be about 0.1 seconds or less. +### Local LZ4 benchmark reproduction and diagnostics + +`tests/lz4_perf.rs` decodes `meta/simplewiki-first-5pct.xml.bz2` before timing, then uses dev-only `lz4_flex` to create a standard Max4MB independent-block LZ4 frame with a content checksum. Fixture construction and both warm-ups are outside the measured intervals. One test then measures exactly one validation run of each CLI and records child-process memory without task-inspection permissions: + +```bash +cargo test --release --test lz4_perf lz4_cli_comparison -- --ignored --exact --nocapture +cargo test --release --test lz4_perf lz4_thread_sweep -- --ignored --exact --nocapture +``` + +Regenerate the underlying SimpleWiki fixture using the shared instructions in [Local Wikipedia benchmarks](#local-wikipedia-benchmarks); no `.lz4` fixture is stored. This section retains only implementation diagnostics. + +The one-measurement-per-count scaling diagnostic explains the four-worker automatic limit: + +| Workers | Validation | Peak RSS | +|---:|---:|---:| +| 1 | 109.727 ms | 46.5 MiB | +| 2 | 54.995 ms | 58.9 MiB | +| 4 | 51.367 ms | 71.0 MiB | +| 6 | 55.286 ms | 79.2 MiB | +| 8 | 52.010 ms | 87.2 MiB | +| 12 | 55.300 ms | 103.4 MiB | +| 18 | 51.915 ms | 124.2 MiB | + +Four workers are at the front of the noisy plateau while using much less memory than 8–18. Automatic mode therefore uses at most four LZ4 workers; an explicit `-P N` still requests exactly `N`. + +`lz4_shape_diagnostics` isolates the two simple structural experiments: + +```bash +cargo test --release --test lz4_perf lz4_shape_diagnostics -- --ignored --exact --nocapture +``` + +Replacing LZ4's offset-sized copy loop with the shared exponential back-reference expander reduced a single 4 MiB long-match decode from 10.588 ms to 0.278 ms. On a 16 MiB stored-only frame, the old worker path took 1.425 ms; bypassing its no-work pool took 1.286 ms, essentially flat in speed but with no unnecessary threads. Both figures are single measured runs after one warm-up per path. + ### Local archive extraction benchmarks `tests/archive_perf.rs` measures the tar layer on the real `meta/simplewiki-first-5pct.xml.bz2` corpus. Fixture decoding and gzip/bzip2 recompression finish before timing. Each ignored test warms one target and measures it once. Run only the implementation changed: ```bash -cargo test --release --test archive_perf tgz_fastbz2_overhead -- --ignored --exact --nocapture +cargo test --release --test archive_perf tgz_fbz_overhead -- --ignored --exact --nocapture cargo test --release --test archive_perf tgz_system_reference -- --ignored --exact --nocapture -cargo test --release --test archive_perf tbz2_fastbz2_overhead -- --ignored --exact --nocapture +cargo test --release --test archive_perf tbz2_fbz_overhead -- --ignored --exact --nocapture cargo test --release --test archive_perf tbz2_system_reference -- --ignored --exact --nocapture cargo test --release --test archive_perf tar_crate_reference -- --ignored --exact --nocapture cargo test --release --test archive_perf tgz_output_cadence -- --ignored --exact --nocapture ``` -These are single runs after owned-suffix transfer and the 512 KiB gzip grid change: +These are the internal overhead measurements after owned-suffix transfer and the 512 KiB gzip grid change: -| Format | Raw decode | fastbz2 extraction | Extraction/raw | System `tar` | Extraction/system | -|---|---:|---:|---:|---:|---:| -| `.tgz` | 39.919 ms | 56.850 ms | 1.424x | 117.962 ms | 0.482x | -| `.tar.bz2` | 148.411 ms | 151.783 ms | 1.023x | 1.168 s | 0.130x | +| Format | Raw decode | Extraction/raw | +|---|---:|---:| +| `.tgz` | 39.919 ms | 1.424x | +| `.tar.bz2` | 148.411 ms | 1.023x | -Direct extraction of the uncompressed in-memory tar through the `tar` crate took 32.069 ms. Raw gzip decode plus direct tar extraction totals 71.988 ms. The combined pipeline takes 56.850 ms and hides 15.138 ms, or 47%, of the direct tar work. +Direct extraction of the uncompressed in-memory tar through the `tar` crate took 32.069 ms. Raw gzip decode plus direct tar extraction totals 71.988 ms. The combined pipeline hides 15.138 ms, or 47%, of the direct tar work. -The cadence benchmark identified ordered gzip output as the main overlap limit. With a 1 MiB speculative grid, output began at 8.491 ms, reached 25% at 29.595 ms, and completed at 33.724 ms. A 512 KiB grid began at 4.798 ms, reached 25% at 23.993 ms, and completed at 32.915 ms. A 256 KiB grid emitted earlier but slowed raw decode to 38.073 ms and extraction to 57.792 ms. The 512 KiB grid gave the best measured balance. Computing each clean suffix CRC in its primary job moved 25% output to 19.818 ms, 75% to 30.355 ms, and completion to 30.678 ms. The corresponding extraction run was effectively flat at 56.850 ms. Tar cannot process later bytes while an earlier ordered gzip segment remains incomplete. A custom tar parser would not remove that dependency. A one-chunk channel buffer regressed extraction to 59.698 ms, so the bridge retains its zero-capacity rendezvous. +The cadence benchmark identified ordered gzip output as the main overlap limit. With a 1 MiB speculative grid, output began at 8.491 ms, reached 25% at 29.595 ms, and completed at 33.724 ms. A 512 KiB grid began at 4.798 ms, reached 25% at 23.993 ms, and completed at 32.915 ms. A 256 KiB grid emitted earlier but slowed raw decode to 38.073 ms and extraction to 57.792 ms. The 512 KiB grid gave the best measured balance. Computing each clean suffix CRC in its primary job moved 25% output to 19.818 ms, 75% to 30.355 ms, and completion to 30.678 ms. The corresponding extraction run was effectively flat. Tar cannot process later bytes while an earlier ordered gzip segment remains incomplete. A custom tar parser would not remove that dependency. A one-chunk channel buffer regressed extraction to 59.698 ms, so the bridge retains its zero-capacity rendezvous. -System `tar` remains the external reference and 1.2x remains the research target. Raw-tar output is a lower bound rather than an extractor reference. The fastbz2 tests use a broad 3x raw-decode regression guard. Keep the measurements single-run; change an implementation before rerunning it. +Raw-tar output is a lower bound rather than an extractor reference. The fbz tests use a broad 3x raw-decode regression guard. Keep the measurements single-run; change an implementation before rerunning it. -### Local ZIP benchmarks +### Local ZIP benchmark reproduction `tests/zip_perf.rs` creates two deterministic ZIPs from the same 84,423,012-byte SimpleWiki prefix used by the tar benchmarks: one DEFLATE entry, and 18 equal-sized DEFLATE entries. `tests/support::simplewiki_prefix` decodes `meta/simplewiki-first-5pct.xml.bz2` before timing, and the ZIP builder then runs before timing. Thus regenerating the dataset is exactly the SimpleWiki 5% procedure below; no ZIP fixture is stored or needs hand-maintained lengths. Each speed test warms its selected executable once, measures it once, and verifies all extracted bytes. Run only the row whose implementation changed: ```bash -cargo test --release --test zip_perf zip_single_fastbz2 -- --ignored --exact --nocapture +cargo test --release --test zip_perf zip_single_fbz -- --ignored --exact --nocapture cargo test --release --test zip_perf zip_single_unzip -- --ignored --exact --nocapture -cargo test --release --test zip_perf zip_many_fastbz2 -- --ignored --exact --nocapture +cargo test --release --test zip_perf zip_many_fbz -- --ignored --exact --nocapture cargo test --release --test zip_perf zip_many_unzip -- --ignored --exact --nocapture ``` -The established external baseline is `UnZip 6.00 of 20 April 2009, by Info-ZIP, with modifications by Apple Inc.` The measured archive was 25.8 MiB and decoded to 80.5 MiB: - -| Shape | fastbz2 | Info-ZIP `unzip` | fastbz2/reference | -|---|---:|---:|---:| -| One entry | 35.571 ms | 355.235 ms | 0.100x | -| 18 entries | 29.700 ms | 360.008 ms | 0.083x | - -Both fastbz2 measurements used the default automatic worker selection, resolving to 18 cores on the primary machine. `FASTBZ2_THREADS` is only an optional diagnostic override; normal tests and benchmarks should leave it unset so `-P 0` follows the machine automatically. +`FBZ_THREADS` is an optional diagnostic override; normal tests and benchmarks should leave it unset so `-P 0` follows the machine automatically. The ungated child-process memory diagnostics use the same many-entry fixture: ```bash -cargo test --release --test zip_perf zip_many_fastbz2_process_metrics -- --ignored --exact --nocapture +cargo test --release --test zip_perf zip_many_fbz_process_metrics -- --ignored --exact --nocapture cargo test --release --test zip_perf zip_many_unzip_process_metrics -- --ignored --exact --nocapture ``` -The single sampled runs measured fastbz2 at 66.4 MiB peak RSS and 36.7 MiB physical footprint, versus `unzip` at 2.8 MiB RSS and 2.4 MiB physical footprint. That is the explicit cost of decoding 18 entries concurrently; it remains small in absolute terms and avoids the much larger speculative-history footprint of parallelising each small entry internally. - -Legacy randomized blocks produced by bzip2 versions before 0.9.5 are intentionally unsupported. Supporting that obsolete format would add complexity to the production decoder for data that is not realistically encountered today. +These diagnostics quantify the memory cost of concurrent entry decoding without duplicating the user-facing results. ### Local Wikipedia benchmarks @@ -135,20 +165,20 @@ This section retains in-process and library-oriented research comparisons that a | Decoder | Mode | Seconds | |---|---|---:| -| fastbz2 | parallel, 18 threads, streaming sink | 2.515 | +| fbz | parallel, 18 threads, streaming sink | 2.515 | | crabz2 0.4.0 | parallel | 4.460 | | bzip2 | serial CLI | 20.310 | | pbzip2 1.1.13 | CLI | 20.240 | | libbz2-rs 0.2.5 | serial, in process | 20.700 | -| fastbz2 | serial, in process | 21.279 | +| fbz | serial, in process | 21.279 | The first 1,000 streams of English Wikipedia (`654,362,682` bytes compressed, `2,715,335,085` bytes decoded, 99,853 pages) exercise scheduling across many short concatenated streams: | Decoder | Mode | Seconds | |---|---|---:| | crabz2 0.4.0 | parallel, in process | 3.815 | -| fastbz2 | parallel, 18 threads, in process | 3.881 | -| fastbz2 | serial, in process | 37.198 | +| fbz | parallel, 18 threads, in process | 3.881 | +| fbz | serial, in process | 37.198 | | crabz2 0.4.0 | serial, in process | 40.602 | | pbzip2 1.1.13 | 18-thread CLI + byte comparison | 88.080 | | bzip2 | serial CLI + byte comparison | 92.960 | @@ -167,14 +197,14 @@ The full bzip2 confirmation streams to a counting sink and validates every block cargo test --release --test wiki_perf simplewiki_full -- --ignored --exact --nocapture ``` -The fastbz2-only gzip test warms with `meta/simplewiki-first-5pct.xml.gz` and performs one full-dump validation. The ratio test warms both executables, measures each full dump once, and fails above 1.2x the sibling rapidgzip-rust checkout: +The fbz-only gzip test warms with `meta/simplewiki-first-5pct.xml.gz` and performs one full-dump validation. The ratio test warms both executables, measures each full dump once, and fails above 1.2x the sibling rapidgzip-rust checkout: ```bash -cargo test --release --test wiki_perf gzip_fastbz2_validation -- --ignored --exact --nocapture +cargo test --release --test wiki_perf gzip_fbz_validation -- --ignored --exact --nocapture cargo test --release --test wiki_perf gzip_reference_ratio -- --ignored --exact --nocapture ``` -The default selects available parallelism automatically. `FASTBZ2_THREADS` is an optional diagnostic override, and `RAPIDGZIP_BIN` can point at another reference executable. The warm-up is deliberately the small fixture, not an unreported repeat of the measured full workload. +The default selects available parallelism automatically. `FBZ_THREADS` is an optional diagnostic override, and `RAPIDGZIP_BIN` can point at another reference executable. The warm-up is deliberately the small fixture, not an unreported repeat of the measured full workload. Time/CPU/RSS and ungated macOS physical-footprint diagnostics are separate because process inspection can perturb sub-second parallel timings: @@ -188,13 +218,13 @@ The metrics helper uses `wait4` and, on macOS, `proc_pid_rusage`'s `ri_phys_foot The 1,000-stream enwiki comparison has a separate ignored test for each implementation and mode so a changed decoder can be measured without rerunning unchanged baselines. Each test reads the compressed fixture before starting its single timed decode, with no warmups or repeats: ```bash -cargo test --release --test wiki_perf enwiki_first_1000_fastbz2_parallel -- --ignored --exact --nocapture +cargo test --release --test wiki_perf enwiki_first_1000_fbz_parallel -- --ignored --exact --nocapture cargo test --release --test wiki_perf enwiki_first_1000_crabz2_parallel -- --ignored --exact --nocapture -cargo test --release --test wiki_perf enwiki_first_1000_fastbz2_serial -- --ignored --exact --nocapture +cargo test --release --test wiki_perf enwiki_first_1000_fbz_serial -- --ignored --exact --nocapture cargo test --release --test wiki_perf enwiki_first_1000_crabz2_serial -- --ignored --exact --nocapture ``` -It compares fastbz2 and crabz2 in parallel and serial modes. Automatic parallelism is the default; `FASTBZ2_THREADS` can give both parallel decoders an explicit count for a diagnostic comparison. Each implementation validates the bzip2 CRCs; the benchmark also checks the exact decoded length. +It compares fbz and crabz2 in parallel and serial modes. Automatic parallelism is the default; `FBZ_THREADS` can give both parallel decoders an explicit count for a diagnostic comparison. Each implementation validates the bzip2 CRCs; the benchmark also checks the exact decoded length. The decoded lengths and the 5% BLAKE3 in `tests/wiki_perf.rs` are acceptance values, not parameters used by the decoder. They prevent a truncated decode from appearing artificially fast without reading a separate multi-gigabyte reference during each timed run. Regenerating a fixture requires independently validating it and updating the corresponding acceptance value. @@ -233,7 +263,7 @@ Line 1000 is the start of stream 1001 because byte zero is stream 1 and is absen To create the separately useful well-formed parser fixture, append only the XML root close after decoding; those 13 bytes are deliberately excluded from `ENWIKI_1000_LEN`: ```bash -fastbz2 "$wiki/data/enwiki-first-1000-streams.xml.bz2" -o "$wiki/data/enwiki-first-1000-streams.xml" +fbz "$wiki/data/enwiki-first-1000-streams.xml.bz2" -o "$wiki/data/enwiki-first-1000-streams.xml" printf '\n' >> "$wiki/data/enwiki-first-1000-streams.xml" xmllint --stream --noout "$wiki/data/enwiki-first-1000-streams.xml" ``` @@ -251,7 +281,7 @@ The thin PEP 517 backend delegates to Maturin after building and staging the nat ## Release 1. Run `cargo build --release --bins && python tools/stage_binaries.py`. -2. Run `maturin develop --release && pytest -q`. +2. Run `uv pip install --reinstall --no-deps -e . && pytest -q` so the custom backend installs both the extension and native CLI. 3. Confirm the release version in `Cargo.toml` (`[package].version`). 4. Run `ship-release`. diff --git a/README.md b/README.md index 0b6c224..a7894b1 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ -# fastbz2 +# fbz -An active compression-format research workbench with fast bzip2, gzip, and ZIP decompression plus safe tar extraction. +**Faster, better zipper:** an active compression-format research workbench with fast bzip2, gzip, LZ4, and ZIP decompression plus safe tar extraction. -`fastbz2` provides a native CLI, a Rust library, and a Python module. The CLI auto-selects bzip2, gzip, or ZIP handling from the filename extension, falling back to stream magic when needed. It streams compressed tar variants through a bounded extractor and adaptively parallelises ZIP work within or across entries. Every path validates decoded sizes and checksums. The bzip2 implementation also provides persistent random-access indexes. +`fbz` provides a native CLI, a Rust library, and a Python module. The CLI auto-selects bzip2, gzip, LZ4, or ZIP handling from the filename extension, falling back to stream magic when needed. It streams compressed tar variants through a bounded extractor and adaptively parallelises ZIP work within or across entries. Every path validates decoded sizes and checksums. The bzip2 implementation also provides persistent random-access indexes. -Compression is planned, but the current implementation decompresses bzip2 and gzip, extracts their tar-wrapped variants, and extracts stored or DEFLATE-compressed ZIP archives. +Compression is planned as a separate whole-surface design. The current implementation only decompresses: bzip2, gzip and LZ4 frames; their tar-wrapped variants; and stored or DEFLATE-compressed ZIP archives. ## Install -PyPI wheels contain both the Python module and the native `fastbz2` executable—there is no Python CLI wrapper: +PyPI wheels contain both the Python module and the native `fbz` executable—there is no Python CLI wrapper: ```bash -pip install fastbz2 +pip install fbz ``` Python 3.10 and later are supported. Prebuilt wheels target Linux on x86-64 and ARM64, and macOS on ARM64. macOS Intel is best-effort and can build from source. @@ -19,43 +19,46 @@ Python 3.10 and later are supported. Prebuilt wheels target Linux on x86-64 and The Rust crate is not yet published separately on crates.io. Install the CLI from the repository, or add the library as a Git dependency: ```bash -cargo install --git https://github.com/AnswerDotAI/fastbz2 -cargo add fastbz2 --git https://github.com/AnswerDotAI/fastbz2 +cargo install --git https://github.com/AnswerDotAI/fbz +cargo add fbz --git https://github.com/AnswerDotAI/fbz ``` ## CLI -Decoding is the default operation. `.bz2`, `.bzip2`, `.gz`, and `.gzip` select their corresponding decoder and are removed from the output name. Compressed tar names—`.tar.bz2`, `.tar.bzip2`, `.tbz`, `.tbz2`, `.tar.gz`, `.tar.gzip`, and `.tgz`—and `.zip` automatically extract into the current directory or `-C/--output-dir`. `-x/--extract` forces archive extraction for stdin or an unusual filename; an explicit `-o/--output` instead writes a decoded tar stream, but is invalid for ZIP because ZIP has no single decoded byte stream. For stdin and unrecognised extensions, bzip2, gzip, or ZIP magic selects the format. Other non-archive input names gain `.out`. +Decoding is the default operation. `.bz2`, `.bzip2`, `.gz`, `.gzip`, and `.lz4` select their corresponding decoder and are removed from the output name. Compressed tar names—`.tar.bz2`, `.tar.bzip2`, `.tbz`, `.tbz2`, `.tar.gz`, `.tar.gzip`, `.tgz`, and `.tar.lz4`—and `.zip` automatically extract into the current directory or `-C/--output-dir`. `-x/--extract` forces archive extraction for stdin or an unusual filename; an explicit `-o/--output` instead writes a decoded tar stream, but is invalid for ZIP because ZIP has no single decoded byte stream. For stdin and unrecognised extensions, bzip2, gzip, LZ4, or ZIP magic selects the format. Other non-archive input names gain `.out`. ```bash -fastbz2 dump.xml.bz2 # write dump.xml -fastbz2 events.json.gz # write events.json -fastbz2 source.tar.gz # extract into the current directory -fastbz2 source.tbz2 -C unpacked # extract into unpacked/ -fastbz2 dataset.zip -C unpacked # extract ZIP entries adaptively in parallel -fastbz2 --extract -C unpacked - # extract tar or ZIP data from stdin -fastbz2 source.tgz -o source.tar # decode without extracting -fastbz2 dump.xml.bz2 -o result.xml # choose the decoded output path -fastbz2 dump.xml.bz2 -o - # write decoded bytes to stdout +fbz dump.xml.bz2 # write dump.xml +fbz events.json.gz # write events.json +fbz events.json.lz4 # write events.json +fbz source.tar.gz # extract into the current directory +fbz source.tar.lz4 -C unpacked # stream-decode and extract +fbz source.tbz2 -C unpacked # extract into unpacked/ +fbz dataset.zip -C unpacked # extract ZIP entries adaptively in parallel +fbz --extract -C unpacked - # extract tar or ZIP data from stdin +fbz source.tgz -o source.tar # decode without extracting +fbz dump.xml.bz2 -o result.xml # choose the decoded output path +fbz dump.xml.bz2 -o - # write decoded bytes to stdout ``` Multiple inputs are processed in order, with parallelism applied inside each compressed stream. `-C/--output-dir` collects decoded files and is the extraction root for archives: ```bash -fastbz2 data/*.bz2 logs/*.gz -C decoded -fastbz2 data/*.bz2 logs/*.gz -C decoded --skip-existing -fastbz2 backups/*.tgz -C restored -fastbz2 datasets/*.zip -C restored +fbz data/*.bz2 logs/*.gz -C decoded +fbz data/*.bz2 logs/*.gz -C decoded --skip-existing +fbz backups/*.tgz -C restored +fbz datasets/*.zip -C restored ``` Validation and inspection remain flags rather than subcommands: ```bash -fastbz2 --test dump.xml.bz2 # fully decode and validate, writing nothing -fastbz2 --index dump.xml.bz2 # write dump.xml.bz2.fbz2i (bzip2 only) -fastbz2 --list events.json.gz # print the validated member/block layout -fastbz2 --list dataset.zip # print the validated entry layout -fastbz2 --list --json dump.xml.bz2 # emit the complete layout as JSON +fbz --test dump.xml.bz2 # fully decode and validate, writing nothing +fbz --index dump.xml.bz2 # write dump.xml.bz2.fbz2i (bzip2 only) +fbz --list events.json.gz # print the validated member/block layout +fbz --list events.json.lz4 # print the validated frame/block layout +fbz --list dataset.zip # print the validated entry layout +fbz --list --json dump.xml.bz2 # emit the complete layout as JSON ``` `--test`, `--index`, `--list`, and explicit `--extract` are mutually exclusive. Human-readable `--list` output labels each input when given multiple files; JSON output is one object for one input and an array for multiple inputs. @@ -71,7 +74,7 @@ fastbz2 --list --json dump.xml.bz2 # emit the complete layout as JSON Long interactive operations report completion, decoded throughput, compression ratio, and ETA on stderr. Progress is disabled automatically when stderr is redirected; `-q/--quiet` also suppresses progress and skip notices. -`-P/--threads 0`, the default, uses the machine's available parallelism; an explicit positive value is honoured by every decoder. `--memory-limit` bounds speculative output in the shared scheduler and defaults to `1G`. Gzip uses parallel dynamic-block discovery only when the input and memory budget can amortize it. ZIP uses one level of parallelism at a time: large entries use the parallel DEFLATE engine, while archives of ordinary entries decode files concurrently without nested worker pools. +`-P/--threads 0`, the default, uses the machine's available parallelism; an explicit positive value is honoured by every decoder. `--memory-limit` bounds speculative output in the shared scheduler and defaults to `1G`. Gzip uses parallel dynamic-block discovery only when the input and memory budget can amortize it. LZ4 frames with independent blocks decode those blocks concurrently and commit them in order; automatic LZ4 decoding caps this memory-bandwidth-bound work at four workers, while explicit `-P` values remain unchanged. Linked-block frames decode serially because each block depends on the preceding 64 KiB history. ZIP uses one level of parallelism at a time: large entries use the parallel DEFLATE engine, while archives of ordinary entries decode files concurrently without nested worker pools. ## Python @@ -80,24 +83,24 @@ The Python API currently exposes the bzip2 backend. Unified Python dispatch will ### One-shot decompression and validation ```python -import fastbz2 +import fbz -plain = fastbz2.decompress(compressed_bytes) -fastbz2.test("dump.xml.bz2") # returns None after successful validation +plain = fbz.decompress(compressed_bytes) +fbz.test("dump.xml.bz2") # returns None after successful validation ``` `decompress` accepts a bytes-like object and returns `bytes`. `test` accepts either compressed bytes or a path and avoids retaining the decoded result. ### Seekable reads and persistent indexes -`fastbz2.open` returns a seekable binary `io.RawIOBase`. Opening without an index performs a complete validation pass and builds an in-memory block index; `build_index` can persist that work for later processes: +`fbz.open` returns a seekable binary `io.RawIOBase`. Opening without an index performs a complete validation pass and builds an in-memory block index; `build_index` can persist that work for later processes: ```python -import fastbz2 +import fbz -fastbz2.build_index("dump.xml.bz2", "dump.xml.bz2.fbz2i") +fbz.build_index("dump.xml.bz2", "dump.xml.bz2.fbz2i") -with fastbz2.open("dump.xml.bz2", index="dump.xml.bz2.fbz2i") as f: +with fbz.open("dump.xml.bz2", index="dump.xml.bz2.fbz2i") as f: f.seek(1_000_000_000) chunk = f.read(64 * 1024) print(f.tell(), f.size) @@ -111,7 +114,7 @@ Building an index fully decodes into a sink but does not write or retain the pla ```python import bz2 -from fastbz2 import scan +from fbz import scan result = scan(bz2.compress(b"hello")) assert result.blocks[0].bit_offset == 32 @@ -124,9 +127,9 @@ Scan results are deliberately untrusted candidates. Use `test`, `decompress`, `b The streaming API accepts any `Write` destination and uses the serial fast path when `threads` is one: ```rust -use fastbz2::{DecodeOptions, Source, decompress_to_writer}; +use fbz::{DecodeOptions, Source, decompress_to_writer}; -fn main() -> fastbz2::Result<()> { +fn main() -> fbz::Result<()> { let source = Source::open("dump.xml.bz2")?; let mut output = std::io::stdout().lock(); decompress_to_writer(source.as_slice(), &mut output, DecodeOptions::default())?; @@ -139,35 +142,59 @@ fn main() -> fastbz2::Result<()> { The in-repo gzip decoder is available separately so callers can choose explicitly: ```rust -let plain = fastbz2::gzip::decompress(&compressed_gzip)?; +let plain = fbz::gzip::decompress(&compressed_gzip)?; ``` `gzip::decompress_to_writer` and `gzip::decompress_to_writer_with_options` return a validated report containing gzip member metadata, each DEFLATE block's kind and ranges, and counts of accepted speculative and serial-fallback chunks. They support stored, fixed-Huffman, and dynamic-Huffman blocks, optional gzip headers, and concatenated members. -The raw shared codec is available as `fastbz2::deflate::decompress_to_sink_with_options_and_progress`; gzip framing and ZIP extraction both use this exact decoder. +The raw shared codec is available as `fbz::deflate::decompress_to_sink_with_options_and_progress`; gzip framing and ZIP extraction both use this exact decoder. + +The in-repo LZ4 frame decoder has the same one-shot, writer, options, progress, and report shapes as gzip: + +```rust +let plain = fbz::lz4::decompress(&compressed_lz4)?; +``` + +It accepts standard independent or linked blocks, stored blocks, all four standard block maxima, optional block/content checksums and sizes, concatenated frames, and skippable frames. External dictionaries and the obsolete legacy frame format are intentionally unsupported. ## Performance -These are single local release-mode CLI runs on the primary 18-core Apple Silicon development machine. The gzip comparison warms each executable with the 5% fixture, then measures exactly one full validation run; peak physical footprint comes from a separate sampled run because process inspection can perturb such a short workload. ZIP likewise warms each CLI once and then measures one extraction. These are observations rather than statistical aggregates. In-process codec and library-oracle comparisons live in [DEV.md](DEV.md#local-wikipedia-benchmarks), not this user-facing table. +These are single local release-mode CLI runs on the primary Apple Silicon development machine. The gzip comparison warms each executable with the 5% fixture, then measures exactly one full validation run; peak physical footprint comes from a separate sampled run because process inspection can perturb such a short workload. ZIP and tar likewise warm each CLI once and then measure one extraction. The LZ4 comparison uses its automatic four-worker limit. These are observations rather than statistical aggregates. In-process codec and library-oracle comparisons live in [DEV.md](DEV.md), not this user-facing section. Full SimpleWiki recompressed with system `gzip -6` (`438,904,466` bytes compressed, `1,688,460,257` bytes decoded): | CLI | Mode | Seconds | Peak physical footprint | |---|---|---:|---:| | rapidgzip-rust, local checkout | auto parallel, `--test` | 0.363 | 460.0 MiB | -| fastbz2 | auto parallel, `--test` | 0.326 | 335.9 MiB | +| fbz | auto parallel, `--test` | 0.326 | 335.9 MiB | | Apple gzip | serial, stdout discarded | 1.371 | 1.2 MiB | -The memory values use macOS physical footprint rather than `ru_maxrss`. The fastbz2 CLI memory-maps its 419 MiB input, so clean reclaimable file pages make RSS look roughly 419 MiB larger; `pread`-based tools leave the same cached pages outside process RSS. Physical footprint makes the comparison meaningful. +The memory values use macOS physical footprint rather than `ru_maxrss`. The fbz CLI memory-maps its 419 MiB input, so clean reclaimable file pages make RSS look roughly 419 MiB larger; `pread`-based tools leave the same cached pages outside process RSS. Physical footprint makes the comparison meaningful. ZIPs containing the same 80.5 MiB SimpleWiki prefix (`25.8 MiB` compressed), after one untimed warm-up per executable: -| Shape | fastbz2, auto parallel | Info-ZIP unzip 6.00 (Apple) | Speedup | +| Shape | fbz, auto parallel | Info-ZIP unzip 6.00 (Apple) | Speedup | |---|---:|---:|---:| | One DEFLATE entry | 35.571 ms | 355.235 ms | 10.0x | | 18 DEFLATE entries | 29.700 ms | 360.008 ms | 12.1x | -The many-entry sampled run used 36.7 MiB physical footprint for fastbz2 versus 2.4 MiB for `unzip`; its 18-way file parallelism deliberately spends modest memory to obtain the throughput above. Both fastbz2 rows used automatic thread selection, which resolved to 18 available cores on this machine. +The many-entry sampled run used 36.7 MiB physical footprint for fbz versus 2.4 MiB for `unzip`; its 18-way file parallelism deliberately spends modest memory to obtain the throughput above. Both fbz rows used automatic thread selection, which resolved to 18 available cores on this machine. + +Compressed tar archives containing the same 80.5 MiB prefix, after one untimed warm-up per CLI: + +| Format | fbz extraction | System `tar` | Speedup | +|---|---:|---:|---:| +| `.tgz` | 56.850 ms | 117.962 ms | 2.1x | +| `.tar.bz2` | 151.783 ms | 1.168 s | 7.7x | + +The same 80.5 MiB prefix in a standard independent-block LZ4 frame (`40.2 MiB` compressed), after one untimed warm-up per executable: + +| CLI | Milliseconds | Peak RSS | fbz/reference | +|---|---:|---:|---:| +| fbz, auto (4 workers) | 51.188 | 71.0 MiB | 0.944x | +| Homebrew `lz4` 1.10.0 | 54.228 | 32.0 MiB | — | + +The larger fbz RSS includes its memory-mapped 40.2 MiB source plus bounded in-flight decoded blocks; it does not grow with decoded file size. Testing higher worker counts showed no meaningful throughput gain and raised RSS, so automatic LZ4 decoding stops at four workers; `-P` remains an explicit override. Homebrew `pbzip2` 1.1.13 could not safely decompress the complete 26,668,484,995-byte English Wikipedia multistream dump on this machine. It segfaulted, and repeated attempts produced divergent and truncated plaintext. Successful smaller-file results therefore do not establish full-file reliability. @@ -175,25 +202,28 @@ Homebrew `pbzip2` 1.1.13 could not safely decompress the complete 26,668,484,995 The production codec logic is portable Rust. The bzip2 decoder uses a tuned 4096-entry Huffman lookup table for codes up to 12 bits and canonical fallback for longer codes. A structural scan finds possible non-byte-aligned block markers; these remain speculative until ordered decoding establishes the exact stream chain and validates all block and combined-stream CRCs. A rolling scheduler keeps workers busy across concatenated streams while bounding decoded results awaiting validation. -The gzip backend implements RFC 1952 framing and DEFLATE directly in this repository. For sufficiently large dynamic-Huffman inputs it discovers independently decodable boundaries, decodes speculative chunks through the shared byte-budgeted scheduler, and represents unknown predecessor bytes as compact markers. The ordered coordinator resolves only the suffix needed to derive the next 32 KiB history window; full marker resolution and per-chunk CRC run as priority work on the same staged worker queue, and CRCs are combined in order. Once a chunk has a marker-free window, the same decoder switches its remaining output from `u16` markers to ordinary bytes. Small, stored-heavy, fixed-heavy, one-thread, and low-memory inputs use the serial path; concatenated members may independently choose either path. FHCRC, CRC32, and ISIZE are always validated. ZIP reuses that raw DEFLATE core and uses the mature `zip` crate only for container structure and metadata. It supports stored and DEFLATE entries, Zip64, streaming data descriptors, Unix symlinks/modes, and Unix/NTFS modification-time fields; encryption and uncommon legacy compression methods are intentionally unsupported. `crc32fast` is the only production codec helper; `flate2` is dev-only. +The gzip backend implements RFC 1952 framing and DEFLATE directly in this repository. For sufficiently large dynamic-Huffman inputs it discovers independently decodable boundaries, decodes speculative chunks through the shared byte-budgeted scheduler, and represents unknown predecessor bytes as compact markers. The ordered coordinator resolves only the suffix needed to derive the next 32 KiB history window; full marker resolution and per-chunk CRC run as priority work on the same staged worker queue, and CRCs are combined in order. Once a chunk has a marker-free window, the same decoder switches its remaining output from `u16` markers to ordinary bytes. Small, stored-heavy, fixed-heavy, one-thread, and low-memory inputs use the serial path; concatenated members may independently choose either path. FHCRC, CRC32, and ISIZE are always validated. LZ4 framing and block decoding are likewise implemented in safe Rust. Independent blocks use the same ordered, byte-budgeted scheduler as bzip2; linked blocks retain only the preceding 64 KiB window. LZ4 and DEFLATE share one optimized overlapping back-reference expansion primitive. Header, block, and content XXH32 checksums are validated where present. ZIP reuses the raw DEFLATE core and uses the mature `zip` crate only for container structure and metadata. It supports stored and DEFLATE entries, Zip64, streaming data descriptors, Unix symlinks/modes, and Unix/NTFS modification-time fields; encryption and uncommon legacy compression methods are intentionally unsupported. `crc32fast` and `twox-hash` are the production checksum helpers; `flate2` and `lz4_flex` are dev-only differential oracles. Legacy randomized blocks generated by bzip2 releases before 0.9.5 are intentionally unsupported. Normal `BZh1` through `BZh9` streams and concatenated streams are supported. ## Research lineage and credits -The gzip work builds on Maximilian Knespel and Holger Brunst's HPDC '23 paper, [*Rapidgzip: Parallel Decompression and Seeking in Gzip Files Using Cache Prefetching*](https://doi.org/10.1145/3588195.3592992). In particular, fastbz2 adapts its central idea of starting DEFLATE decoding without the preceding 32 KiB window, representing uncertain output until the true history becomes available, and committing independently decoded chunks in order. +The gzip work builds on Maximilian Knespel and Holger Brunst's HPDC '23 paper, [*Rapidgzip: Parallel Decompression and Seeking in Gzip Files Using Cache Prefetching*](https://doi.org/10.1145/3588195.3592992). In particular, fbz adapts its central idea of starting DEFLATE decoding without the preceding 32 KiB window, representing uncertain output until the true history becomes available, and committing independently decoded chunks in order. The open-source implementations and codebases consulted were: - [`rapidgzip`](https://github.com/mxmlnkn/rapidgzip), the C++ implementation described by the paper. -- [`rapidgzip-rust`](https://github.com/COMBINE-lab/rapidgzip-rust), a pure-Rust reimplementation and fastbz2's local gzip performance and memory reference. +- [`rapidgzip-rust`](https://github.com/COMBINE-lab/rapidgzip-rust), a pure-Rust reimplementation and fbz's local gzip performance and memory reference. - [`librapidarchive`](https://github.com/mxmlnkn/librapidarchive), an experimental shared architecture for parallel bzip2 and gzip access. - [`indexed_bzip2`](https://github.com/mxmlnkn/indexed_bzip2), for non-byte-aligned marker scanning, independent bzip2 block decoding, ordered prefetch, and indexed seeking. - [`zip`](https://github.com/zip-rs/zip2), used without codec features for maintained ZIP structure and metadata handling. -- Rob Landley's 0BSD [`bzcat` implementation in Toybox](https://github.com/landley/toybox), from which fastbz2's specialised bzip2 decoder is derived. +- [`LZ4`](https://github.com/lz4/lz4), the reference format and Homebrew CLI performance baseline. +- [`lz4_flex`](https://github.com/pseitz/lz4_flex), used dev-only to generate a broad interoperability matrix and benchmark frames. +- [`lz4-rs`](https://github.com/bozaro/lz4-rs), consulted as a second local implementation reference. +- Rob Landley's 0BSD [`bzcat` implementation in Toybox](https://github.com/landley/toybox), from which fbz's specialised bzip2 decoder is derived. ## Development [DEV.md](DEV.md) documents the architecture, test strategy, benchmark fixture generation, build commands, and release process. -`fastbz2` is licensed under the [Apache License 2.0](LICENSE). +`fbz` is licensed under the [Apache License 2.0](LICENSE). diff --git a/build_backend.py b/build_backend.py index b09e002..49d313a 100644 --- a/build_backend.py +++ b/build_backend.py @@ -4,7 +4,7 @@ import maturin def _stage_cli(): - subprocess.run(["cargo", "build", "--release", "--bin", "fastbz2"], check=True) + subprocess.run(["cargo", "build", "--release", "--bin", "fbz"], check=True) subprocess.run([sys.executable, "tools/stage_binaries.py"], check=True) def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): diff --git a/pyproject.toml b/pyproject.toml index af71fe0..0f03612 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,9 @@ build-backend = "build_backend" backend-path = ["."] [project] -name = "fastbz2" +name = "fbz" dynamic = ["version"] -description = "Compression-format research workbench with fast bzip2 and gzip decompression" +description = "Compression-format research workbench with fast bzip2, gzip, LZ4, and ZIP decompression" license = {text = "Apache-2.0"} requires-python = ">=3.10" readme = "README.md" @@ -20,14 +20,14 @@ classifiers = [ dev = ["fastship>=0.0.11", "maturin>=1.0,<2.0", "pytest"] [project.urls] -Homepage = "https://github.com/AnswerDotAI/fastbz2" -Repository = "https://github.com/AnswerDotAI/fastbz2" -Issues = "https://github.com/AnswerDotAI/fastbz2/issues" +Homepage = "https://github.com/AnswerDotAI/fbz" +Repository = "https://github.com/AnswerDotAI/fbz" +Issues = "https://github.com/AnswerDotAI/fbz/issues" [tool.maturin] features = ["extension-module"] python-source = "python" -module-name = "fastbz2._core" +module-name = "fbz._core" data = "target/wheel-data" [tool.uv] diff --git a/python/fastbz2/__init__.py b/python/fbz/__init__.py similarity index 100% rename from python/fastbz2/__init__.py rename to python/fbz/__init__.py diff --git a/src/bin/fastbz2.rs b/src/bin/fbz.rs similarity index 81% rename from src/bin/fastbz2.rs rename to src/bin/fbz.rs index e8eb9c5..25d147d 100644 --- a/src/bin/fastbz2.rs +++ b/src/bin/fbz.rs @@ -1,8 +1,8 @@ -#[path = "fastbz2/archive_extract.rs"] +#[path = "fbz/archive_extract.rs"] mod archive_extract; -#[path = "fastbz2/tar_extract.rs"] +#[path = "fbz/tar_extract.rs"] mod tar_extract; -#[path = "fastbz2/zip_extract.rs"] +#[path = "fbz/zip_extract.rs"] mod zip_extract; use std::{ @@ -14,9 +14,9 @@ use std::{ }; use clap::{ArgGroup, Parser}; -use fastbz2::{ +use fbz::{ DecodeOptions, DecodeProgress, Error, Index, OutputSink, Source, WriterSink, build_index_with_progress, decode_to_writer_with_progress, - decompress_to_sink_with_progress, gzip, + decompress_to_sink_with_progress, gzip, lz4, }; use serde_json::{Value, json}; use tempfile::NamedTempFile; @@ -24,15 +24,17 @@ use tempfile::NamedTempFile; #[derive(Parser)] #[command( version, - about = "Parallel bzip2/gzip/ZIP decompression and safe archive extraction", - long_about = "Parallel bzip2 and gzip decompression, streaming tar extraction, and adaptive parallel ZIP extraction. Decoding is the default operation. Recognised codec suffixes are removed from normal output names. ZIP and compressed tar archives extract automatically unless -o is given; -o is not valid for ZIP.", + about = "Parallel bzip2, gzip, LZ4, and ZIP decompression with safe archive extraction", + long_about = "Parallel bzip2, gzip, and LZ4 frame decompression, streaming tar extraction, and adaptive parallel ZIP extraction. Decoding is the default operation. Recognised codec suffixes are removed from normal output names. ZIP and compressed tar archives extract automatically unless -o is given; -o is not valid for ZIP. Independent LZ4 blocks decode in parallel; linked blocks decode serially.", after_help = r#"Examples: - fastbz2 dump.xml.bz2 Write dump.xml - fastbz2 events.json.gz -o - Write decoded bytes to stdout - fastbz2 backup.tgz -C restored Extract into restored/ - fastbz2 dataset.zip -C restored Extract ZIP entries in parallel - fastbz2 --test archive.tar.bz2 Validate without writing output - fastbz2 --list --json data.gz Show the validated layout as JSON"#, + fbz dump.xml.bz2 Write dump.xml + fbz events.json.gz -o - Write decoded bytes to stdout + fbz events.json.lz4 Write events.json + fbz backup.tgz -C restored Extract into restored/ + fbz backup.tar.lz4 -C restored Extract a tar-wrapped LZ4 frame + fbz dataset.zip -C restored Extract ZIP entries in parallel + fbz --test archive.tar.bz2 Validate without writing output + fbz --list --json data.gz Show the validated layout as JSON"#, group(ArgGroup::new("mode").args(["test", "index", "list", "extract"])) )] struct Cli { @@ -45,7 +47,7 @@ struct Cli { /// Build validated, source-bound .fbz2i indexes for bzip2 inputs. #[arg(long)] index: bool, - /// Validate and show bzip2 streams/blocks, gzip members/blocks, or ZIP entries. + /// Validate and show bzip2 streams/blocks, gzip members/blocks, LZ4 frames/blocks, or ZIP entries. #[arg(long)] list: bool, /// Extract a tar or ZIP archive; automatic for recognised archive suffixes. @@ -87,6 +89,7 @@ struct Cli { enum Format { Bzip2, Gzip, + Lz4, Zip, } @@ -94,13 +97,13 @@ fn main() -> ExitCode { match run(Cli::parse()) { Ok(()) => ExitCode::SUCCESS, Err(error) => { - eprintln!("fastbz2: {error}"); + eprintln!("fbz: {error}"); ExitCode::from(exit_status(&error)) } } } -fn run(cli: Cli) -> fastbz2::Result<()> { +fn run(cli: Cli) -> fbz::Result<()> { validate_cli(&cli)?; let options = DecodeOptions { threads: cli.threads, memory_limit: cli.memory_limit }; if cli.test { @@ -118,7 +121,7 @@ fn run(cli: Cli) -> fastbz2::Result<()> { run_decode(&cli, options) } -fn validate_cli(cli: &Cli) -> fastbz2::Result<()> { +fn validate_cli(cli: &Cli) -> fbz::Result<()> { if cli.output.is_some() && cli.inputs.len() != 1 { return Err(invalid("--output requires exactly one input")); } @@ -141,7 +144,7 @@ fn should_extract(cli: &Cli, input: &str) -> bool { cli.extract || (cli.output.is_none() && is_archive(input)) } -fn run_decode(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { +fn run_decode(cli: &Cli, options: DecodeOptions) -> fbz::Result<()> { if let Some(directory) = &cli.output_dir { fs::create_dir_all(directory)?; } @@ -180,7 +183,7 @@ fn run_decode(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { Ok(()) } -fn run_index(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { +fn run_index(cli: &Cli, options: DecodeOptions) -> fbz::Result<()> { for input in &cli.inputs { let input_path = Path::new(input); let output = cli.output.clone().unwrap_or_else(|| PathBuf::from(format!("{}.fbz2i", input_path.display()))); @@ -202,7 +205,7 @@ fn run_index(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { Ok(()) } -fn run_list(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { +fn run_list(cli: &Cli, options: DecodeOptions) -> fbz::Result<()> { let mut values = Vec::new(); for input in &cli.inputs { let source = Source::open(input)?; @@ -223,6 +226,14 @@ fn run_list(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { print_gzip_report((cli.inputs.len() > 1).then_some(input), &report); } } + Format::Lz4 => { + let report = build_lz4_report_data(source.as_slice(), input, options, cli.max_output, cli.quiet)?; + if cli.json { + values.push(lz4_json(input, &report)); + } else { + print_lz4_report((cli.inputs.len() > 1).then_some(input), &report); + } + } Format::Zip => { let mut display = ProgressDisplay::new(input, source.as_slice().len() as u64, cli.quiet); let report = zip_extract::validate(source.as_slice(), options, cli.max_output, |progress| display.update(progress))?; @@ -242,7 +253,7 @@ fn run_list(cli: &Cli, options: DecodeOptions) -> fastbz2::Result<()> { Ok(()) } -fn extract_input(input: &str, destination: &Path, overwrite: bool, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result<()> { +fn extract_input(input: &str, destination: &Path, overwrite: bool, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result<()> { if input == "-" { let mut data = Vec::new(); io::stdin().lock().read_to_end(&mut data)?; @@ -261,7 +272,7 @@ fn extract_data( options: DecodeOptions, max_output: Option, quiet: bool, -) -> fastbz2::Result<()> { +) -> fbz::Result<()> { if select_format(label, data)? == Format::Zip { let mut display = ProgressDisplay::new(label, data.len() as u64, quiet); zip_extract::unpack(data, destination, overwrite, options, max_output, |progress| display.update(progress)).map(|_| ()) @@ -270,7 +281,7 @@ fn extract_data( } } -fn test_input(input: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result<()> { +fn test_input(input: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result<()> { if input == "-" { let mut data = Vec::new(); io::stdin().lock().read_to_end(&mut data)?; @@ -280,7 +291,7 @@ fn test_input(input: &str, options: DecodeOptions, max_output: Option, qu test_data(source.as_slice(), input, options, max_output, quiet) } -fn test_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result<()> { +fn test_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result<()> { if select_format(label, data)? == Format::Zip { let mut display = ProgressDisplay::new(label, data.len() as u64, quiet); zip_extract::validate(data, options, max_output, |progress| display.update(progress)).map(|_| ()) @@ -289,7 +300,7 @@ fn test_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Optio } } -fn decode_input(input: &str, output: &mut impl Write, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result<()> { +fn decode_input(input: &str, output: &mut impl Write, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result<()> { if input == "-" { let mut data = Vec::new(); io::stdin().lock().read_to_end(&mut data)?; @@ -300,7 +311,7 @@ fn decode_input(input: &str, output: &mut impl Write, options: DecodeOptions, ma } } -fn decode_data(data: &[u8], label: &str, output: &mut impl Write, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result<()> { +fn decode_data(data: &[u8], label: &str, output: &mut impl Write, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result<()> { let mut output = WriterSink::new(output); decode_data_to_sink(data, label, &mut output, options, max_output, quiet) } @@ -312,23 +323,30 @@ fn decode_data_to_sink( options: DecodeOptions, max_output: Option, quiet: bool, -) -> fastbz2::Result<()> { +) -> fbz::Result<()> { let mut output = LimitedOutput::new(output, max_output); let mut display = ProgressDisplay::new(label, data.len() as u64, quiet); match select_format(label, data)? { Format::Bzip2 => decompress_to_sink_with_progress(data, &mut output, options, |progress| display.update(progress)), Format::Gzip => gzip::decompress_to_sink_with_options_and_progress(data, &mut output, options, |progress| display.update(progress)).map(|_| ()), + Format::Lz4 => lz4::decompress_to_sink_with_options_and_progress(data, &mut output, options, |progress| display.update(progress)).map(|_| ()), Format::Zip => Err(invalid("ZIP archives extract to a directory and cannot be decoded to one output stream")), } } -fn build_gzip_report_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result { +fn build_gzip_report_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result { let mut sink = LimitedOutput::new(io::sink(), max_output); let mut display = ProgressDisplay::new(label, data.len() as u64, quiet); gzip::decompress_to_writer_with_options_and_progress(data, &mut sink, options, |progress| display.update(progress)) } -fn build_index_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fastbz2::Result { +fn build_lz4_report_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result { + let mut sink = LimitedOutput::new(io::sink(), max_output); + let mut display = ProgressDisplay::new(label, data.len() as u64, quiet); + lz4::decompress_to_writer_with_options_and_progress(data, &mut sink, options, |progress| display.update(progress)) +} + +fn build_index_data(data: &[u8], label: &str, options: DecodeOptions, max_output: Option, quiet: bool) -> fbz::Result { let mut display = ProgressDisplay::new(label, data.len() as u64, quiet); if let Some(limit) = max_output { let mut sink = LimitedOutput::new(io::sink(), Some(limit)); @@ -456,12 +474,12 @@ fn should_skip(path: &Path, skip_existing: bool, quiet: bool) -> bool { return false; } if !quiet { - eprintln!("fastbz2: skipping existing {}", path.display()); + eprintln!("fbz: skipping existing {}", path.display()); } true } -fn preserve_metadata(input: &Path, output: &Path) -> fastbz2::Result<()> { +fn preserve_metadata(input: &Path, output: &Path) -> fbz::Result<()> { let metadata = fs::metadata(input)?; if let Ok(modified) = metadata.modified() { fs::OpenOptions::new().write(true).open(output)?.set_times(fs::FileTimes::new().set_modified(modified))?; @@ -470,7 +488,7 @@ fn preserve_metadata(input: &Path, output: &Path) -> fastbz2::Result<()> { Ok(()) } -fn atomic_write(path: &Path, force: bool, write: impl FnOnce(&mut fs::File) -> fastbz2::Result<()>) -> fastbz2::Result<()> { +fn atomic_write(path: &Path, force: bool, write: impl FnOnce(&mut fs::File) -> fbz::Result<()>) -> fbz::Result<()> { if path.exists() && !force { return Err(Error::Io(io::Error::new(io::ErrorKind::AlreadyExists, format!("{} already exists (use --force)", path.display())))); } @@ -498,6 +516,7 @@ fn format_extension(input: &Path) -> Option<(Format, &'static str)> { "tbz" | "tbz2" => Some((Format::Bzip2, "tar")), "gz" | "gzip" => Some((Format::Gzip, "")), "tgz" => Some((Format::Gzip, "tar")), + "lz4" => Some((Format::Lz4, "")), "zip" => Some((Format::Zip, "")), _ => None, } @@ -505,7 +524,7 @@ fn format_extension(input: &Path) -> Option<(Format, &'static str)> { fn is_tar_archive(input: &str) -> bool { let input = input.to_ascii_lowercase(); - [".tar.bz2", ".tar.bzip2", ".tbz", ".tbz2", ".tar.gz", ".tar.gzip", ".tgz"].iter().any(|extension| input.ends_with(extension)) + [".tar.bz2", ".tar.bzip2", ".tbz", ".tbz2", ".tar.gz", ".tar.gzip", ".tgz", ".tar.lz4"].iter().any(|extension| input.ends_with(extension)) } fn is_zip_archive(input: &str) -> bool { @@ -520,7 +539,7 @@ fn default_output(input: &Path) -> PathBuf { format_extension(input).map_or_else(|| PathBuf::from(format!("{}.out", input.display())), |(_, extension)| input.with_extension(extension)) } -fn select_format(input: &str, data: &[u8]) -> fastbz2::Result { +fn select_format(input: &str, data: &[u8]) -> fbz::Result { if let Some((format, _)) = format_extension(Path::new(input)) { return Ok(format); } @@ -528,10 +547,14 @@ fn select_format(input: &str, data: &[u8]) -> fastbz2::Result { Ok(Format::Bzip2) } else if data.starts_with(&[0x1f, 0x8b]) { Ok(Format::Gzip) + } else if data.starts_with(&[0x04, 0x22, 0x4d, 0x18]) + || data.get(..4).is_some_and(|magic| (0x184d_2a50..=0x184d_2a5f).contains(&u32::from_le_bytes(magic.try_into().unwrap()))) + { + Ok(Format::Lz4) } else if data.starts_with(b"PK\x03\x04") || data.starts_with(b"PK\x05\x06") || data.starts_with(b"PK\x07\x08") { Ok(Format::Zip) } else { - Err(invalid(format!("cannot determine compression format for {input}; expected a bzip2, gzip, or ZIP extension or magic"))) + Err(invalid(format!("cannot determine compression format for {input}; expected a bzip2, gzip, LZ4, or ZIP extension or magic"))) } } @@ -574,6 +597,27 @@ fn print_gzip_report(input: Option<&String>, report: &gzip::Report) { } } +fn print_lz4_report(input: Option<&String>, report: &lz4::Report) { + if let Some(input) = input { + println!("input\t{input}"); + } + println!("format\tlz4"); + println!("compressed_bytes\t{}", report.source_len); + println!("decoded_bytes\t{}", report.decoded_len); + println!("frames\t{}", report.frames.len()); + println!("blocks\t{}", report.blocks.len()); + for (number, frame) in report.frames.iter().enumerate() { + let mode = match frame.block_mode { + lz4::BlockMode::Independent => "independent", + lz4::BlockMode::Linked => "linked", + }; + println!( + "frame\t{number}\tmode={mode}\tblock_max={}\tblocks={}\tdecoded={}\tblock_checksums={}\tcontent_checksum={}", + frame.block_max_size, frame.block_count, frame.decoded_len, frame.block_checksums, frame.content_checksum, + ); + } +} + fn print_zip_report(input: Option<&String>, report: &zip_extract::Report) { if let Some(input) = input { println!("input\t{input}"); @@ -663,6 +707,38 @@ fn gzip_json(input: &str, report: &gzip::Report) -> Value { }) } +fn lz4_json(input: &str, report: &lz4::Report) -> Value { + json!({ + "input": input, + "format": "lz4", + "source_bytes": report.source_len, + "decoded_bytes": report.decoded_len, + "frames": report.frames.iter().enumerate().map(|(number, frame)| json!({ + "number": number, + "compressed_start": frame.compressed_start, + "compressed_end": frame.compressed_end, + "decoded_start": frame.decoded_start, + "decoded_bytes": frame.decoded_len, + "block_max_size": frame.block_max_size, + "block_mode": match frame.block_mode { lz4::BlockMode::Independent => "independent", lz4::BlockMode::Linked => "linked" }, + "block_checksums": frame.block_checksums, + "content_checksum": frame.content_checksum, + "declared_content_size": frame.declared_content_size, + "first_block": frame.first_block, + "block_count": frame.block_count, + })).collect::>(), + "blocks": report.blocks.iter().enumerate().map(|(number, block)| json!({ + "number": number, + "frame": block.frame, + "compressed_start": block.compressed_start, + "compressed_end": block.compressed_end, + "decoded_start": block.decoded_start, + "decoded_bytes": block.decoded_len, + "stored": block.stored, + })).collect::>(), + }) +} + fn zip_json(input: &str, report: &zip_extract::Report) -> Value { json!({ "input": input, @@ -729,7 +805,7 @@ fn exit_status(error: &Error) -> u8 { Error::Io(source) if source.kind() == io::ErrorKind::InvalidData => 3, Error::Io(_) => 1, Error::InvalidConfiguration(_) => 2, - Error::InvalidStreamHeader | Error::InvalidGzip(_) | Error::InvalidZip(_) | Error::Decode { .. } | Error::InvalidIndex(_) => 3, + Error::InvalidStreamHeader | Error::InvalidGzip(_) | Error::InvalidLz4(_) | Error::InvalidZip(_) | Error::Decode { .. } | Error::InvalidIndex(_) => 3, _ => 4, } } diff --git a/src/bin/fastbz2/archive_extract.rs b/src/bin/fbz/archive_extract.rs similarity index 99% rename from src/bin/fastbz2/archive_extract.rs rename to src/bin/fbz/archive_extract.rs index 39d278c..457cf41 100644 --- a/src/bin/fastbz2/archive_extract.rs +++ b/src/bin/fbz/archive_extract.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use fastbz2::{Error, Result}; +use fbz::{Error, Result}; use tempfile::TempDir; fn path_metadata(path: &Path) -> io::Result> { diff --git a/src/bin/fastbz2/tar_extract.rs b/src/bin/fbz/tar_extract.rs similarity index 98% rename from src/bin/fastbz2/tar_extract.rs rename to src/bin/fbz/tar_extract.rs index d922179..fc603fb 100644 --- a/src/bin/fastbz2/tar_extract.rs +++ b/src/bin/fbz/tar_extract.rs @@ -6,7 +6,7 @@ use std::{ thread, }; -use fastbz2::{Error, OutputSink, Result}; +use fbz::{Error, OutputSink, Result}; use super::archive_extract; diff --git a/src/bin/fastbz2/zip_extract.rs b/src/bin/fbz/zip_extract.rs similarity index 99% rename from src/bin/fastbz2/zip_extract.rs rename to src/bin/fbz/zip_extract.rs index 31d1183..4b796cf 100644 --- a/src/bin/fastbz2/zip_extract.rs +++ b/src/bin/fbz/zip_extract.rs @@ -10,7 +10,7 @@ use std::{ time::{Duration, SystemTime, UNIX_EPOCH}, }; -use fastbz2::{DecodeOptions, DecodeProgress, Error, OutputSink, Result, WriterSink, deflate, gzip}; +use fbz::{DecodeOptions, DecodeProgress, Error, OutputSink, Result, WriterSink, deflate, gzip}; use rayon::prelude::*; use zip::{CompressionMethod, ZipArchive, extra_fields::ExtraField}; @@ -301,7 +301,7 @@ where } let pool = rayon::ThreadPoolBuilder::new() .num_threads(threads) - .thread_name(|index| format!("fastbz2-zip-{index}")) + .thread_name(|index| format!("fbz-zip-{index}")) .build() .map_err(|error| invalid(error.to_string()))?; pool.install(|| across.par_iter().try_for_each(|entry| run(entry, DecodeOptions { threads: 1, ..options }))) diff --git a/src/decode.rs b/src/decode.rs index 0c4fa01..06d181c 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -284,7 +284,7 @@ pub(crate) fn thread_pool(threads: usize) -> Result>> { } ThreadPoolBuilder::new() .num_threads(threads) - .thread_name(|number| format!("fastbz2-{number}")) + .thread_name(|number| format!("fbz-{number}")) .build() .map(Arc::new) .map(Some) diff --git a/src/error.rs b/src/error.rs index 60d179f..3dc82ec 100644 --- a/src/error.rs +++ b/src/error.rs @@ -38,6 +38,7 @@ pub enum Error { UnexpectedEof { bit_offset: u64, requested: u64, remaining: u64 }, InvalidStreamHeader, InvalidGzip(String), + InvalidLz4(String), InvalidZip(String), Decode { bit_offset: u64, source: DecodeError }, InvalidIndex(String), @@ -57,9 +58,10 @@ impl fmt::Display for Error { } Self::InvalidStreamHeader => write!(f, "input does not start with a bzip2 BZh1-BZh9 header"), Self::InvalidGzip(message) => write!(f, "invalid gzip stream: {message}"), + Self::InvalidLz4(message) => write!(f, "invalid LZ4 frame: {message}"), Self::InvalidZip(message) => write!(f, "invalid ZIP archive: {message}"), Self::Decode { bit_offset, source } => write!(f, "bzip2 decode error at bit {bit_offset}: {source}"), - Self::InvalidIndex(message) => write!(f, "invalid fastbz2 index: {message}"), + Self::InvalidIndex(message) => write!(f, "invalid fbz index: {message}"), Self::InvalidConfiguration(message) => write!(f, "invalid configuration: {message}"), Self::Io(source) => source.fmt(f), } diff --git a/src/gzip.rs b/src/gzip.rs index 07fac3c..0ad22f9 100644 --- a/src/gzip.rs +++ b/src/gzip.rs @@ -4,6 +4,7 @@ use std::{io::Write, sync::OnceLock}; use crate::{ DecodeOptions, DecodeProgress, Error, OutputSink, Result, WriterSink, + history::extend_match, pipeline::{Job, PipelineLimits, run_staged_ordered}, }; @@ -224,19 +225,6 @@ enum InitialHistory { Unknown, } -fn extend_match(output: &mut Vec, distance: usize, length: usize) { - output.reserve(length); - let append_start = output.len(); - let first = distance.min(length); - output.extend_from_within(append_start - distance..append_start - distance + first); - let mut copied = first; - while copied < length { - let count = copied.min(length - copied); - output.extend_from_within(append_start..append_start + count); - copied += count; - } -} - struct MarkerOutput { marked: Vec, clean: Vec, @@ -1132,24 +1120,7 @@ impl<'a, W: OutputSink> Emitter<'a, W> { if distance == 0 || distance > available { return Err(invalid(format!("back-reference distance {distance} exceeds {available} available bytes"))); } - let original = self.buffer.len(); - if distance >= length { - self.buffer.extend_from_within(original - distance..original - distance + length); - } else if distance == 1 { - self.buffer.resize(original + length, self.buffer[original - 1]); - } else if length <= distance * 2 { - self.buffer.extend_from_within(original - distance..original); - self.buffer.extend_from_within(original..original + length - distance); - } else { - self.buffer.resize(original + length, 0); - self.buffer.copy_within(original - distance..original, original); - let mut copied = distance; - while copied < length { - let count = copied.min(length - copied); - self.buffer.copy_within(original..original + count, original + copied); - copied += count; - } - } + extend_match(&mut self.buffer, distance, length); self.member_decoded = self.member_decoded.checked_add(length as u64).ok_or_else(|| invalid("decoded offset overflow"))?; if self.buffer.len() - self.history_len >= OUTPUT_CHUNK { self.flush_pending()?; diff --git a/src/history.rs b/src/history.rs new file mode 100644 index 0000000..d595bf6 --- /dev/null +++ b/src/history.rs @@ -0,0 +1,42 @@ +pub(crate) fn extend_match(output: &mut Vec, distance: usize, length: usize) { + let original = output.len(); + if distance >= length { + output.extend_from_within(original - distance..original - distance + length); + } else if distance == 1 { + output.resize(original + length, output[original - 1]); + } else if length <= distance * 2 { + output.extend_from_within(original - distance..original); + output.extend_from_within(original..original + length - distance); + } else { + output.resize(original + length, output[original - distance]); + output.copy_within(original - distance..original, original); + let mut copied = distance; + while copied < length { + let count = copied.min(length - copied); + output.copy_within(original..original + count, original + copied); + copied += count; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn overlapping_matches_repeat_the_requested_history() { + for initial_len in 1..20 { + for distance in 1..=initial_len { + for length in 0..80 { + let mut expected: Vec<_> = (0..initial_len as u16).collect(); + for _ in 0..length { + expected.push(expected[expected.len() - distance]); + } + let mut actual: Vec<_> = (0..initial_len as u16).collect(); + extend_match(&mut actual, distance, length); + assert_eq!(actual, expected, "initial={initial_len}, distance={distance}, length={length}"); + } + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 5a47ff2..9a1fd1b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,10 @@ pub mod deflate; mod error; mod format; pub mod gzip; +mod history; mod index; mod indexed; +pub mod lz4; mod output; mod pipeline; mod source; @@ -43,7 +45,7 @@ mod python { types::PyBytes, }; - create_exception!(fastbz2, BadBzip2File, PyOSError); + create_exception!(fbz, BadBzip2File, PyOSError); fn python_error(error: crate::Error) -> PyErr { match error { diff --git a/src/lz4.rs b/src/lz4.rs new file mode 100644 index 0000000..c0de632 --- /dev/null +++ b/src/lz4.rs @@ -0,0 +1,585 @@ +//! LZ4 frame and block decompression implemented in safe Rust. + +use std::{hash::Hasher, io::Write}; + +use rayon::ThreadPoolBuilder; +use twox_hash::XxHash32; + +use crate::history::extend_match; +use crate::pipeline::{Job, PipelineLimits, run_ordered}; +use crate::{DecodeOptions, DecodeProgress, Error, OutputSink, Result, WriterSink}; + +const FRAME_MAGIC: u32 = 0x184d_2204; +const LEGACY_MAGIC: u32 = 0x184c_2102; +const SKIPPABLE_MAGIC_START: u32 = 0x184d_2a50; +const SKIPPABLE_MAGIC_END: u32 = 0x184d_2a5f; +const UNCOMPRESSED_BIT: u32 = 1 << 31; +const WINDOW_SIZE: usize = 64 * 1024; +const MIN_PARALLEL_INPUT: usize = 1024 * 1024; +const AUTO_THREAD_LIMIT: usize = 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BlockMode { + Independent, + Linked, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Frame { + pub compressed_start: u64, + pub compressed_end: u64, + pub decoded_start: u64, + pub decoded_len: u64, + pub block_max_size: u32, + pub block_mode: BlockMode, + pub block_checksums: bool, + pub content_checksum: bool, + pub declared_content_size: Option, + pub first_block: usize, + pub block_count: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Block { + pub frame: u32, + pub compressed_start: u64, + pub compressed_end: u64, + pub decoded_start: u64, + pub decoded_len: u64, + pub stored: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Report { + pub source_len: u64, + pub decoded_len: u64, + pub frames: Vec, + pub blocks: Vec, +} + +#[derive(Clone, Debug)] +struct BlockLayout { + data_start: usize, + data_end: usize, + stored: bool, + expected_checksum: Option, +} + +#[derive(Clone, Debug)] +struct FrameLayout { + source_start: usize, + source_end: usize, + max_block_size: usize, + mode: BlockMode, + block_checksums: bool, + content_checksum: bool, + content_size: Option, + expected_content_checksum: Option, + blocks: Vec, +} + +fn invalid(message: impl Into) -> Error { + Error::InvalidLz4(message.into()) +} + +fn worker_threads(options: DecodeOptions) -> usize { + let requested = options.resolved_threads(); + if options.threads == 0 { requested.min(AUTO_THREAD_LIMIT) } else { requested } +} + +fn read_u32(data: &[u8], position: usize, context: &str) -> Result { + let bytes = data.get(position..position.saturating_add(4)).ok_or_else(|| invalid(format!("truncated {context} at byte {position}")))?; + Ok(u32::from_le_bytes(bytes.try_into().unwrap())) +} + +fn xxhash32(data: &[u8]) -> u32 { + let mut hasher = XxHash32::with_seed(0); + hasher.write(data); + hasher.finish() as u32 +} + +fn parse_frame(data: &[u8], start: usize) -> Result { + let flg = *data.get(start + 4).ok_or_else(|| invalid("truncated frame descriptor"))?; + let bd = *data.get(start + 5).ok_or_else(|| invalid("truncated frame descriptor"))?; + if flg & 0xc0 != 0x40 { + return Err(invalid(format!("unsupported frame version bits {:02x}", flg & 0xc0))); + } + if flg & 0x02 != 0 || bd & 0x8f != 0 { + return Err(invalid("reserved frame descriptor bits are set")); + } + let mode = if flg & 0x20 != 0 { BlockMode::Independent } else { BlockMode::Linked }; + let block_checksums = flg & 0x10 != 0; + let has_content_size = flg & 0x08 != 0; + let content_checksum = flg & 0x04 != 0; + let has_dictionary = flg & 0x01 != 0; + let max_block_size = match (bd >> 4) & 0x07 { + 4 => 64 * 1024, + 5 => 256 * 1024, + 6 => 1024 * 1024, + 7 => 4 * 1024 * 1024, + value => return Err(invalid(format!("unsupported block maximum-size code {value}"))), + }; + let mut position = start + 6; + let content_size = if has_content_size { + let bytes = data.get(position..position.saturating_add(8)).ok_or_else(|| invalid("truncated frame content size"))?; + position += 8; + Some(u64::from_le_bytes(bytes.try_into().unwrap())) + } else { + None + }; + if has_dictionary { + let dictionary = read_u32(data, position, "dictionary ID")?; + return Err(invalid(format!("external dictionary {dictionary:08x} is not supported"))); + } + let expected_header_checksum = *data.get(position).ok_or_else(|| invalid("truncated frame header checksum"))?; + let descriptor = data.get(start + 4..position).ok_or_else(|| invalid("invalid frame descriptor range"))?; + let header_checksum = (xxhash32(descriptor) >> 8) as u8; + if header_checksum != expected_header_checksum { + return Err(invalid(format!("header checksum mismatch: expected {expected_header_checksum:02x}, decoded {header_checksum:02x}"))); + } + position += 1; + let mut blocks = Vec::new(); + loop { + let value = read_u32(data, position, "block header")?; + position += 4; + if value == 0 { + break; + } + let stored = value & UNCOMPRESSED_BIT != 0; + let size = (value & !UNCOMPRESSED_BIT) as usize; + if size == 0 || size > max_block_size { + return Err(invalid(format!("block at byte {} has invalid size {size} for {max_block_size}-byte frames", position - 4))); + } + let data_start = position; + let data_end = position.checked_add(size).filter(|&end| end <= data.len()).ok_or_else(|| invalid("block data exceeds the frame"))?; + position = data_end; + let expected_checksum = if block_checksums { + let checksum = read_u32(data, position, "block checksum")?; + position += 4; + Some(checksum) + } else { + None + }; + blocks.push(BlockLayout { data_start, data_end, stored, expected_checksum }); + } + let expected_content_checksum = if content_checksum { + let checksum = read_u32(data, position, "content checksum")?; + position += 4; + Some(checksum) + } else { + None + }; + Ok(FrameLayout { + source_start: start, + source_end: position, + max_block_size, + mode, + block_checksums, + content_checksum, + content_size, + expected_content_checksum, + blocks, + }) +} + +fn parse(data: &[u8]) -> Result> { + let mut frames = Vec::new(); + let mut position = 0; + while position < data.len() { + let magic = read_u32(data, position, "frame magic")?; + if (SKIPPABLE_MAGIC_START..=SKIPPABLE_MAGIC_END).contains(&magic) { + let length = read_u32(data, position + 4, "skippable-frame length")? as usize; + position = position + .checked_add(8) + .and_then(|value| value.checked_add(length)) + .filter(|&end| end <= data.len()) + .ok_or_else(|| invalid("skippable frame exceeds the input"))?; + continue; + } + if magic == LEGACY_MAGIC { + return Err(invalid("legacy LZ4 frames are not supported")); + } + if magic != FRAME_MAGIC { + return Err(invalid(format!("wrong magic {magic:08x} at byte {position}"))); + } + let frame = parse_frame(data, position)?; + position = frame.source_end; + frames.push(frame); + } + if frames.is_empty() { + return Err(invalid("input contains no LZ4 frames")); + } + Ok(frames) +} + +fn read_length(input: &[u8], position: &mut usize, initial: usize) -> Result { + if initial != 15 { + return Ok(initial); + } + let mut length = initial; + loop { + let extra = *input.get(*position).ok_or_else(|| invalid("truncated extended sequence length"))? as usize; + *position += 1; + length = length.checked_add(extra).ok_or_else(|| invalid("sequence length overflows usize"))?; + if extra != 255 { + return Ok(length); + } + } +} + +fn copy_match(output: &mut Vec, dictionary: &[u8], offset: usize, mut length: usize) -> Result<()> { + if offset == 0 || offset > dictionary.len() + output.len() { + return Err(invalid(format!("match offset {offset} exceeds the available {}-byte history", dictionary.len() + output.len()))); + } + let dictionary_needed = offset.saturating_sub(output.len()); + if dictionary_needed != 0 { + let start = dictionary.len().checked_sub(dictionary_needed).ok_or_else(|| invalid("match offset exceeds the external dictionary"))?; + let take = length.min(dictionary_needed); + output.extend_from_slice(&dictionary[start..start + take]); + length -= take; + } + if length != 0 { + extend_match(output, offset, length); + } + Ok(()) +} + +fn decompress_block(input: &[u8], dictionary: &[u8], max_output: usize) -> Result> { + let mut output = Vec::with_capacity(max_output); + let mut position = 0; + while position < input.len() { + let token = input[position]; + position += 1; + let literal_len = read_length(input, &mut position, (token >> 4) as usize)?; + let literal_end = position.checked_add(literal_len).filter(|&end| end <= input.len()).ok_or_else(|| invalid("literal run exceeds the block"))?; + if output.len().checked_add(literal_len).is_none_or(|length| length > max_output) { + return Err(invalid(format!("decoded block exceeds its {max_output}-byte maximum"))); + } + output.extend_from_slice(&input[position..literal_end]); + position = literal_end; + if position == input.len() { + break; + } + let offset_bytes = input.get(position..position.saturating_add(2)).ok_or_else(|| invalid("truncated match offset"))?; + position += 2; + let offset = u16::from_le_bytes(offset_bytes.try_into().unwrap()) as usize; + let match_len = read_length(input, &mut position, (token & 0x0f) as usize)?.checked_add(4).ok_or_else(|| invalid("match length overflows usize"))?; + if output.len().checked_add(match_len).is_none_or(|length| length > max_output) { + return Err(invalid(format!("decoded block exceeds its {max_output}-byte maximum"))); + } + copy_match(&mut output, dictionary, offset, match_len)?; + } + Ok(output) +} + +enum DecodedBlock { + Stored { start: usize, end: usize }, + Decoded(Vec), +} + +impl DecodedBlock { + fn bytes<'a>(&'a self, source: &'a [u8]) -> &'a [u8] { + match self { + Self::Stored { start, end } => &source[*start..*end], + Self::Decoded(bytes) => bytes, + } + } + + fn retained_bytes(&self) -> usize { + match self { + Self::Stored { .. } => 0, + Self::Decoded(bytes) => bytes.capacity(), + } + } +} + +fn decode_layout_block(data: &[u8], block: &BlockLayout, dictionary: &[u8], max_block_size: usize) -> Result { + let encoded = &data[block.data_start..block.data_end]; + if let Some(expected) = block.expected_checksum { + let actual = xxhash32(encoded); + if actual != expected { + return Err(invalid(format!("block checksum mismatch at byte {}: expected {expected:08x}, decoded {actual:08x}", block.data_start))); + } + } + if block.stored { + Ok(DecodedBlock::Stored { start: block.data_start, end: block.data_end }) + } else { + Ok(DecodedBlock::Decoded(decompress_block(encoded, dictionary, max_block_size)?)) + } +} + +struct FrameCommitter<'a, S, P> { + data: &'a [u8], + output: &'a mut S, + progress: &'a mut P, + hasher: Option, + decoded_base: u64, + decoded: u64, + frame_number: u32, + blocks: &'a mut Vec, +} + +impl FrameCommitter<'_, S, P> { + fn commit(&mut self, layout: &BlockLayout, decoded: DecodedBlock) -> Result<()> { + let bytes = decoded.bytes(self.data); + if let Some(hasher) = &mut self.hasher { + hasher.write(bytes); + } + let decoded_len = bytes.len() as u64; + match decoded { + DecodedBlock::Stored { start, end } => self.output.write_borrowed(&self.data[start..end])?, + DecodedBlock::Decoded(bytes) => self.output.write_owned_from(bytes, 0)?, + } + self.blocks.push(Block { + frame: self.frame_number, + compressed_start: layout.data_start as u64, + compressed_end: layout.data_end as u64, + decoded_start: self.decoded_base + self.decoded, + decoded_len, + stored: layout.stored, + }); + self.decoded = self.decoded.checked_add(decoded_len).ok_or_else(|| invalid("decoded length overflows u64"))?; + (self.progress)(DecodeProgress { compressed_bytes: layout.data_end as u64, decoded_bytes: self.decoded_base + self.decoded }); + Ok(()) + } +} + +fn decode_independent( + data: &[u8], + frame: &FrameLayout, + options: DecodeOptions, + committer: &mut FrameCommitter<'_, S, P>, +) -> Result<()> { + let threads = worker_threads(options); + let source_bytes = frame.source_end - frame.source_start; + let parallel_work = frame.blocks.iter().any(|block| !block.stored || block.expected_checksum.is_some()); + if threads == 1 || frame.blocks.len() < 2 || source_bytes < MIN_PARALLEL_INPUT || !parallel_work { + for block in &frame.blocks { + let decoded = decode_layout_block(data, block, &[], frame.max_block_size)?; + committer.commit(block, decoded)?; + } + return Ok(()); + } + let jobs: Vec<_> = frame + .blocks + .iter() + .cloned() + .enumerate() + .map(|(key, block)| Job { key, reservation: if block.stored { 0 } else { frame.max_block_size }, payload: block }) + .collect(); + let pool = + ThreadPoolBuilder::new().num_threads(threads).thread_name(|index| format!("fbz-lz4-{index}")).build().map_err(|error| invalid(error.to_string()))?; + run_ordered( + &pool, + &jobs, + PipelineLimits { memory: options.memory_limit, active: threads.saturating_add(2) }, + |block| decode_layout_block(data, block, &[], frame.max_block_size), + |result| result.as_ref().map_or(0, DecodedBlock::retained_bytes), + |results| { + for (key, block) in frame.blocks.iter().enumerate() { + committer.commit(block, results.take(key)??)?; + } + Ok(()) + }, + ) +} + +fn update_history(history: &mut Vec, bytes: &[u8]) { + if bytes.len() >= WINDOW_SIZE { + history.clear(); + history.extend_from_slice(&bytes[bytes.len() - WINDOW_SIZE..]); + return; + } + let excess = history.len().saturating_add(bytes.len()).saturating_sub(WINDOW_SIZE); + if excess != 0 { + history.drain(..excess); + } + history.extend_from_slice(bytes); +} + +fn decode_linked(data: &[u8], frame: &FrameLayout, committer: &mut FrameCommitter<'_, S, P>) -> Result<()> { + let mut history = Vec::with_capacity(WINDOW_SIZE); + for block in &frame.blocks { + let decoded = decode_layout_block(data, block, &history, frame.max_block_size)?; + update_history(&mut history, decoded.bytes(data)); + committer.commit(block, decoded)?; + } + Ok(()) +} + +pub fn decompress(data: &[u8]) -> Result> { + decompress_with_options(data, DecodeOptions::default()) +} + +pub fn decompress_with_options(data: &[u8], options: DecodeOptions) -> Result> { + let mut output = Vec::new(); + decompress_to_writer_with_options(data, &mut output, options)?; + Ok(output) +} + +pub fn decompress_to_writer(data: &[u8], output: &mut impl Write) -> Result { + decompress_to_writer_with_options(data, output, DecodeOptions::default()) +} + +pub fn decompress_to_writer_with_options(data: &[u8], output: &mut impl Write, options: DecodeOptions) -> Result { + decompress_to_writer_with_options_and_progress(data, output, options, |_| {}) +} + +pub fn decompress_to_writer_with_progress(data: &[u8], output: &mut impl Write, progress: impl FnMut(DecodeProgress)) -> Result { + decompress_to_writer_with_options_and_progress(data, output, DecodeOptions::default(), progress) +} + +pub fn decompress_to_writer_with_options_and_progress( + data: &[u8], + output: &mut impl Write, + options: DecodeOptions, + progress: impl FnMut(DecodeProgress), +) -> Result { + let mut output = WriterSink::new(output); + decompress_to_sink_with_options_and_progress(data, &mut output, options, progress) +} + +#[doc(hidden)] +pub fn decompress_to_sink_with_options_and_progress( + data: &[u8], + output: &mut S, + options: DecodeOptions, + mut progress: P, +) -> Result { + let options = options.validate()?; + let layouts = parse(data)?; + let mut frames = Vec::with_capacity(layouts.len()); + let mut blocks = Vec::new(); + let mut decoded_total = 0_u64; + for (frame_number, layout) in layouts.iter().enumerate() { + let first_block = blocks.len(); + let mut committer = FrameCommitter { + data, + output, + progress: &mut progress, + hasher: layout.content_checksum.then(|| XxHash32::with_seed(0)), + decoded_base: decoded_total, + decoded: 0, + frame_number: u32::try_from(frame_number).map_err(|_| invalid("too many frames"))?, + blocks: &mut blocks, + }; + match layout.mode { + BlockMode::Independent => decode_independent(data, layout, options, &mut committer)?, + BlockMode::Linked => decode_linked(data, layout, &mut committer)?, + } + if let Some(expected) = layout.content_size + && committer.decoded != expected + { + return Err(invalid(format!("content size mismatch: expected {expected}, decoded {}", committer.decoded))); + } + if let Some(expected) = layout.expected_content_checksum { + let actual = committer.hasher.take().unwrap().finish() as u32; + if actual != expected { + return Err(invalid(format!("content checksum mismatch: expected {expected:08x}, decoded {actual:08x}"))); + } + } + let decoded_len = committer.decoded; + decoded_total = decoded_total.checked_add(decoded_len).ok_or_else(|| invalid("decoded length overflows u64"))?; + frames.push(Frame { + compressed_start: layout.source_start as u64, + compressed_end: layout.source_end as u64, + decoded_start: decoded_total - decoded_len, + decoded_len, + block_max_size: layout.max_block_size as u32, + block_mode: layout.mode, + block_checksums: layout.block_checksums, + content_checksum: layout.content_checksum, + declared_content_size: layout.content_size, + first_block, + block_count: blocks.len() - first_block, + }); + progress(DecodeProgress { compressed_bytes: layout.source_end as u64, decoded_bytes: decoded_total }); + } + output.flush()?; + progress(DecodeProgress { compressed_bytes: data.len() as u64, decoded_bytes: decoded_total }); + Ok(Report { source_len: data.len() as u64, decoded_len: decoded_total, frames, blocks }) +} + +#[cfg(test)] +mod tests { + use std::io::{Read, Write}; + + use lz4_flex::frame::{BlockMode as OracleMode, BlockSize, FrameDecoder, FrameEncoder, FrameInfo}; + + use super::*; + + fn oracle_frame(data: &[u8], mode: OracleMode, block_checksums: bool, content_checksum: bool) -> Vec { + let info = FrameInfo::new() + .block_size(BlockSize::Max64KB) + .block_mode(mode) + .block_checksums(block_checksums) + .content_checksum(content_checksum) + .content_size(Some(data.len() as u64)); + let mut encoder = FrameEncoder::with_frame_info(info, Vec::new()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() + } + + #[test] + fn decodes_oracle_independent_and_linked_frames() { + let data = [b"linked and independent LZ4 frames ".repeat(10_000), (0_u8..=255).collect::>().repeat(1_000)].concat(); + for mode in [OracleMode::Independent, OracleMode::Linked] { + for checksums in [false, true] { + let encoded = oracle_frame(&data, mode, checksums, checksums); + for threads in [1, 4] { + assert_eq!(decompress_with_options(&encoded, DecodeOptions { threads, ..DecodeOptions::default() }).unwrap(), data); + } + } + } + } + + #[test] + fn handles_concatenated_and_skippable_frames() { + let first = oracle_frame(b"first", OracleMode::Independent, true, true); + let second = oracle_frame(b"second", OracleMode::Linked, false, false); + let mut encoded = first; + encoded.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); + encoded.extend_from_slice(&3_u32.to_le_bytes()); + encoded.extend_from_slice(b"xyz"); + encoded.extend_from_slice(&second); + let mut output = Vec::new(); + let report = decompress_to_writer_with_options(&encoded, &mut output, DecodeOptions { threads: 4, ..DecodeOptions::default() }).unwrap(); + assert_eq!(output, b"firstsecond"); + assert_eq!(report.frames.len(), 2); + assert_eq!(report.decoded_len, 11); + } + + #[test] + fn rejects_corruption_and_output_overflow() { + let data = b"checksum coverage".repeat(10_000); + let mut encoded = oracle_frame(&data, OracleMode::Independent, true, true); + let middle = encoded.len() / 2; + encoded[middle] ^= 1; + assert!(matches!(decompress(&encoded), Err(Error::InvalidLz4(_)))); + assert!(matches!(decompress_block(&[0x1f, 1, 0, 255, 255, 255, 255], &[], 32), Err(Error::InvalidLz4(_)))); + } + + #[test] + fn oracle_decodes_a_minimal_fbz_block_frame() { + let payload = b"literal-only interoperability"; + let mut encoded = Vec::new(); + encoded.extend_from_slice(&FRAME_MAGIC.to_le_bytes()); + let descriptor = [0x64, 0x40]; + encoded.extend_from_slice(&descriptor); + encoded.push((xxhash32(&descriptor) >> 8) as u8); + encoded.extend_from_slice(&((payload.len() as u32) | UNCOMPRESSED_BIT).to_le_bytes()); + encoded.extend_from_slice(payload); + encoded.extend_from_slice(&0_u32.to_le_bytes()); + encoded.extend_from_slice(&xxhash32(payload).to_le_bytes()); + let mut decoder = FrameDecoder::new(encoded.as_slice()); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); + } + + #[test] + fn automatic_threads_are_bounded_but_explicit_counts_are_preserved() { + assert!(worker_threads(DecodeOptions::default()) <= AUTO_THREAD_LIMIT); + assert_eq!(worker_threads(DecodeOptions { threads: 12, ..DecodeOptions::default() }), 12); + } +} diff --git a/tests/archive_perf.rs b/tests/archive_perf.rs index d64a227..43a9e7b 100644 --- a/tests/archive_perf.rs +++ b/tests/archive_perf.rs @@ -6,7 +6,7 @@ use std::{ }; use crabz2::{Level, compress}; -use fastbz2::{DecodeOptions, OutputSink, gzip as gzip_decoder}; +use fbz::{DecodeOptions, OutputSink, gzip as gzip_decoder}; use flate2::{Compression, write::GzEncoder}; #[allow(dead_code)] @@ -14,11 +14,11 @@ mod support; use support::simplewiki_prefix; fn requested_threads() -> usize { - std::env::var("FASTBZ2_THREADS").ok().map(|value| value.parse().expect("FASTBZ2_THREADS must be an integer")).unwrap_or(0) + std::env::var("FBZ_THREADS").ok().map(|value| value.parse().expect("FBZ_THREADS must be an integer")).unwrap_or(0) } fn binary() -> Command { - let mut command = Command::new(env!("CARGO_BIN_EXE_fastbz2")); + let mut command = Command::new(env!("CARGO_BIN_EXE_fbz")); command.args(["-P", &requested_threads().to_string()]); command } @@ -70,7 +70,7 @@ fn fixture(extension: &str, encode: impl Fn(&[u8]) -> Vec) -> Fixture { Fixture { directory, contents, input } } -fn fastbz2_overhead(extension: &str, encode: impl Fn(&[u8]) -> Vec) { +fn fbz_overhead(extension: &str, encode: impl Fn(&[u8]) -> Vec) { let fixture = fixture(extension, encode); let warm = fixture.directory.path().join("warm"); fs::create_dir(&warm).unwrap(); @@ -83,7 +83,7 @@ fn fastbz2_overhead(extension: &str, encode: impl Fn(&[u8]) -> Vec) { assert_eq!(fs::read(extracted.join("payload.bin")).unwrap(), fixture.contents); let raw_ratio = extract_time.as_secs_f64() / raw_time.as_secs_f64(); - eprintln!("{extension}: raw tar {raw_time:.3?}, fastbz2 extract {extract_time:.3?} ({raw_ratio:.3}x raw)"); + eprintln!("{extension}: raw tar {raw_time:.3?}, fbz extract {extract_time:.3?} ({raw_ratio:.3}x raw)"); assert!(raw_ratio <= 3.0, "tar extraction exceeded the broad 3x raw-decode guard; measured {raw_ratio:.3}x"); } @@ -179,9 +179,9 @@ fn tgz_output_cadence() { } #[test] -#[ignore = "local single-run fastbz2 gzip tar extraction overhead"] -fn tgz_fastbz2_overhead() { - fastbz2_overhead("tgz", gzip); +#[ignore = "local single-run fbz gzip tar extraction overhead"] +fn tgz_fbz_overhead() { + fbz_overhead("tgz", gzip); } #[test] @@ -191,9 +191,9 @@ fn tgz_system_reference() { } #[test] -#[ignore = "local single-run fastbz2 bzip2 tar extraction overhead"] -fn tbz2_fastbz2_overhead() { - fastbz2_overhead("tbz2", |contents| compress(contents, Level::BEST)); +#[ignore = "local single-run fbz bzip2 tar extraction overhead"] +fn tbz2_fbz_overhead() { + fbz_overhead("tbz2", |contents| compress(contents, Level::BEST)); } #[test] diff --git a/tests/cli.rs b/tests/cli.rs index ec8a1dd..f2fae15 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -8,6 +8,7 @@ use std::{ use crabz2::{Level, compress}; use flate2::{Compression, write::GzEncoder}; +use lz4_flex::frame::{BlockMode as Lz4BlockMode, BlockSize as Lz4BlockSize, FrameEncoder as Lz4Encoder, FrameInfo as Lz4FrameInfo}; use zip::{CompressionMethod as ZipCompression, ZipArchive, ZipWriter, write::FullFileOptions}; #[allow(dead_code)] @@ -15,7 +16,7 @@ mod support; use support::{ZipMethod, zip_bytes, zip_with_modes}; fn binary() -> Command { - Command::new(env!("CARGO_BIN_EXE_fastbz2")) + Command::new(env!("CARGO_BIN_EXE_fbz")) } fn write_compressed(path: &Path, plain: &[u8]) { @@ -32,6 +33,18 @@ fn write_gzip(path: &Path, plain: &[u8]) { fs::write(path, gzip_bytes(plain)).unwrap(); } +fn lz4_bytes(plain: &[u8], mode: Lz4BlockMode) -> Vec { + let info = Lz4FrameInfo::new() + .block_size(Lz4BlockSize::Max64KB) + .block_mode(mode) + .block_checksums(true) + .content_checksum(true) + .content_size(Some(plain.len() as u64)); + let mut encoder = Lz4Encoder::with_frame_info(info, Vec::new()); + encoder.write_all(plain).unwrap(); + encoder.finish().unwrap() +} + fn linked_zip() -> Vec { zip_with_modes(&[ ("nested/", b"", ZipMethod::Stored, 0o040750), @@ -361,6 +374,64 @@ fn gzip_magic_fallback_stdin_limits_and_corruption_work() { assert!(!output.exists()); } +#[test] +fn lz4_extension_magic_reporting_limits_and_corruption_work() { + let directory = tempfile::tempdir().unwrap(); + let input = directory.path().join("sample.lz4"); + let output = directory.path().join("sample"); + let plain: Vec<_> = (0..2_000_000).map(|i| ((i * 31 + i / 97) & 255) as u8).collect(); + let encoded = lz4_bytes(&plain, Lz4BlockMode::Independent); + fs::write(&input, &encoded).unwrap(); + + let decoded = binary().args(["-P", "4", input.to_str().unwrap()]).output().unwrap(); + assert!(decoded.status.success(), "{}", String::from_utf8_lossy(&decoded.stderr)); + assert_eq!(fs::read(&output).unwrap(), plain); + + let tested = binary().args(["--test", "-P", "4", input.to_str().unwrap()]).output().unwrap(); + assert!(tested.status.success()); + let listed = binary().args(["--list", "--json", "-P", "4", input.to_str().unwrap()]).output().unwrap(); + assert!(listed.status.success()); + let value: serde_json::Value = serde_json::from_slice(&listed.stdout).unwrap(); + assert_eq!(value["format"], "lz4"); + assert_eq!(value["decoded_bytes"], plain.len()); + assert_eq!(value["frames"][0]["block_mode"], "independent"); + assert!(value["blocks"].as_array().unwrap().len() > 1); + + let magic_input = directory.path().join("mystery.data"); + fs::write(&magic_input, &encoded).unwrap(); + let magic_output = directory.path().join("mystery.data.out"); + let magic = binary().args(["-P", "4", magic_input.to_str().unwrap()]).output().unwrap(); + assert!(magic.status.success()); + assert_eq!(fs::read(&magic_output).unwrap(), plain); + + fs::remove_file(&output).unwrap(); + let limited = binary().args(["--max-output", "1K", input.to_str().unwrap()]).output().unwrap(); + assert_eq!(limited.status.code(), Some(3)); + assert!(!output.exists()); + + let mut corrupt = encoded; + let last = corrupt.len() - 1; + corrupt[last] ^= 1; + fs::write(&input, corrupt).unwrap(); + let rejected = binary().arg(input.to_str().unwrap()).output().unwrap(); + assert_eq!(rejected.status.code(), Some(3)); + assert!(!output.exists()); +} + +#[test] +fn linked_tar_lz4_streams_through_the_shared_extractor() { + let directory = tempfile::tempdir().unwrap(); + let input = directory.path().join("bundle.tar.lz4"); + let output = directory.path().join("unpacked"); + let plain_tar = tar_bytes(&[("first.txt", b"first"), ("nested/second.txt", b"second")]); + fs::write(&input, lz4_bytes(&plain_tar, Lz4BlockMode::Linked)).unwrap(); + + let extracted = binary().args(["-P", "4", "-C", output.to_str().unwrap(), input.to_str().unwrap()]).output().unwrap(); + assert!(extracted.status.success(), "{}", String::from_utf8_lossy(&extracted.stderr)); + assert_eq!(fs::read(output.join("first.txt")).unwrap(), b"first"); + assert_eq!(fs::read(output.join("nested/second.txt")).unwrap(), b"second"); +} + #[test] fn mixed_bzip2_gzip_and_tar_inputs_share_output_policy() { let directory = tempfile::tempdir().unwrap(); diff --git a/tests/corpus.rs b/tests/corpus.rs index cdafb7f..68c7ae8 100644 --- a/tests/corpus.rs +++ b/tests/corpus.rs @@ -10,7 +10,7 @@ use std::{ }; use crabz2::{Level, compress}; -use fastbz2::{DecodeOptions, decompress}; +use fbz::{DecodeOptions, decompress}; use libbz2_rs_sys::{BZ_OK, BZ_STREAM_END, BZ2_bzDecompress, BZ2_bzDecompressEnd, BZ2_bzDecompressInit, bz_stream}; const CHUNK: usize = 64 * 1024; @@ -31,7 +31,7 @@ fn corpus_files(extension: &str) -> Vec { } /// Decode every concatenated stream through the low-level API so the oracle -/// has the same whole-file semantics as fastbz2. +/// has the same whole-file semantics as fbz. fn oracle_decompress(input: &[u8]) -> Result, i32> { if input.is_empty() { return Err(libbz2_rs_sys::BZ_DATA_ERROR_MAGIC); @@ -106,7 +106,7 @@ fn valid_upstream_corpus_matches_oracle() { let expected = oracle_decompress(&encoded).unwrap_or_else(|status| panic!("oracle rejected {} with {status}", path.display())); for threads in [1, 2] { let actual = decompress(&encoded, DecodeOptions { threads, ..DecodeOptions::default() }) - .unwrap_or_else(|error| panic!("fastbz2 rejected {} with {error}", path.display())); + .unwrap_or_else(|error| panic!("fbz rejected {} with {error}", path.display())); assert_eq!(actual, expected, "{} with {threads} threads", path.display()); } } @@ -119,7 +119,7 @@ fn corrupt_upstream_corpus_is_rejected() { for path in files { let encoded = fs::read(&path).unwrap(); assert!(oracle_decompress(&encoded).is_err(), "oracle accepted {}", path.display()); - assert!(decompress(&encoded, DecodeOptions::default()).is_err(), "fastbz2 accepted {}", path.display()); + assert!(decompress(&encoded, DecodeOptions::default()).is_err(), "fbz accepted {}", path.display()); } } @@ -139,11 +139,11 @@ fn performance_regression_stays_bounded() { let plain = source.repeat(2); let encoded = compress(&plain, Level::FASTEST); let repeats = 3; - let fastbz2_time = benchmark::elapsed(repeats, || { + let fbz_time = benchmark::elapsed(repeats, || { std::hint::black_box(decompress(&encoded, DecodeOptions { threads: 2, ..DecodeOptions::default() }).unwrap()); }); let oracle_time = benchmark::elapsed(repeats, || { std::hint::black_box(oracle_decompress(&encoded).unwrap()); }); - assert!(fastbz2_time.as_secs_f64() <= oracle_time.as_secs_f64() * 1.3, "fastbz2 {fastbz2_time:?} exceeded 1.3x oracle {oracle_time:?}"); + assert!(fbz_time.as_secs_f64() <= oracle_time.as_secs_f64() * 1.3, "fbz {fbz_time:?} exceeded 1.3x oracle {oracle_time:?}"); } diff --git a/tests/corpus/README.md b/tests/corpus/README.md index 0548c6a..d6d2f6c 100644 --- a/tests/corpus/README.md +++ b/tests/corpus/README.md @@ -15,4 +15,4 @@ its hot path for no realistic modern input. The uncompressed reference bytes are deliberately not stored. Tests decode valid inputs with the maintained pure-Rust `libbz2-rs-sys` implementation and -compare `fastbz2` byte-for-byte with that differential oracle. +compare `fbz` byte-for-byte with that differential oracle. diff --git a/tests/gzip_oracle.rs b/tests/gzip_oracle.rs index 76effb0..d286157 100644 --- a/tests/gzip_oracle.rs +++ b/tests/gzip_oracle.rs @@ -5,8 +5,8 @@ mod benchmark; use std::io::Read; #[cfg(not(debug_assertions))] -use fastbz2::DecodeOptions; -use fastbz2::gzip; +use fbz::DecodeOptions; +use fbz::gzip; use flate2::{Compression, read::MultiGzDecoder, write::GzEncoder}; fn oracle_decompress(input: &[u8]) -> Vec { @@ -87,13 +87,13 @@ fn assert_performance(plain: &[u8], limit: f64) { let encoded = compress(plain); assert_eq!(gzip::decompress(&encoded).unwrap(), oracle_decompress(&encoded)); let repeats = 2; - let fastbz2_time = benchmark::elapsed(repeats, || { + let fbz_time = benchmark::elapsed(repeats, || { std::hint::black_box(gzip::decompress(&encoded).unwrap()); }); let oracle_time = benchmark::elapsed(repeats, || { std::hint::black_box(oracle_decompress(&encoded)); }); - assert!(fastbz2_time.as_secs_f64() <= oracle_time.as_secs_f64() * limit, "fastbz2 gzip {fastbz2_time:?} exceeded {limit}x oracle {oracle_time:?}"); + assert!(fbz_time.as_secs_f64() <= oracle_time.as_secs_f64() * limit, "fbz gzip {fbz_time:?} exceeded {limit}x oracle {oracle_time:?}"); } #[test] diff --git a/tests/lz4_oracle.rs b/tests/lz4_oracle.rs new file mode 100644 index 0000000..729ecbe --- /dev/null +++ b/tests/lz4_oracle.rs @@ -0,0 +1,76 @@ +use std::io::{Read, Write}; + +use fbz::{DecodeOptions, lz4}; +use lz4_flex::frame::{BlockMode, BlockSize, FrameDecoder, FrameEncoder, FrameInfo}; + +fn encode(data: &[u8], block_size: BlockSize, block_mode: BlockMode, block_checksum: bool, content_checksum: bool, content_size: bool) -> Vec { + let info = FrameInfo::new() + .block_size(block_size) + .block_mode(block_mode) + .block_checksums(block_checksum) + .content_checksum(content_checksum) + .content_size(content_size.then_some(data.len() as u64)); + let mut encoder = FrameEncoder::with_frame_info(info, Vec::new()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + +fn oracle_decode(encoded: &[u8]) -> Vec { + let mut output = Vec::new(); + FrameDecoder::new(encoded).read_to_end(&mut output).unwrap(); + output +} + +fn pseudorandom(length: u32) -> Vec { + (0..length) + .scan(0x9e37_79b9_u32, |state, _| { + *state ^= *state << 13; + *state ^= *state >> 17; + *state ^= *state << 5; + Some(*state as u8) + }) + .collect() +} + +fn assert_matches(data: &[u8], encoded: &[u8], options: DecodeOptions) { + assert_eq!(lz4::decompress_with_options(encoded, options).unwrap(), data); + assert_eq!(oracle_decode(encoded), data); +} + +#[test] +fn frame_matrix_matches_lz4_flex() { + let options = DecodeOptions { threads: 4, ..DecodeOptions::default() }; + let matrix_data = b"frame descriptor and block boundary coverage ".repeat(2_500); + for block_size in [BlockSize::Max64KB, BlockSize::Max256KB, BlockSize::Max1MB, BlockSize::Max4MB] { + for block_mode in [BlockMode::Independent, BlockMode::Linked] { + for block_checksum in [false, true] { + for content_checksum in [false, true] { + for content_size in [false, true] { + let encoded = encode(&matrix_data, block_size, block_mode, block_checksum, content_checksum, content_size); + assert_matches(&matrix_data, &encoded, options); + } + } + } + } + } + + let shapes = [ + Vec::new(), + b"short LZ4 payload".to_vec(), + b"repeated dictionary material ".repeat(2_000), + (0_u8..=255).collect::>().repeat(200), + pseudorandom(50_000), + ]; + for data in shapes { + let encoded = encode(&data, BlockSize::Max64KB, BlockMode::Independent, true, true, true); + assert_matches(&data, &encoded, options); + } +} + +#[test] +fn incompressible_multiblock_input_exercises_parallel_scheduler() { + let data = pseudorandom(1_200_000); + let encoded = encode(&data, BlockSize::Max64KB, BlockMode::Independent, true, true, true); + assert!(encoded.len() > 1024 * 1024); + assert_matches(&data, &encoded, DecodeOptions { threads: 4, ..DecodeOptions::default() }); +} diff --git a/tests/lz4_perf.rs b/tests/lz4_perf.rs new file mode 100644 index 0000000..51969c3 --- /dev/null +++ b/tests/lz4_perf.rs @@ -0,0 +1,146 @@ +#[allow(dead_code, unused_imports)] +mod common; +#[allow(dead_code)] +mod support; + +use std::{fs, hint::black_box, io::Write, path::PathBuf, process::Command, time::Instant}; + +use fbz::{DecodeOptions, lz4}; +use lz4_flex::frame::{BlockMode, BlockSize, FrameEncoder, FrameInfo}; +use support::simplewiki_prefix; + +fn requested_threads() -> usize { + std::env::var("FBZ_THREADS").ok().map(|value| value.parse().expect("FBZ_THREADS must be an integer")).unwrap_or(0) +} + +struct Fixture { + _directory: tempfile::TempDir, + input: PathBuf, +} + +impl Fixture { + fn new() -> Self { + let directory = tempfile::tempdir().unwrap(); + let contents = simplewiki_prefix(); + let info = FrameInfo::new().block_size(BlockSize::Max4MB).block_mode(BlockMode::Independent).block_checksums(false).content_checksum(true); + let mut encoder = FrameEncoder::with_frame_info(info, Vec::new()); + encoder.write_all(&contents).unwrap(); + let encoded = encoder.finish().unwrap(); + let input = directory.path().join("simplewiki-first-5pct.xml.lz4"); + fs::write(&input, &encoded).unwrap(); + eprintln!("LZ4 fixture: {:.1} MiB compressed, {:.1} MiB decoded", encoded.len() as f64 / 1_048_576.0, contents.len() as f64 / 1_048_576.0); + Self { _directory: directory, input } + } +} + +fn fbz_command(input: &std::path::Path) -> Command { + fbz_command_with_threads(input, requested_threads()) +} + +fn fbz_command_with_threads(input: &std::path::Path, threads: usize) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_fbz")); + command.args(["--test", "-q", "-P", &threads.to_string()]).arg(input); + command +} + +fn lz4_command(input: &std::path::Path) -> Command { + let mut command = Command::new("lz4"); + command.args(["-t", "-q"]).arg(input); + command +} + +#[test] +#[cfg(unix)] +#[ignore = "local single-run LZ4 comparison against the Homebrew CLI"] +fn lz4_cli_comparison() { + let fixture = Fixture::new(); + assert!(fbz_command(&fixture.input).status().unwrap().success()); + assert!(lz4_command(&fixture.input).status().unwrap().success()); + + let ours = common::measure(&mut fbz_command(&fixture.input)).unwrap(); + let reference = common::measure(&mut lz4_command(&fixture.input)).unwrap(); + assert!(ours.status.success()); + assert!(reference.status.success()); + eprintln!( + "fbz LZ4: {:.3} ms, peak RSS {:.1} MiB, physical {:.1} MiB", + ours.wall.as_secs_f64() * 1_000.0, + ours.peak_rss_bytes as f64 / 1_048_576.0, + ours.peak_phys_footprint_bytes.map_or(f64::NAN, |bytes| bytes as f64 / 1_048_576.0), + ); + eprintln!( + "lz4 1.10.0: {:.3} ms, peak RSS {:.1} MiB, physical {:.1} MiB", + reference.wall.as_secs_f64() * 1_000.0, + reference.peak_rss_bytes as f64 / 1_048_576.0, + reference.peak_phys_footprint_bytes.map_or(f64::NAN, |bytes| bytes as f64 / 1_048_576.0), + ); + let ratio = ours.wall.as_secs_f64() / reference.wall.as_secs_f64(); + eprintln!("fbz/reference: {ratio:.3}x"); + assert!(ratio <= 1.2, "fbz must remain within 20% of lz4 1.10.0; measured {ratio:.3}x"); +} + +#[test] +#[cfg(unix)] +#[ignore = "local one-run-per-count LZ4 thread-scaling diagnostic"] +fn lz4_thread_sweep() { + let fixture = Fixture::new(); + for threads in [1, 2, 4, 6, 8, 12, 18] { + assert!(fbz_command_with_threads(&fixture.input, threads).status().unwrap().success()); + let result = common::measure(&mut fbz_command_with_threads(&fixture.input, threads)).unwrap(); + assert!(result.status.success()); + eprintln!("{threads:>2} threads: {:>7.3} ms, peak RSS {:>5.1} MiB", result.wall.as_secs_f64() * 1_000.0, result.peak_rss_bytes as f64 / 1_048_576.0,); + } +} + +fn frame(contents: &[u8], block_size: BlockSize) -> Vec { + let info = FrameInfo::new() + .block_size(block_size) + .block_mode(BlockMode::Independent) + .block_checksums(false) + .content_checksum(false) + .content_size(Some(contents.len() as u64)); + let mut encoder = FrameEncoder::with_frame_info(info, Vec::new()); + encoder.write_all(contents).unwrap(); + encoder.finish().unwrap() +} + +fn timed_decode(encoded: &[u8], threads: usize) -> (std::time::Duration, lz4::Report) { + let mut output = Vec::new(); + let start = Instant::now(); + let report = lz4::decompress_to_writer_with_options(encoded, &mut output, DecodeOptions { threads, ..DecodeOptions::default() }).unwrap(); + let elapsed = start.elapsed(); + black_box(output); + (elapsed, report) +} + +#[test] +#[ignore = "local LZ4 long-match and stored-block diagnostics"] +fn lz4_shape_diagnostics() { + let repeated = vec![b'x'; 4 * 1024 * 1024]; + let repeated_frame = frame(&repeated, BlockSize::Max4MB); + timed_decode(&repeated_frame, 1); + let (repeated_time, repeated_report) = timed_decode(&repeated_frame, 1); + assert!(repeated_report.blocks.iter().any(|block| !block.stored)); + + let mut state = 0x9e37_79b9_u32; + let random: Vec<_> = (0..16 * 1024 * 1024) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let stored_frame = frame(&random, BlockSize::Max4MB); + timed_decode(&stored_frame, 1); + timed_decode(&stored_frame, 4); + let (stored_serial, serial_report) = timed_decode(&stored_frame, 1); + let (stored_parallel, parallel_report) = timed_decode(&stored_frame, 4); + assert!(serial_report.blocks.iter().all(|block| block.stored)); + assert!(parallel_report.blocks.iter().all(|block| block.stored)); + eprintln!( + "long match: {:.3} ms; stored serial: {:.3} ms; stored four-thread: {:.3} ms", + repeated_time.as_secs_f64() * 1_000.0, + stored_serial.as_secs_f64() * 1_000.0, + stored_parallel.as_secs_f64() * 1_000.0, + ); +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 7f5afc2..a2b30d3 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -1,6 +1,6 @@ use std::{fs, io::Write, path::Path}; -use fastbz2::{DecodeOptions, decompress}; +use fbz::{DecodeOptions, decompress}; use flate2::{Compression, write::DeflateEncoder}; #[derive(Clone, Copy)] diff --git a/tests/test_api.py b/tests/test_api.py index 02adc91..c69ec8a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,28 +1,27 @@ -import bz2 -import io +import bz2, io from threading import Event, Thread import pytest -import fastbz2 +import fbz def patterned(size): return bytes((i * 37 + i // 251) & 255 for i in range(size)) @pytest.mark.parametrize("level", range(1, 10)) def test_decompress_matches_libbz2_at_every_level(level): plain = patterned(20_000) - assert fastbz2.decompress(bz2.compress(plain, compresslevel=level), threads=2) == plain + assert fbz.decompress(bz2.compress(plain, compresslevel=level), threads=2) == plain def test_parallel_multiblock_and_concatenated_are_deterministic(): first, second = patterned(350_000), patterned(75_000) compressed = bz2.compress(first, compresslevel=1) + bz2.compress(second, compresslevel=9) expected = first + second - for threads in (1, 2, 4, 0): assert fastbz2.decompress(compressed, threads=threads) == expected + for threads in (1, 2, 4, 0): assert fbz.decompress(compressed, threads=threads) == expected def test_bad_crc_raises_package_error(): compressed = bytearray(bz2.compress(b"integrity matters")) compressed[-2] ^= 1 - with pytest.raises(fastbz2.BadBzip2File): fastbz2.decompress(bytes(compressed)) + with pytest.raises(fbz.BadBzip2File): fbz.decompress(bytes(compressed)) def test_seekable_file_and_persisted_index(tmp_path): plain = patterned(350_000) @@ -30,9 +29,9 @@ def test_seekable_file_and_persisted_index(tmp_path): source = tmp_path / "data.bz2" index_path = tmp_path / "data.fbz2i" source.write_bytes(compressed) - encoded = fastbz2.build_index(source, index_path, threads=2) + encoded = fbz.build_index(source, index_path, threads=2) - with fastbz2.open(source, index=index_path) as handle: + with fbz.open(source, index=index_path) as handle: assert handle.size == len(plain) assert handle.seek(99_990) == 99_990 assert handle.read(40) == plain[99_990:100_030] @@ -46,12 +45,12 @@ def test_seekable_file_and_persisted_index(tmp_path): def test_index_is_bound_to_source(): first = bz2.compress(b"first") second = bz2.compress(b"other") - index = fastbz2.build_index(first) - with pytest.raises(ValueError, match="source identity mismatch"): fastbz2.open(second, index=index) + index = fbz.build_index(first) + with pytest.raises(ValueError, match="source identity mismatch"): fbz.open(second, index=index) def test_buffered_reader_compatibility(): plain = patterned(180_000) - with io.BufferedReader(fastbz2.open(bz2.compress(plain, compresslevel=1))) as handle: + with io.BufferedReader(fbz.open(bz2.compress(plain, compresslevel=1))) as handle: assert handle.read(1234) == plain[:1234] handle.seek(100_000) assert handle.read() == plain[100_000:] @@ -70,7 +69,7 @@ def spin(): thread.start() started.wait() before = counter[0] - try: assert fastbz2.decompress(compressed, threads=2) == plain + try: assert fbz.decompress(compressed, threads=2) == plain finally: stop.set() thread.join() diff --git a/tests/test_install.py b/tests/test_install.py index ef04c6b..525a6d7 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,17 +1,17 @@ import gzip, os, subprocess, sysconfig from pathlib import Path -import fastbz2 +import fbz def test_pip_installs_native_cli(): - executable = Path(sysconfig.get_path("scripts")) / ("fastbz2.exe" if os.name == "nt" else "fastbz2") + executable = Path(sysconfig.get_path("scripts")) / ("fbz.exe" if os.name == "nt" else "fbz") assert executable.exists() assert not executable.read_bytes().startswith(b"#!") result = subprocess.run([executable, "--version"], check=True, capture_output=True, text=True) - assert result.stdout.strip() == f"fastbz2 {fastbz2.__version__}" + assert result.stdout.strip() == f"fbz {fbz.__version__}" def test_pip_installed_cli_decodes_gzip(tmp_path): - executable = Path(sysconfig.get_path("scripts")) / ("fastbz2.exe" if os.name == "nt" else "fastbz2") + executable = Path(sysconfig.get_path("scripts")) / ("fbz.exe" if os.name == "nt" else "fbz") plain = b"wheel-installed gzip decoder" * 10_000 source = tmp_path / "sample.gz" source.write_bytes(gzip.compress(plain, mtime=0)) diff --git a/tests/test_scan.py b/tests/test_scan.py index 9631c58..2c47fc9 100644 --- a/tests/test_scan.py +++ b/tests/test_scan.py @@ -2,7 +2,7 @@ import pytest -from fastbz2 import bz2_crc32, scan +from fbz import bz2_crc32, scan def combined_crc(blocks): crc = 0 diff --git a/tests/wiki_perf.rs b/tests/wiki_perf.rs index d36170a..9a5e1ba 100644 --- a/tests/wiki_perf.rs +++ b/tests/wiki_perf.rs @@ -9,7 +9,7 @@ use std::{ time::{Duration, Instant}, }; -use fastbz2::{DecodeOptions, Source, decompress, decompress_to_writer}; +use fbz::{DecodeOptions, Source, decompress, decompress_to_writer}; const FIVE_PERCENT_LEN: usize = 84_423_012; const FIVE_PERCENT_BLAKE3: &str = "69f41f28dc8ac74509d368c6aaec02f3cdf891c9da4ccf8caf625687dcd61908"; @@ -35,7 +35,7 @@ fn corpus_path(name: &str) -> std::path::PathBuf { } fn requested_threads() -> usize { - std::env::var("FASTBZ2_THREADS").ok().map(|value| value.parse().expect("FASTBZ2_THREADS must be an integer")).unwrap_or(0) + std::env::var("FBZ_THREADS").ok().map(|value| value.parse().expect("FBZ_THREADS must be an integer")).unwrap_or(0) } fn physical_footprint_mib(metrics: &common::ProcessMetrics) -> f64 { @@ -73,7 +73,7 @@ fn print_process_metrics(label: &str, metrics: &common::ProcessMetrics) { ); } -fn timed_fastbz2(path: &Path, threads: usize) -> Duration { +fn timed_fbz(path: &Path, threads: usize) -> Duration { let start = Instant::now(); let source = Source::open(path).unwrap(); let mut output = CountingSink::default(); @@ -109,8 +109,8 @@ fn simplewiki_full() { let path = corpus_path("simplewiki-full.xml.bz2"); let options = DecodeOptions { threads: requested_threads(), ..DecodeOptions::default() }; - let elapsed = timed_fastbz2(&path, options.threads); - eprintln!("fastbz2 ({} threads): {elapsed:.3?}", options.resolved_threads()); + let elapsed = timed_fbz(&path, options.threads); + eprintln!("fbz ({} threads): {elapsed:.3?}", options.resolved_threads()); } #[test] @@ -118,10 +118,10 @@ fn simplewiki_full() { #[ignore = "local full SimpleWiki subprocess time and peak-memory benchmark"] fn simplewiki_cli_process_metrics() { let path = corpus_path("simplewiki-full.xml.bz2"); - let mut command = validation_command(env!("CARGO_BIN_EXE_fastbz2"), &path, requested_threads()); + let mut command = validation_command(env!("CARGO_BIN_EXE_fbz"), &path, requested_threads()); let metrics = common::measure(&mut command).unwrap(); assert!(metrics.status.success()); - print_process_metrics("fastbz2 bzip2", &metrics); + print_process_metrics("fbz bzip2", &metrics); } #[test] @@ -130,11 +130,11 @@ fn simplewiki_cli_process_metrics() { fn gzip_cli_process_metrics() { let path = corpus_path("simplewiki-full.xml.gz"); let threads = requested_threads(); - let mut command = validation_command(env!("CARGO_BIN_EXE_fastbz2"), &path, threads); + let mut command = validation_command(env!("CARGO_BIN_EXE_fbz"), &path, threads); let metrics = common::measure(&mut command).unwrap(); assert!(metrics.status.success()); let threads = if threads == 0 { "auto".to_owned() } else { threads.to_string() }; - print_process_metrics(&format!("fastbz2 gzip ({threads} threads)"), &metrics); + print_process_metrics(&format!("fbz gzip ({threads} threads)"), &metrics); } #[test] @@ -167,16 +167,16 @@ fn rapidgzip_rust_process_metrics() { #[test] #[cfg(unix)] -#[ignore = "local single-run fastbz2 full gzip validation"] -fn gzip_fastbz2_validation() { +#[ignore = "local single-run fbz full gzip validation"] +fn gzip_fbz_validation() { let path = corpus_path("simplewiki-full.xml.gz"); let warm_path = corpus_path("simplewiki-first-5pct.xml.gz"); let threads = requested_threads(); - let binary = env!("CARGO_BIN_EXE_fastbz2"); + let binary = env!("CARGO_BIN_EXE_fbz"); warm_validation(binary, &warm_path, threads); let result = timed_validation(binary, &path, threads); assert!(result.status.success()); - eprintln!("fastbz2 full gzip: {:.3}s", result.wall.as_secs_f64()); + eprintln!("fbz full gzip: {:.3}s", result.wall.as_secs_f64()); } #[test] @@ -186,7 +186,7 @@ fn gzip_reference_ratio() { let path = corpus_path("simplewiki-full.xml.gz"); let warm_path = corpus_path("simplewiki-first-5pct.xml.gz"); let threads = requested_threads(); - let ours_binary = env!("CARGO_BIN_EXE_fastbz2"); + let ours_binary = env!("CARGO_BIN_EXE_fbz"); let reference_binary = rapidgzip_binary(); warm_validation(ours_binary, &warm_path, threads); @@ -198,8 +198,8 @@ fn gzip_reference_ratio() { assert!(reference.status.success()); let ratio = ours.wall.as_secs_f64() / reference.wall.as_secs_f64(); - eprintln!("fastbz2 {:.3}s / rapidgzip-rust {:.3}s = {ratio:.3}x", ours.wall.as_secs_f64(), reference.wall.as_secs_f64()); - assert!(ratio <= 1.2, "fastbz2 must remain within 20% of rapidgzip-rust; measured {ratio:.3}x"); + eprintln!("fbz {:.3}s / rapidgzip-rust {:.3}s = {ratio:.3}x", ours.wall.as_secs_f64(), reference.wall.as_secs_f64()); + assert!(ratio <= 1.2, "fbz must remain within 20% of rapidgzip-rust; measured {ratio:.3}x"); } fn timed_vec(name: &str, decode: impl FnOnce() -> Vec) { @@ -217,10 +217,10 @@ fn enwiki_fixture() -> (Vec, usize) { #[test] #[ignore = "local enwiki multistream performance comparison"] -fn enwiki_first_1000_fastbz2_parallel() { +fn enwiki_first_1000_fbz_parallel() { let (encoded, threads) = enwiki_fixture(); let options = DecodeOptions { threads, ..DecodeOptions::default() }; - timed_vec(&format!("fastbz2 parallel ({} threads)", options.resolved_threads()), || decompress(&encoded, options).unwrap()); + timed_vec(&format!("fbz parallel ({} threads)", options.resolved_threads()), || decompress(&encoded, options).unwrap()); } #[test] @@ -232,9 +232,9 @@ fn enwiki_first_1000_crabz2_parallel() { #[test] #[ignore = "local enwiki multistream performance comparison"] -fn enwiki_first_1000_fastbz2_serial() { +fn enwiki_first_1000_fbz_serial() { let (encoded, _) = enwiki_fixture(); - timed_vec("fastbz2 serial", || decompress(&encoded, DecodeOptions { threads: 1, ..DecodeOptions::default() }).unwrap()); + timed_vec("fbz serial", || decompress(&encoded, DecodeOptions { threads: 1, ..DecodeOptions::default() }).unwrap()); } #[test] diff --git a/tests/zip_perf.rs b/tests/zip_perf.rs index a33ed02..c6976e2 100644 --- a/tests/zip_perf.rs +++ b/tests/zip_perf.rs @@ -14,7 +14,7 @@ use support::{ZipMethod, simplewiki_prefix, zip_bytes}; const ENTRY_COUNT: usize = 18; fn requested_threads() -> usize { - std::env::var("FASTBZ2_THREADS").ok().map(|value| value.parse().expect("FASTBZ2_THREADS must be an integer")).unwrap_or(0) + std::env::var("FBZ_THREADS").ok().map(|value| value.parse().expect("FBZ_THREADS must be an integer")).unwrap_or(0) } #[derive(Clone, Copy)] @@ -71,8 +71,8 @@ impl Fixture { } } -fn fastbz2_command(input: &std::path::Path, output: &std::path::Path) -> Command { - let mut command = Command::new(env!("CARGO_BIN_EXE_fastbz2")); +fn fbz_command(input: &std::path::Path, output: &std::path::Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_fbz")); command.args(["-q", "-P", &requested_threads().to_string(), "-C"]).arg(output).arg(input); command } @@ -89,19 +89,19 @@ fn timed(command: &mut Command) -> Duration { start.elapsed() } -fn benchmark(shape: Shape, fastbz2: bool) { +fn benchmark(shape: Shape, fbz: bool) { let fixture = Fixture::new(shape); let warm = fixture.directory.path().join("warm"); let measured = fixture.directory.path().join("measured"); - let warm_time = if fastbz2 { timed(&mut fastbz2_command(&fixture.input, &warm)) } else { timed(&mut unzip_command(&fixture.input, &warm)) }; - let elapsed = if fastbz2 { timed(&mut fastbz2_command(&fixture.input, &measured)) } else { timed(&mut unzip_command(&fixture.input, &measured)) }; + let warm_time = if fbz { timed(&mut fbz_command(&fixture.input, &warm)) } else { timed(&mut unzip_command(&fixture.input, &warm)) }; + let elapsed = if fbz { timed(&mut fbz_command(&fixture.input, &measured)) } else { timed(&mut unzip_command(&fixture.input, &measured)) }; fixture.verify(&measured); - eprintln!("{}: warm {warm_time:.3?}, measured {elapsed:.3?}", if fastbz2 { "fastbz2" } else { "Info-ZIP unzip 6.00 (Apple)" }); + eprintln!("{}: warm {warm_time:.3?}, measured {elapsed:.3?}", if fbz { "fbz" } else { "Info-ZIP unzip 6.00 (Apple)" }); } #[test] #[ignore = "local single-run one-entry ZIP extraction benchmark"] -fn zip_single_fastbz2() { +fn zip_single_fbz() { benchmark(Shape::Single, true); } @@ -113,7 +113,7 @@ fn zip_single_unzip() { #[test] #[ignore = "local single-run many-entry ZIP extraction benchmark"] -fn zip_many_fastbz2() { +fn zip_many_fbz() { benchmark(Shape::Many, true); } @@ -126,14 +126,14 @@ fn zip_many_unzip() { #[test] #[cfg(unix)] #[ignore = "local ZIP extraction time and peak-memory benchmark"] -fn zip_many_fastbz2_process_metrics() { +fn zip_many_fbz_process_metrics() { let fixture = Fixture::new(Shape::Many); let output = fixture.directory.path().join("measured"); - let metrics = common::measure(&mut fastbz2_command(&fixture.input, &output)).unwrap(); + let metrics = common::measure(&mut fbz_command(&fixture.input, &output)).unwrap(); assert!(metrics.status.success()); fixture.verify(&output); eprintln!( - "fastbz2 ZIP: wall {:.3}s, CPU {:.3}s user + {:.3}s system, peak RSS {:.1} MiB, peak physical footprint {:.1} MiB", + "fbz ZIP: wall {:.3}s, CPU {:.3}s user + {:.3}s system, peak RSS {:.1} MiB, peak physical footprint {:.1} MiB", metrics.wall.as_secs_f64(), metrics.user.as_secs_f64(), metrics.system.as_secs_f64(), diff --git a/tools/stage_binaries.py b/tools/stage_binaries.py index ebf2e7a..204040c 100644 --- a/tools/stage_binaries.py +++ b/tools/stage_binaries.py @@ -6,4 +6,4 @@ destination = root / "target" / "wheel-data" / "scripts" destination.mkdir(parents=True, exist_ok=True) suffix = ".exe" if os.name == "nt" else "" -shutil.copy2(source / f"fastbz2{suffix}", destination / f"fastbz2{suffix}") +shutil.copy2(source / f"fbz{suffix}", destination / f"fbz{suffix}")