RAR4 (v29) decode: in-band filters, PPMd-II variant H, solid groups

Cargo.toml

@@ -196,7 +196,9 @@ arsenic = ["alloc"]
 # the encoders permanently return `Error::Unsupported`.
 rar1    = ["alloc"]
 rar2    = ["alloc"]
-rar3    = ["alloc"]
+# RAR3/4 v29: LZ77+Huffman, in-band standard filters, and PPMd-II var H
+# blocks (the latter via the shared `ppmd` model core).
+rar3    = ["alloc", "ppmd"]
 rar5    = ["alloc"]
 # ZIP method 1 (Shrink): PKZIP 1.x dynamic LZW with partial-clear marker.
 # Decoder-only — the encoder permanently returns `Error::Unsupported`.

README.md

@@ -74,8 +74,8 @@ flag, and a `compcol` binary turns the library into a Unix-style filter.
 | StuffIt 5 Arsenic (method 15) | `arsenic` | `.sit` | `Unsupported` (decode-only) | full (range coder + inverse BWT + MTF/RLE + de-randomization) | **real StuffIt 5 fixtures (in-stream CRC-32 + SHA vs `unar`)** |
 | RAR 1.x | `rar1` | `.rar` | `Unsupported` (license) | building blocks only (Huffman tables not license-clean) | — |
 | RAR 2.x | `rar2` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + audio predictor | real rar-2.60 fixtures |
-| RAR 3.x | `rar3` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + E8 filter; PPMd & VM filters refused | libarchive RAR3 fixtures |
-| RAR 5.x | `rar5` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + x86 filter; Delta/ARM refused | RARLAB-CLI fixtures |
+| RAR 3.x | `rar3` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + PPMd-II variant H + standard filters (Delta, x86 E8/E8E9), incl. solid groups; non-standard VM programs refused | libarchive RAR3 fixtures + **real rar-6.24 archives (differential vs UnRAR 7.23)** |
+| RAR 5.x | `rar5` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + Delta/x86 filters (incl. solid groups); ARM refused | RARLAB-CLI fixtures + **real WinRAR 7.23 archives (differential vs UnRAR 7.23)** |
 | HTTP/2 HPACK (RFC 7541) | `hpack` | — | full (header codec + `h2-huffman` string codec) | full (static+dynamic tables, integer/string coding) | RFC 7541 Appendix C vectors |
 | HTTP/3 QPACK (RFC 9204) | `qpack` | — | full (static + dynamic-table encoder driving the encoder stream; eviction-safe) | full (static+dynamic tables via encoder stream, all field representations) | RFC 9204 Appendix B vectors |
 | Canonical Huffman (standalone) | `huffman` | `.huff` | full (length-limited, self-delimiting) | full | own round-trip |

fuzz/Cargo.toml

@@ -275,3 +275,24 @@ path             = "fuzz_targets/roundtrip.rs"
 test             = false
 doc              = false
 bench            = false
+
+[[bin]]
+name             = "decoder_rar2"
+path             = "fuzz_targets/decoder_rar2.rs"
+test             = false
+doc              = false
+bench            = false
+
+[[bin]]
+name             = "decoder_rar3"
+path             = "fuzz_targets/decoder_rar3.rs"
+test             = false
+doc              = false
+bench            = false
+
+[[bin]]
+name             = "decoder_rar5"
+path             = "fuzz_targets/decoder_rar5.rs"
+test             = false
+doc              = false
+bench            = false

fuzz/fuzz_targets/decoder_rar2.rs

@@ -0,0 +1,64 @@
+#![no_main]
+use compcol::Decoder as _;
+use compcol::rar2::Decoder;
+use libfuzzer_sys::fuzz_target;
+
+// Smoke property: the decoder must not panic on arbitrary input.
+// libfuzzer feeds us garbage bytes; we drive the decoder over them
+// and discard the result. Any panic, abort, or undefined behavior
+// trips the harness.
+//
+// RAR2 streams don't self-delimit — the decompressed length lives in
+// the archive container's file header — so we read a 4-byte LE length
+// prefix (capped to 1 MiB so a hostile size field can't make the
+// harness itself allocate unbounded output). `decode` only buffers;
+// the real decompression runs on the first `finish` call.
+fn drive(mut dec: Decoder, payload: &[u8]) {
+    let mut out = vec![0u8; 64 * 1024];
+    let mut consumed = 0;
+    let mut steps = 0;
+    while consumed < payload.len() {
+        match dec.decode(&payload[consumed..], &mut out) {
+            Ok((p, _)) => {
+                if p.consumed == 0 && p.written == 0 {
+                    break;
+                }
+                consumed += p.consumed;
+            }
+            Err(_) => return,
+        }
+        steps += 1;
+        if steps > 4096 {
+            // Defensive: pathological inputs shouldn't make us loop.
+            return;
+        }
+    }
+    let mut steps = 0;
+    while let Ok((p, status)) = dec.finish(&mut out) {
+        if matches!(status, compcol::Status::StreamEnd) {
+            return;
+        }
+        if p.written == 0 {
+            return;
+        }
+        steps += 1;
+        if steps > 4096 {
+            return;
+        }
+    }
+}
+
+fuzz_target!(|data: &[u8]| {
+    let (len_bytes, payload) = if data.len() >= 4 {
+        data.split_at(4)
+    } else {
+        // Too short to carry a length prefix: still fuzz the
+        // no-declared-size path with the whole input as payload.
+        drive(Decoder::new(), data);
+        return;
+    };
+    let raw = u32::from_le_bytes([len_bytes[0], len_bytes[1], len_bytes[2], len_bytes[3]]);
+    let unpack = (raw as u64) % (1024 * 1024 + 1);
+
+    drive(Decoder::with_unpack_size(unpack), payload);
+});

fuzz/fuzz_targets/decoder_rar3.rs

@@ -0,0 +1,120 @@
+#![no_main]
+use compcol::Decoder as _;
+use compcol::rar3::Decoder;
+use libfuzzer_sys::fuzz_target;
+
+// Smoke property: the decoder must not panic on arbitrary input.
+// libfuzzer feeds us garbage bytes; we drive the decoder over them
+// and discard the result. Any panic, abort, or undefined behavior
+// trips the harness.
+//
+// RAR3 streams don't carry their uncompressed length in-band — it
+// lives in the archive container's file header — so we read a 5-byte
+// prefix: 4 LE bytes for the unpack size (capped to 1 MiB so a hostile
+// size field can't make the harness itself allocate unbounded output)
+// and 1 flag byte:
+//   bit 0: enable the standalone E8/E9 post-pass filter
+//   bit 1: ... also translating E9 jumps
+//   bit 2: solid-group mode — the payload becomes up to 4 members, each
+//          introduced by a 5-byte header (2-byte LE chunk length + 3-byte
+//          LE unpack size, capped to 256 KiB — 1 MiB per group); the
+//          shared LZ window, tables, filter programs and PPMd model
+//          persist across them. The outer 4-byte unpack prefix is unused
+//          in this mode.
+
+/// Feed one member's payload and drain it. Returns false when the decoder
+/// errored (a fine outcome — the input is garbage; we only care that it
+/// never panics or loops).
+fn drive_member(dec: &mut Decoder, payload: &[u8]) -> bool {
+    let mut out = vec![0u8; 64 * 1024];
+    let mut consumed = 0;
+    let mut steps = 0;
+    while consumed < payload.len() {
+        match dec.decode(&payload[consumed..], &mut out) {
+            Ok((p, _)) => {
+                if p.consumed == 0 && p.written == 0 {
+                    break;
+                }
+                consumed += p.consumed;
+            }
+            Err(_) => return false,
+        }
+        steps += 1;
+        if steps > 4096 {
+            // Defensive: pathological inputs shouldn't make us loop.
+            return false;
+        }
+    }
+    let mut steps = 0;
+    loop {
+        match dec.finish(&mut out) {
+            Ok((p, status)) => {
+                if matches!(status, compcol::Status::StreamEnd) {
+                    return true;
+                }
+                if p.written == 0 {
+                    return true;
+                }
+            }
+            Err(_) => return false,
+        }
+        steps += 1;
+        if steps > 4096 {
+            return false;
+        }
+    }
+}
+
+fn drive(mut dec: Decoder, payload: &[u8]) {
+    drive_member(&mut dec, payload);
+}
+
+fn drive_solid(mut payload: &[u8]) {
+    let mut dec: Option<Decoder> = None;
+    for _ in 0..4 {
+        if payload.len() < 5 {
+            return;
+        }
+        let (hdr, rest) = payload.split_at(5);
+        let want = u16::from_le_bytes([hdr[0], hdr[1]]) as usize;
+        let unpack = (u32::from_le_bytes([hdr[2], hdr[3], hdr[4], 0]) as u64) % (256 * 1024 + 1);
+        let (chunk, rest) = rest.split_at(want.min(rest.len()));
+        payload = rest;
+        let d = match dec.as_mut() {
+            None => dec.insert(Decoder::with_unpack_size(unpack).with_solid()),
+            Some(d) => {
+                if d.begin_solid_member(unpack).is_err() {
+                    return;
+                }
+                d
+            }
+        };
+        if !drive_member(d, chunk) {
+            return;
+        }
+    }
+}
+
+fuzz_target!(|data: &[u8]| {
+    let (prefix, payload) = if data.len() >= 5 {
+        data.split_at(5)
+    } else {
+        // Too short to carry a prefix: still fuzz the no-declared-size
+        // path with the whole input as payload.
+        drive(Decoder::new(), data);
+        return;
+    };
+    let raw = u32::from_le_bytes([prefix[0], prefix[1], prefix[2], prefix[3]]);
+    let unpack = (raw as u64) % (1024 * 1024 + 1);
+
+    if prefix[4] & 4 != 0 {
+        drive_solid(payload);
+        return;
+    }
+
+    let mut dec = Decoder::with_unpack_size(unpack);
+    if prefix[4] & 1 != 0 {
+        dec = dec.with_e8_filter(prefix[4] & 2 != 0);
+    }
+    drive(dec, payload);
+});

fuzz/fuzz_targets/decoder_rar5.rs

@@ -0,0 +1,70 @@
+#![no_main]
+use compcol::Decoder as _;
+use compcol::rar5::Decoder;
+use libfuzzer_sys::fuzz_target;
+
+// Smoke property: the decoder must not panic on arbitrary input.
+// libfuzzer feeds us garbage bytes; we drive the decoder over them
+// and discard the result. Any panic, abort, or undefined behavior
+// trips the harness.
+//
+// RAR5 streams don't carry the unpack size or window size in-band —
+// both live in the archive container's file header, so the caller
+// supplies them out of band. We read a 5-byte prefix: 4 LE bytes for
+// the unpack size (capped to 1 MiB so a hostile size field can't make
+// the harness itself allocate unbounded output) and 1 byte selecting
+// the window size (128 KiB << 0..=5, i.e. capped at 4 MiB so the
+// fuzzer explores window wrap-around without gigabyte allocations).
+fn drive(mut dec: Decoder, payload: &[u8]) {
+    let mut out = vec![0u8; 64 * 1024];
+    let mut consumed = 0;
+    let mut steps = 0;
+    while consumed < payload.len() {
+        match dec.decode(&payload[consumed..], &mut out) {
+            Ok((p, _)) => {
+                if p.consumed == 0 && p.written == 0 {
+                    break;
+                }
+                consumed += p.consumed;
+            }
+            Err(_) => return,
+        }
+        steps += 1;
+        if steps > 4096 {
+            // Defensive: pathological inputs shouldn't make us loop.
+            return;
+        }
+    }
+    let mut steps = 0;
+    while let Ok((p, status)) = dec.finish(&mut out) {
+        if matches!(status, compcol::Status::StreamEnd) {
+            return;
+        }
+        if p.written == 0 {
+            return;
+        }
+        steps += 1;
+        if steps > 4096 {
+            return;
+        }
+    }
+}
+
+fuzz_target!(|data: &[u8]| {
+    let (prefix, payload) = if data.len() >= 5 {
+        data.split_at(5)
+    } else {
+        // Too short to carry a prefix: still fuzz the default-window,
+        // no-declared-size path with the whole input as payload.
+        drive(Decoder::new(), data);
+        return;
+    };
+    let raw = u32::from_le_bytes([prefix[0], prefix[1], prefix[2], prefix[3]]);
+    let unpack = (raw as u64) % (1024 * 1024 + 1);
+    let window = 0x20000usize << (prefix[4] % 6); // 128 KiB ..= 4 MiB
+
+    drive(
+        Decoder::with_unpack_size_and_window(unpack, window),
+        payload,
+    );
+});

src/checksum.rs

@@ -57,13 +57,13 @@ impl Default for Adler32 {
 
 /// IEEE / gzip CRC-32. Polynomial `0xEDB88320` (reflected), initial value
 /// `0xFFFFFFFF`, final XOR `0xFFFFFFFF`.
-#[cfg(any(feature = "gzip", test))]
+#[cfg(any(feature = "gzip", feature = "rar3", test))]
 #[derive(Debug, Clone, Copy)]
 pub struct Crc32 {
     state: u32,
 }
 
-#[cfg(any(feature = "gzip", test))]
+#[cfg(any(feature = "gzip", feature = "rar3", test))]
 impl Crc32 {
     pub const fn new() -> Self {
         Self { state: 0xFFFF_FFFF }
@@ -102,12 +102,15 @@ impl Crc32 {
         self.state ^ 0xFFFF_FFFF
     }
 
+    /// Only the gzip codec re-arms a CRC mid-stream; rar3's filter
+    /// recognition uses one-shot instances.
+    #[cfg(any(feature = "gzip", test))]
     pub fn reset(&mut self) {
         *self = Self::new();
     }
 }
 
-#[cfg(any(feature = "gzip", test))]
+#[cfg(any(feature = "gzip", feature = "rar3", test))]
 impl Default for Crc32 {
     fn default() -> Self {
         Self::new()
@@ -118,7 +121,7 @@ impl Default for Crc32 {
 /// standard 256-entry CRC-32 table; `CRC32_TABLE8[n]` for `n >= 1` advances
 /// the CRC by an extra byte position, so eight bytes can be folded per
 /// iteration. See Intel's "Slicing-by-8" technique.
-#[cfg(any(feature = "gzip", test))]
+#[cfg(any(feature = "gzip", feature = "rar3", test))]
 const CRC32_TABLE8: [[u32; 256]; 8] = {
     let mut tables = [[0u32; 256]; 8];
 

src/lib.rs

@@ -47,7 +47,7 @@ pub mod tokio_io;
 // `lz4`) doesn't pull them in via `cfg(test)`.
 #[cfg(any(feature = "deflate", feature = "deflate64"))]
 mod bits;
-#[cfg(any(feature = "zlib", feature = "gzip"))]
+#[cfg(any(feature = "zlib", feature = "gzip", feature = "rar3"))]
 mod checksum;
 #[cfg(any(feature = "deflate", feature = "deflate64"))]
 mod huffman;
@@ -171,6 +171,12 @@ pub mod rar3;
 #[cfg(feature = "rar5")]
 pub mod rar5;
 
+// Standard post-decompression filter transforms (x86 E8/E8E9, Delta) shared
+// by the rar3 and rar5 decoders — the two container generations declare
+// filters differently but run the same byte transforms.
+#[cfg(any(feature = "rar3", feature = "rar5"))]
+pub(crate) mod rar_filters;
+
 #[cfg(feature = "zip_reduce")]
 pub mod zip_reduce;
 #[cfg(feature = "zip_shrink")]