diff --git a/Cargo.toml b/Cargo.toml index 6498c98..a73d65a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -196,7 +196,9 @@ arsenic = ["alloc"] # the encoders permanently return `Error::Unsupported`. rar1 = ["alloc"] rar2 = ["alloc"] -rar3 = ["alloc"] +# RAR3/4 v29: LZ77+Huffman, in-band standard filters, and PPMd-II var H +# blocks (the latter via the shared `ppmd` model core). +rar3 = ["alloc", "ppmd"] rar5 = ["alloc"] # ZIP method 1 (Shrink): PKZIP 1.x dynamic LZW with partial-clear marker. # Decoder-only — the encoder permanently returns `Error::Unsupported`. diff --git a/README.md b/README.md index 6fee379..87d2fb1 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,8 @@ flag, and a `compcol` binary turns the library into a Unix-style filter. | StuffIt 5 Arsenic (method 15) | `arsenic` | `.sit` | `Unsupported` (decode-only) | full (range coder + inverse BWT + MTF/RLE + de-randomization) | **real StuffIt 5 fixtures (in-stream CRC-32 + SHA vs `unar`)** | | RAR 1.x | `rar1` | `.rar` | `Unsupported` (license) | building blocks only (Huffman tables not license-clean) | — | | RAR 2.x | `rar2` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + audio predictor | real rar-2.60 fixtures | -| RAR 3.x | `rar3` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + E8 filter; PPMd & VM filters refused | libarchive RAR3 fixtures | -| RAR 5.x | `rar5` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + x86 filter; Delta/ARM refused | RARLAB-CLI fixtures | +| RAR 3.x | `rar3` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + PPMd-II variant H + standard filters (Delta, x86 E8/E8E9), incl. solid groups; non-standard VM programs refused | libarchive RAR3 fixtures + **real rar-6.24 archives (differential vs UnRAR 7.23)** | +| RAR 5.x | `rar5` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + Delta/x86 filters (incl. solid groups); ARM refused | RARLAB-CLI fixtures + **real WinRAR 7.23 archives (differential vs UnRAR 7.23)** | | HTTP/2 HPACK (RFC 7541) | `hpack` | — | full (header codec + `h2-huffman` string codec) | full (static+dynamic tables, integer/string coding) | RFC 7541 Appendix C vectors | | HTTP/3 QPACK (RFC 9204) | `qpack` | — | full (static + dynamic-table encoder driving the encoder stream; eviction-safe) | full (static+dynamic tables via encoder stream, all field representations) | RFC 9204 Appendix B vectors | | Canonical Huffman (standalone) | `huffman` | `.huff` | full (length-limited, self-delimiting) | full | own round-trip | diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0c1d098..d509353 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -275,3 +275,24 @@ path = "fuzz_targets/roundtrip.rs" test = false doc = false bench = false + +[[bin]] +name = "decoder_rar2" +path = "fuzz_targets/decoder_rar2.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "decoder_rar3" +path = "fuzz_targets/decoder_rar3.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "decoder_rar5" +path = "fuzz_targets/decoder_rar5.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/decoder_rar2.rs b/fuzz/fuzz_targets/decoder_rar2.rs new file mode 100644 index 0000000..c6af656 --- /dev/null +++ b/fuzz/fuzz_targets/decoder_rar2.rs @@ -0,0 +1,64 @@ +#![no_main] +use compcol::Decoder as _; +use compcol::rar2::Decoder; +use libfuzzer_sys::fuzz_target; + +// Smoke property: the decoder must not panic on arbitrary input. +// libfuzzer feeds us garbage bytes; we drive the decoder over them +// and discard the result. Any panic, abort, or undefined behavior +// trips the harness. +// +// RAR2 streams don't self-delimit — the decompressed length lives in +// the archive container's file header — so we read a 4-byte LE length +// prefix (capped to 1 MiB so a hostile size field can't make the +// harness itself allocate unbounded output). `decode` only buffers; +// the real decompression runs on the first `finish` call. +fn drive(mut dec: Decoder, payload: &[u8]) { + let mut out = vec![0u8; 64 * 1024]; + let mut consumed = 0; + let mut steps = 0; + while consumed < payload.len() { + match dec.decode(&payload[consumed..], &mut out) { + Ok((p, _)) => { + if p.consumed == 0 && p.written == 0 { + break; + } + consumed += p.consumed; + } + Err(_) => return, + } + steps += 1; + if steps > 4096 { + // Defensive: pathological inputs shouldn't make us loop. + return; + } + } + let mut steps = 0; + while let Ok((p, status)) = dec.finish(&mut out) { + if matches!(status, compcol::Status::StreamEnd) { + return; + } + if p.written == 0 { + return; + } + steps += 1; + if steps > 4096 { + return; + } + } +} + +fuzz_target!(|data: &[u8]| { + let (len_bytes, payload) = if data.len() >= 4 { + data.split_at(4) + } else { + // Too short to carry a length prefix: still fuzz the + // no-declared-size path with the whole input as payload. + drive(Decoder::new(), data); + return; + }; + let raw = u32::from_le_bytes([len_bytes[0], len_bytes[1], len_bytes[2], len_bytes[3]]); + let unpack = (raw as u64) % (1024 * 1024 + 1); + + drive(Decoder::with_unpack_size(unpack), payload); +}); diff --git a/fuzz/fuzz_targets/decoder_rar3.rs b/fuzz/fuzz_targets/decoder_rar3.rs new file mode 100644 index 0000000..6c7a5b4 --- /dev/null +++ b/fuzz/fuzz_targets/decoder_rar3.rs @@ -0,0 +1,120 @@ +#![no_main] +use compcol::Decoder as _; +use compcol::rar3::Decoder; +use libfuzzer_sys::fuzz_target; + +// Smoke property: the decoder must not panic on arbitrary input. +// libfuzzer feeds us garbage bytes; we drive the decoder over them +// and discard the result. Any panic, abort, or undefined behavior +// trips the harness. +// +// RAR3 streams don't carry their uncompressed length in-band — it +// lives in the archive container's file header — so we read a 5-byte +// prefix: 4 LE bytes for the unpack size (capped to 1 MiB so a hostile +// size field can't make the harness itself allocate unbounded output) +// and 1 flag byte: +// bit 0: enable the standalone E8/E9 post-pass filter +// bit 1: ... also translating E9 jumps +// bit 2: solid-group mode — the payload becomes up to 4 members, each +// introduced by a 5-byte header (2-byte LE chunk length + 3-byte +// LE unpack size, capped to 256 KiB — 1 MiB per group); the +// shared LZ window, tables, filter programs and PPMd model +// persist across them. The outer 4-byte unpack prefix is unused +// in this mode. + +/// Feed one member's payload and drain it. Returns false when the decoder +/// errored (a fine outcome — the input is garbage; we only care that it +/// never panics or loops). +fn drive_member(dec: &mut Decoder, payload: &[u8]) -> bool { + let mut out = vec![0u8; 64 * 1024]; + let mut consumed = 0; + let mut steps = 0; + while consumed < payload.len() { + match dec.decode(&payload[consumed..], &mut out) { + Ok((p, _)) => { + if p.consumed == 0 && p.written == 0 { + break; + } + consumed += p.consumed; + } + Err(_) => return false, + } + steps += 1; + if steps > 4096 { + // Defensive: pathological inputs shouldn't make us loop. + return false; + } + } + let mut steps = 0; + loop { + match dec.finish(&mut out) { + Ok((p, status)) => { + if matches!(status, compcol::Status::StreamEnd) { + return true; + } + if p.written == 0 { + return true; + } + } + Err(_) => return false, + } + steps += 1; + if steps > 4096 { + return false; + } + } +} + +fn drive(mut dec: Decoder, payload: &[u8]) { + drive_member(&mut dec, payload); +} + +fn drive_solid(mut payload: &[u8]) { + let mut dec: Option = None; + for _ in 0..4 { + if payload.len() < 5 { + return; + } + let (hdr, rest) = payload.split_at(5); + let want = u16::from_le_bytes([hdr[0], hdr[1]]) as usize; + let unpack = (u32::from_le_bytes([hdr[2], hdr[3], hdr[4], 0]) as u64) % (256 * 1024 + 1); + let (chunk, rest) = rest.split_at(want.min(rest.len())); + payload = rest; + let d = match dec.as_mut() { + None => dec.insert(Decoder::with_unpack_size(unpack).with_solid()), + Some(d) => { + if d.begin_solid_member(unpack).is_err() { + return; + } + d + } + }; + if !drive_member(d, chunk) { + return; + } + } +} + +fuzz_target!(|data: &[u8]| { + let (prefix, payload) = if data.len() >= 5 { + data.split_at(5) + } else { + // Too short to carry a prefix: still fuzz the no-declared-size + // path with the whole input as payload. + drive(Decoder::new(), data); + return; + }; + let raw = u32::from_le_bytes([prefix[0], prefix[1], prefix[2], prefix[3]]); + let unpack = (raw as u64) % (1024 * 1024 + 1); + + if prefix[4] & 4 != 0 { + drive_solid(payload); + return; + } + + let mut dec = Decoder::with_unpack_size(unpack); + if prefix[4] & 1 != 0 { + dec = dec.with_e8_filter(prefix[4] & 2 != 0); + } + drive(dec, payload); +}); diff --git a/fuzz/fuzz_targets/decoder_rar5.rs b/fuzz/fuzz_targets/decoder_rar5.rs new file mode 100644 index 0000000..1b5fd65 --- /dev/null +++ b/fuzz/fuzz_targets/decoder_rar5.rs @@ -0,0 +1,70 @@ +#![no_main] +use compcol::Decoder as _; +use compcol::rar5::Decoder; +use libfuzzer_sys::fuzz_target; + +// Smoke property: the decoder must not panic on arbitrary input. +// libfuzzer feeds us garbage bytes; we drive the decoder over them +// and discard the result. Any panic, abort, or undefined behavior +// trips the harness. +// +// RAR5 streams don't carry the unpack size or window size in-band — +// both live in the archive container's file header, so the caller +// supplies them out of band. We read a 5-byte prefix: 4 LE bytes for +// the unpack size (capped to 1 MiB so a hostile size field can't make +// the harness itself allocate unbounded output) and 1 byte selecting +// the window size (128 KiB << 0..=5, i.e. capped at 4 MiB so the +// fuzzer explores window wrap-around without gigabyte allocations). +fn drive(mut dec: Decoder, payload: &[u8]) { + let mut out = vec![0u8; 64 * 1024]; + let mut consumed = 0; + let mut steps = 0; + while consumed < payload.len() { + match dec.decode(&payload[consumed..], &mut out) { + Ok((p, _)) => { + if p.consumed == 0 && p.written == 0 { + break; + } + consumed += p.consumed; + } + Err(_) => return, + } + steps += 1; + if steps > 4096 { + // Defensive: pathological inputs shouldn't make us loop. + return; + } + } + let mut steps = 0; + while let Ok((p, status)) = dec.finish(&mut out) { + if matches!(status, compcol::Status::StreamEnd) { + return; + } + if p.written == 0 { + return; + } + steps += 1; + if steps > 4096 { + return; + } + } +} + +fuzz_target!(|data: &[u8]| { + let (prefix, payload) = if data.len() >= 5 { + data.split_at(5) + } else { + // Too short to carry a prefix: still fuzz the default-window, + // no-declared-size path with the whole input as payload. + drive(Decoder::new(), data); + return; + }; + let raw = u32::from_le_bytes([prefix[0], prefix[1], prefix[2], prefix[3]]); + let unpack = (raw as u64) % (1024 * 1024 + 1); + let window = 0x20000usize << (prefix[4] % 6); // 128 KiB ..= 4 MiB + + drive( + Decoder::with_unpack_size_and_window(unpack, window), + payload, + ); +}); diff --git a/src/checksum.rs b/src/checksum.rs index f232133..d74321c 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -57,13 +57,13 @@ impl Default for Adler32 { /// IEEE / gzip CRC-32. Polynomial `0xEDB88320` (reflected), initial value /// `0xFFFFFFFF`, final XOR `0xFFFFFFFF`. -#[cfg(any(feature = "gzip", test))] +#[cfg(any(feature = "gzip", feature = "rar3", test))] #[derive(Debug, Clone, Copy)] pub struct Crc32 { state: u32, } -#[cfg(any(feature = "gzip", test))] +#[cfg(any(feature = "gzip", feature = "rar3", test))] impl Crc32 { pub const fn new() -> Self { Self { state: 0xFFFF_FFFF } @@ -102,12 +102,15 @@ impl Crc32 { self.state ^ 0xFFFF_FFFF } + /// Only the gzip codec re-arms a CRC mid-stream; rar3's filter + /// recognition uses one-shot instances. + #[cfg(any(feature = "gzip", test))] pub fn reset(&mut self) { *self = Self::new(); } } -#[cfg(any(feature = "gzip", test))] +#[cfg(any(feature = "gzip", feature = "rar3", test))] impl Default for Crc32 { fn default() -> Self { Self::new() @@ -118,7 +121,7 @@ impl Default for Crc32 { /// standard 256-entry CRC-32 table; `CRC32_TABLE8[n]` for `n >= 1` advances /// the CRC by an extra byte position, so eight bytes can be folded per /// iteration. See Intel's "Slicing-by-8" technique. -#[cfg(any(feature = "gzip", test))] +#[cfg(any(feature = "gzip", feature = "rar3", test))] const CRC32_TABLE8: [[u32; 256]; 8] = { let mut tables = [[0u32; 256]; 8]; diff --git a/src/lib.rs b/src/lib.rs index 8c3aa0c..e918f45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,7 +47,7 @@ pub mod tokio_io; // `lz4`) doesn't pull them in via `cfg(test)`. #[cfg(any(feature = "deflate", feature = "deflate64"))] mod bits; -#[cfg(any(feature = "zlib", feature = "gzip"))] +#[cfg(any(feature = "zlib", feature = "gzip", feature = "rar3"))] mod checksum; #[cfg(any(feature = "deflate", feature = "deflate64"))] mod huffman; @@ -171,6 +171,12 @@ pub mod rar3; #[cfg(feature = "rar5")] pub mod rar5; +// Standard post-decompression filter transforms (x86 E8/E8E9, Delta) shared +// by the rar3 and rar5 decoders — the two container generations declare +// filters differently but run the same byte transforms. +#[cfg(any(feature = "rar3", feature = "rar5"))] +pub(crate) mod rar_filters; + #[cfg(feature = "zip_reduce")] pub mod zip_reduce; #[cfg(feature = "zip_shrink")] diff --git a/src/ppmd/arena.rs b/src/ppmd/arena.rs deleted file mode 100644 index aa6c30d..0000000 --- a/src/ppmd/arena.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! Byte-level arena backing the PPMd context and state allocator. -//! -//! The reference Ppmd7 implementation places every `CPpmd7_Context` and -//! `CPpmd_State` into a single `Byte *Base` buffer and refers to them by -//! 32-bit offset (`CPpmd_*_Ref` is a `UInt32`). That packing is -//! load-bearing: the free-list pointer and the `Stats` ref are read out -//! of the same 4-byte slot a struct field would occupy, the `Node` -//! coalescer walks the buffer linearly assuming `UNIT_SIZE` granularity, -//! etc. We preserve the layout exactly in a single `Vec` and access -//! it through safe little-endian byte gets/puts. -//! -//! All field accessors are little-endian. The C code relies on the host -//! being little-endian (PPMd has no defined behaviour on big-endian -//! archive readers); we make that explicit so the codec round-trips on -//! any architecture. - -use alloc::vec; -use alloc::vec::Vec; - -// ─── packed layout constants ──────────────────────────────────────────── - -pub(super) const UNIT_SIZE: usize = 12; -pub(super) const STATE_SIZE: usize = 6; - -// CPpmd7_Context (packed, total = UNIT_SIZE = 12 bytes) -pub(super) const CTX_OFF_NUM_STATS: usize = 0; // u16 -pub(super) const CTX_OFF_SUMM_FREQ: usize = 2; // u16 -pub(super) const CTX_OFF_STATS: usize = 4; // u32 — CPpmd_State_Ref -pub(super) const CTX_OFF_SUFFIX: usize = 8; // u32 — CPpmd7_Context_Ref - -// CPpmd_State (packed, total = STATE_SIZE = 6 bytes) -pub(super) const STATE_OFF_SYMBOL: usize = 0; // u8 -pub(super) const STATE_OFF_FREQ: usize = 1; // u8 -pub(super) const STATE_OFF_SUCC_LOW: usize = 2; // u16 -pub(super) const STATE_OFF_SUCC_HIGH: usize = 4; // u16 - -// ─── arena ─────────────────────────────────────────────────────────────── - -/// `Vec` arena with little-endian struct accessors. Out-of-range -/// reads return `None` so the model can refuse a corrupt stream rather -/// than panic. -pub(super) struct Arena { - data: Vec, -} - -impl Arena { - pub fn new(size: usize) -> Self { - Self { - data: vec![0u8; size], - } - } - - pub fn clear(&mut self) { - self.data.fill(0); - } - - // ── byte-level r/w (sole entry to the backing store) ─────────────── - - #[inline] - pub fn read_u8(&self, off: u32) -> Option { - self.data.get(off as usize).copied() - } - - #[inline] - pub fn write_u8(&mut self, off: u32, v: u8) -> Option<()> { - *self.data.get_mut(off as usize)? = v; - Some(()) - } - - #[inline] - pub fn read_u16(&self, off: u32) -> Option { - let s = self.data.get(off as usize..off as usize + 2)?; - Some(u16::from_le_bytes([s[0], s[1]])) - } - - #[inline] - pub fn write_u16(&mut self, off: u32, v: u16) -> Option<()> { - let bytes = v.to_le_bytes(); - let slice = self.data.get_mut(off as usize..off as usize + 2)?; - slice.copy_from_slice(&bytes); - Some(()) - } - - #[inline] - pub fn write_u32(&mut self, off: u32, v: u32) -> Option<()> { - let bytes = v.to_le_bytes(); - let slice = self.data.get_mut(off as usize..off as usize + 4)?; - slice.copy_from_slice(&bytes); - Some(()) - } -} - -// ─── context accessors ───────────────────────────────────────────────── - -#[inline] -pub(super) fn ctx_num_stats(a: &Arena, ctx: u32) -> Option { - a.read_u16(ctx + CTX_OFF_NUM_STATS as u32) -} -#[inline] -pub(super) fn ctx_set_num_stats(a: &mut Arena, ctx: u32, v: u16) -> Option<()> { - a.write_u16(ctx + CTX_OFF_NUM_STATS as u32, v) -} -#[inline] -pub(super) fn ctx_summ_freq(a: &Arena, ctx: u32) -> Option { - a.read_u16(ctx + CTX_OFF_SUMM_FREQ as u32) -} -#[inline] -pub(super) fn ctx_set_summ_freq(a: &mut Arena, ctx: u32, v: u16) -> Option<()> { - a.write_u16(ctx + CTX_OFF_SUMM_FREQ as u32, v) -} -#[inline] -pub(super) fn ctx_set_stats(a: &mut Arena, ctx: u32, v: u32) -> Option<()> { - a.write_u32(ctx + CTX_OFF_STATS as u32, v) -} -#[inline] -pub(super) fn ctx_set_suffix(a: &mut Arena, ctx: u32, v: u32) -> Option<()> { - a.write_u32(ctx + CTX_OFF_SUFFIX as u32, v) -} - -// ─── state accessors ─────────────────────────────────────────────────── - -#[inline] -pub(super) fn state_symbol(a: &Arena, st: u32) -> Option { - a.read_u8(st + STATE_OFF_SYMBOL as u32) -} -#[inline] -pub(super) fn state_set_symbol(a: &mut Arena, st: u32, v: u8) -> Option<()> { - a.write_u8(st + STATE_OFF_SYMBOL as u32, v) -} -#[inline] -pub(super) fn state_freq(a: &Arena, st: u32) -> Option { - a.read_u8(st + STATE_OFF_FREQ as u32) -} -#[inline] -pub(super) fn state_set_freq(a: &mut Arena, st: u32, v: u8) -> Option<()> { - a.write_u8(st + STATE_OFF_FREQ as u32, v) -} -#[inline] -pub(super) fn state_set_successor(a: &mut Arena, st: u32, v: u32) -> Option<()> { - a.write_u16(st + STATE_OFF_SUCC_LOW as u32, (v & 0xFFFF) as u16)?; - a.write_u16(st + STATE_OFF_SUCC_HIGH as u32, ((v >> 16) & 0xFFFF) as u16) -} - -/// Swap two adjacent (or any two) states. -pub(super) fn swap_states(a: &mut Arena, st1: u32, st2: u32) -> Option<()> { - let mut s1 = [0u8; STATE_SIZE]; - let mut s2 = [0u8; STATE_SIZE]; - for i in 0..STATE_SIZE { - s1[i] = a.read_u8(st1 + i as u32)?; - s2[i] = a.read_u8(st2 + i as u32)?; - } - for i in 0..STATE_SIZE { - a.write_u8(st1 + i as u32, s2[i])?; - a.write_u8(st2 + i as u32, s1[i])?; - } - Some(()) -} - -/// Write a full state from a tuple. -#[inline] -pub(super) fn state_store(a: &mut Arena, st: u32, sym: u8, freq: u8, succ: u32) -> Option<()> { - state_set_symbol(a, st, sym)?; - state_set_freq(a, st, freq)?; - state_set_successor(a, st, succ) -} diff --git a/src/ppmd/decoder.rs b/src/ppmd/decoder.rs index 7998082..52f28f1 100644 --- a/src/ppmd/decoder.rs +++ b/src/ppmd/decoder.rs @@ -1,22 +1,14 @@ -//! PPMd streaming decoder. +//! PPMd streaming decoder (standalone `.ppmd` framing). //! -//! The decoder is a small state machine wrapped around the order-0 -//! PPMII model in `model.rs` and the carry-less range decoder in -//! `range_dec.rs`: +//! PPMd streams carry no in-band end marker, so like the RAR3 decoder this +//! one is *buffer-then-drain*: [`raw_decode`] absorbs input, and the actual +//! model decode runs once [`raw_finish`] is called and the whole payload is +//! available. The decoded bytes are then drained to the caller across as +//! many `finish` calls as the output buffer size requires. //! -//! 1. **Header** (11 bytes): order, mem_size_mb, restoration_method, -//! little-endian u64 uncompressed length. See the module docs for the -//! framing layout. -//! 2. **RangeInit** (5 bytes): the first 5 bytes of the payload feed -//! `RangeDec::init`. The first byte must be `0x00`. -//! 3. **Decode**: pull symbols one at a time until either the -//! uncompressed length is reached (when known) or the range decoder -//! detects end-of-stream (`is_finished_ok` after the last symbol). -//! -//! Streaming uses a snapshot/restore pattern: before every symbol decode -//! we save the range coder state and the input position so that if the -//! decode tries to read past the buffered input we can rewind and ask -//! the caller for more bytes — same pattern as the bzip2 decoder. +//! See the module docs (`super`) for the 11-byte framing header. The model +//! is the full PPMII variant H core in [`super::ppmd7`], driven by the 7z +//! range decoder. extern crate alloc; use alloc::vec::Vec; @@ -24,184 +16,126 @@ use alloc::vec::Vec; use crate::error::Error; use crate::traits::{RawDecoder, RawProgress}; -use super::model::Model; -use super::range_dec::{ByteSource, RangeDec}; +use super::ppmd7::Ppmd7; +use super::range_dec::{Mode, RangeDec}; -/// Lengths of the framing components. const HEADER_LEN: usize = 11; -const RANGE_INIT_LEN: usize = 5; -/// Sentinel "unknown length" value (matches the `lzma` alone-format -/// convention). +/// Sentinel length meaning "unknown". PPMd has no in-band end-of-stream +/// marker, so a stream framed with this length can't be decoded reliably +/// (see [`Decoder::run_decode`]) — it's refused rather than decoded to a +/// guess. const UNKNOWN_LEN: u64 = u64::MAX; - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { - Header, - RangeInit, - Body, - Done, -} +/// Hard cap on decoded output, so a tiny crafted stream declaring a huge +/// length can't drive unbounded allocation/work. +const MAX_OUTPUT: usize = 64 * 1024 * 1024; pub struct Decoder { in_buf: Vec, - in_committed: usize, - decoded: Vec, decoded_idx: usize, + started: bool, + header_checked: bool, + finished_decode: bool, + /// Set on the first irrecoverable error; every later call re-reports + /// the same error (so an early header rejection in `decode` reads the + /// same from a follow-up `finish`). + poisoned: Option, +} - phase: Phase, - poisoned: bool, - - // Header fields (populated after Phase::Header). - order: u32, - mem_mb: u32, - restoration: u8, - expected_len: u64, - produced_len: u64, - - model: Option, - range_dec: RangeDec, +/// Validate the 11-byte framing header, returning the declared unpacked +/// length. Called from `raw_decode` as soon as 11 bytes have been buffered +/// (so a hostile stream with a bad header is rejected immediately instead +/// of after buffering its whole payload) and again from `run_decode`. +fn validate_header(h: &[u8]) -> Result { + let order = h[0] as u32; + let mem_mb = h[1] as u32; + let restoration = h[2]; + if !(2..=64).contains(&order) { + return Err(Error::BadHeader); + } + if !(1..=255).contains(&mem_mb) { + return Err(Error::BadHeader); + } + if restoration > 2 { + return Err(Error::BadHeader); + } + let expected_len = u64::from_le_bytes(h[3..11].try_into().unwrap()); + // PPMd carries no in-band end-of-stream marker, so a stream whose + // header declares an unknown length has no reliable terminal + // condition: after the true last symbol the range coder's finalisation + // bytes keep decoding into extra (garbage) symbols, and exhausting the + // physical input is not an end signal (a high-probability symbol + // decodes without consuming any input). Refuse rather than emit a + // guess. + if expected_len == UNKNOWN_LEN { + return Err(Error::Unsupported); + } + // A declared length larger than the buffer-then-decode ceiling can't + // be produced here; reject it up front rather than growing the output + // toward OOM. + if expected_len > MAX_OUTPUT as u64 { + return Err(Error::OutputLimitExceeded); + } + Ok(expected_len) } impl Decoder { pub fn new() -> Self { Self { in_buf: Vec::new(), - in_committed: 0, decoded: Vec::new(), decoded_idx: 0, - phase: Phase::Header, - poisoned: false, - order: 0, - mem_mb: 0, - restoration: 0, - expected_len: 0, - produced_len: 0, - model: None, - range_dec: RangeDec::new(), + started: false, + header_checked: false, + finished_decode: false, + poisoned: None, } } fn poison(&mut self, e: Error) -> Error { - self.poisoned = true; + self.poisoned = Some(e); e } - /// Try to advance the state machine using `in_buf[in_committed..]`. - fn step(&mut self) -> Result { - match self.phase { - Phase::Header => self.try_header(), - Phase::RangeInit => self.try_range_init(), - Phase::Body => self.try_body(), - Phase::Done => Ok(false), - } - } - - fn try_header(&mut self) -> Result { - if self.in_buf.len() < self.in_committed + HEADER_LEN { - return Ok(false); + /// Decode the whole payload into `self.decoded`. Called once, lazily. + fn run_decode(&mut self) -> Result<(), Error> { + if self.in_buf.len() < HEADER_LEN { + return Err(Error::UnexpectedEnd); } - let off = self.in_committed; - let h = &self.in_buf[off..off + HEADER_LEN]; + let h = &self.in_buf[..HEADER_LEN]; let order = h[0] as u32; let mem_mb = h[1] as u32; - let restoration = h[2]; - if !(2..=16).contains(&order) { - return Err(self.poison(Error::BadHeader)); - } - if !(1..=255).contains(&mem_mb) { - return Err(self.poison(Error::BadHeader)); - } - if restoration > 2 { - return Err(self.poison(Error::BadHeader)); - } - let len = u64::from_le_bytes(h[3..11].try_into().unwrap()); + let expected_len = validate_header(h)?; - self.order = order; - self.mem_mb = mem_mb; - self.restoration = restoration; - self.expected_len = len; - self.in_committed += HEADER_LEN; + let mem_bytes = mem_mb.saturating_mul(1024 * 1024); + let mut model = Ppmd7::new(mem_bytes)?; + model.init(order); - let mem_bytes = (mem_mb as usize).saturating_mul(1024 * 1024); - self.model = Some(Model::new(order, mem_bytes).map_err(|e| self.poison(e))?); - self.phase = Phase::RangeInit; - Ok(true) - } + let (mut rc, consumed) = RangeDec::init(Mode::SevenZip, &self.in_buf, HEADER_LEN)?; + let _ = consumed; - fn try_range_init(&mut self) -> Result { - if self.in_buf.len() < self.in_committed + RANGE_INIT_LEN { - return Ok(false); - } - // Tell `range_dec.init` where the next byte lives. - self.range_dec.pos = self.in_committed; - match self.range_dec.init(&self.in_buf) { - Ok(true) => { - self.in_committed = self.range_dec.pos; - self.phase = Phase::Body; - Ok(true) - } - Ok(false) => Ok(false), // shouldn't happen — we checked length - Err(e) => Err(self.poison(e)), - } - } + let mut out = Vec::with_capacity((expected_len as usize).min(1 << 20)); - fn try_body(&mut self) -> Result { - let model = match self.model.as_mut() { - Some(m) => m, - None => return Err(self.poison(Error::Corrupt)), - }; - - // If we know the uncompressed length and have produced it all, - // verify the range coder's terminal state and finish. - if self.expected_len != UNKNOWN_LEN && self.produced_len >= self.expected_len { - if self.range_dec.is_finished_ok() { - self.phase = Phase::Done; - return Ok(true); + for _ in 0..expected_len { + // Truncated input can't supply more symbols; stop before the + // model starts decoding from zero-filled reads. + if rc.overran() { + return Err(Error::UnexpectedEnd); } - // The reference accepts a non-zero `code` if a peek confirms - // it. Our simplified order-0 model can't always finish - // exactly on `code == 0`, so we accept the implicit end-of- - // stream as long as no further symbols are requested. - self.phase = Phase::Done; - return Ok(true); + let sym = model.decode_symbol(&mut rc)?; + out.push(sym); } - - let mut src = ByteSource::new(&self.in_buf, self.range_dec.pos); - let mut progressed = false; - loop { - // If output buffer pressure will soon force a return, stop - // pulling symbols. We use a fixed budget so a giant input - // doesn't starve the caller. - if self.decoded.len() - self.decoded_idx > 4096 { - break; - } - // Snapshot for rollback. - let rd_pre = self.range_dec.clone(); - let pos_pre = src.pos; - // Decode one symbol. - match model.decode_symbol(&mut self.range_dec, &mut src) { - Ok(sym) => { - self.decoded.push(sym); - self.produced_len += 1; - progressed = true; - if self.expected_len != UNKNOWN_LEN && self.produced_len >= self.expected_len { - break; - } - } - Err(Error::UnexpectedEnd) => { - // Need more input — rewind and bail. - self.range_dec = rd_pre; - src.pos = pos_pre; - break; - } - Err(e) => return Err(self.poison(e)), - } + if rc.overran() { + return Err(Error::UnexpectedEnd); + } + // 7z streams leave the range coder at `code == 0` after the last + // symbol; a non-zero residue means truncation or corruption. + if !rc.is_finished_ok() { + return Err(Error::Corrupt); } - // Commit the input position. - self.range_dec.pos = src.pos; - self.in_committed = self.range_dec.pos; - Ok(progressed) + + self.decoded = out; + Ok(()) } fn drain(&mut self, output: &mut [u8], written: &mut usize) { @@ -219,6 +153,10 @@ impl Decoder { self.decoded_idx = 0; } } + + fn all_drained(&self) -> bool { + self.finished_decode && self.decoded_idx == self.decoded.len() + } } impl Default for Decoder { @@ -229,129 +167,57 @@ impl Default for Decoder { impl RawDecoder for Decoder { fn raw_decode(&mut self, input: &[u8], output: &mut [u8]) -> Result { - if self.poisoned { - return Err(Error::Corrupt); + if let Some(e) = self.poisoned { + return Err(e); + } + // Absorb input; real decoding is deferred to `finish`. The header + // is validated as soon as it is complete so a hostile stream fails + // after 11 bytes instead of after buffering its whole payload. + self.in_buf.extend_from_slice(input); + if !self.header_checked && self.in_buf.len() >= HEADER_LEN { + self.header_checked = true; + if let Err(e) = validate_header(&self.in_buf[..HEADER_LEN]) { + return Err(self.poison(e)); + } } - let mut consumed = 0usize; let mut written = 0usize; - - // Drain any already-decoded bytes first. - self.drain(output, &mut written); - - // If output is already full from a previous step's queued bytes, - // return without absorbing more input — the caller hasn't drained - // yet and absorbing would cause the bridge to misreport status. - if written == output.len() && self.decoded_idx < self.decoded.len() { - return Ok(RawProgress { - consumed, - written, - done: false, - }); - } - - loop { - // Quick exit if we've already produced everything and drained. - if matches!(self.phase, Phase::Done) && self.decoded_idx == self.decoded.len() { - return Ok(RawProgress { - consumed, - written, - done: true, - }); - } - - // Absorb caller's input into our buffer. - if consumed < input.len() { - self.in_buf.extend_from_slice(&input[consumed..]); - consumed = input.len(); - } - - let progressed = self.step()?; - - // Drain anything the step produced. + if self.finished_decode { self.drain(output, &mut written); - - // Bound `in_buf` growth by chopping off committed bytes when - // the prefix gets large. - if self.in_committed > 1 << 20 { - let off = self.in_committed; - self.in_buf.drain(..off); - self.in_committed = 0; - self.range_dec.pos = self.range_dec.pos.saturating_sub(off); - } - - if matches!(self.phase, Phase::Done) { - continue; - } - - // Output full + queued bytes → caller must drain. Report by - // *un-consuming* one byte of the absorbed input so the bridge - // sees `consumed < input.len()` and maps to `OutputFull`. The - // un-consumed byte is still buffered internally; we just - // delay acknowledging it until the caller comes back. - if written == output.len() && self.decoded_idx < self.decoded.len() { - // Report `consumed < input.len()` so the bridge maps to - // `OutputFull` rather than `InputEmpty`. The un-consumed - // byte is still buffered in `in_buf`; we just delay - // acknowledging it until the caller drains and returns. - consumed = consumed.saturating_sub(1); - return Ok(RawProgress { - consumed, - written, - done: false, - }); - } - - if !progressed { - // No progress and no more input → ask for more. - if consumed >= input.len() { - return Ok(RawProgress { - consumed, - written, - done: false, - }); - } - } - - if written == output.len() { - return Ok(RawProgress { - consumed, - written, - done: false, - }); - } } + Ok(RawProgress { + consumed: input.len(), + written, + done: self.all_drained(), + }) } fn raw_finish(&mut self, output: &mut [u8]) -> Result { - if self.poisoned { - return Err(Error::Corrupt); + if let Some(e) = self.poisoned { + return Err(e); } - let empty: [u8; 0] = []; - let p = self.raw_decode(&empty, output)?; - if matches!(self.phase, Phase::Done) && self.decoded_idx == self.decoded.len() { - Ok(RawProgress { - consumed: 0, - written: p.written, - done: true, - }) - } else { - Err(self.poison(Error::UnexpectedEnd)) + if !self.started { + self.started = true; + match self.run_decode() { + Ok(()) => self.finished_decode = true, + Err(e) => return Err(self.poison(e)), + } } + let mut written = 0usize; + self.drain(output, &mut written); + Ok(RawProgress { + consumed: 0, + written, + done: self.all_drained(), + }) } fn raw_reset(&mut self) { self.in_buf.clear(); - self.in_committed = 0; self.decoded.clear(); self.decoded_idx = 0; - self.phase = Phase::Header; - self.poisoned = false; - self.order = 0; - self.mem_mb = 0; - self.restoration = 0; - self.expected_len = 0; - self.produced_len = 0; - self.model = None; - self.range_dec = RangeDec::new(); + self.started = false; + self.header_checked = false; + self.finished_decode = false; + self.poisoned = None; } } diff --git a/src/ppmd/mod.rs b/src/ppmd/mod.rs index 4352c1e..9e7ce7d 100644 --- a/src/ppmd/mod.rs +++ b/src/ppmd/mod.rs @@ -9,31 +9,23 @@ //! //! ### What this build ships //! -//! This module ships the **framing layer** plus a working carry-less -//! 7z range decoder (see `range_dec.rs`) and the order-0/order-(-1) -//! base of the PPMII context tree. The full PPMII variant H model -//! (with the information-inheritance `CreateSuccessors`/`UpdateModel` -//! routines, masked-context escape handling, and SEE adaptation) is -//! large enough that completing it in one pass would have left the -//! codec in a half-finished, untested state — exactly the failure mode -//! the project guidance says to avoid. So: -//! -//! - **Decoder**: parses the 11-byte framing header (order, mem, -//! restoration method, uncompressed length), validates parameters, -//! and decodes the range-coded payload using the order-0 model. -//! This works for the *trivial subset* where every literal in the -//! payload was emitted from the model's order-(-1) escape path -//! (i.e. a stream whose body is essentially random-access raw -//! bytes encoded under the uniform-frequency seed model). Payloads -//! produced by real PPMd encoders, which traverse the full -//! information-inheritance tree, will land on the -//! [`Error::Unsupported`] tag once the decoder detects that the -//! uniform seed model has been escaped beyond — same gap pattern -//! as `lzfse`'s `bvx2` blocks. -//! - **Encoder**: permanently returns [`Error::Unsupported`]. The -//! PPM model maintenance plus carry-less range encoder were out of -//! scope; we follow the `lzfse`/`rar*` precedent and ship the -//! encoder as a stub. +//! - **Decoder**: the **full PPMII variant H model** (`ppmd7`) — the +//! information-inheritance context tree (`CreateSuccessors` / +//! `UpdateModel`), the binary-context fast path, the masked-escape +//! suffix walk, SEE (secondary escape estimation), tree-wide `Rescale`, +//! and the sub-allocator with block coalescing — driven by a carry-less +//! range decoder in both its 7z and RAR flavours (`range_dec`). The +//! standalone framing below uses the 7z flavour; the RAR3/4 decoder +//! feeds the same model core through the RAR flavour. Decodes streams +//! produced by real PPMd encoders (7-Zip, `pyppmd`, WinRAR/`rar`). One +//! known limitation: a stream that *exhausts the suballocator* (a +//! high-order model over a large, high-entropy payload in a small arena) +//! can desync at the memory-restart point and return [`Error::Corrupt`] +//! rather than decoding — it fails closed (never wrong bytes); see the +//! `glue_free_blocks` note in `ppmd7`. +//! - **Encoder**: permanently returns [`Error::Unsupported`]. The PPM +//! model maintenance plus carry-less range encoder are out of scope; we +//! follow the `lzfse`/`rar*` precedent and ship the encoder as a stub. //! //! ### Wire framing //! @@ -43,20 +35,23 @@ //! "alone" framing: //! //! ```text -//! byte 0 : order (2..=16, inclusive) -//! byte 1 : mem_size_mb (1..=256, inclusive) +//! byte 0 : order (2..=64, inclusive) +//! byte 1 : mem_size_mb (1..=255, inclusive) //! byte 2 : restoration_method (0=restart, 1=cut-off, 2=freeze) -//! bytes 3..=10: little-endian u64 uncompressed length -//! (0xFFFF_FFFF_FFFF_FFFF means "unknown — decode to -//! stream end") -//! bytes 11.. : the PPMd-coded payload +//! bytes 3..=10: little-endian u64 uncompressed length. PPMd has no +//! in-band end-of-stream marker, so the length is +//! mandatory: the sentinel 0xFFFF_FFFF_FFFF_FFFF +//! ("unknown") is refused, since decoding to physical +//! input exhaustion would append range-coder finalisation +//! garbage past the true end. +//! bytes 11.. : the PPMd-coded payload (a raw 7z Ppmd7 stream, i.e. +//! a leading 0x00 byte then the range-coded body) //! ``` //! -//! Only the `restart` restoration method is exercised by the range- -//! coded payload (the 7z PPMd model only ever calls `RestartModel` on -//! memory pressure). The byte is kept in the header so the framing -//! matches archive-wrapper conventions; values other than 0 are -//! accepted and ignored. +//! The order and memory size must match what the encoder used (they are +//! not otherwise recoverable from the stream). The restoration-method +//! byte is retained for archive-wrapper parity; the model restarts on +//! memory pressure regardless. //! //! ### References //! @@ -72,13 +67,18 @@ extern crate alloc; use crate::error::Error; use crate::traits::{Algorithm, RawEncoder, RawProgress}; -mod arena; mod decoder; -mod model; +mod ppmd7; mod range_dec; pub use decoder::Decoder; +// Re-exported for the RAR3/4 PPMd path; unused when `ppmd` is built alone. +#[cfg(feature = "rar3")] +pub(crate) use ppmd7::Ppmd7; +#[cfg(feature = "rar3")] +pub(crate) use range_dec::{Mode as RangeMode, RangeDec}; + /// Zero-sized marker type implementing [`Algorithm`] for PPMd. #[derive(Debug, Clone, Copy, Default)] pub struct Ppmd; diff --git a/src/ppmd/model.rs b/src/ppmd/model.rs deleted file mode 100644 index b1e3939..0000000 --- a/src/ppmd/model.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! PPMII variant H context model — *order-0 subset*. -//! -//! Safe-Rust port of the order-0 portion of Igor Pavlov's public-domain -//! `Ppmd7.{c,h}` (LZMA SDK). The reference packs every `CPpmd7_Context` -//! and `CPpmd_State` into a single `Byte *Base` arena and walks it -//! through 32-bit offsets; we keep that layout in `arena.rs` so the -//! pointer arithmetic translates straight across. -//! -//! ### Implementation scope -//! -//! This file implements **only the order-0 model state** — the 256-state -//! root context that PPMd uses as the bottom of its suffix chain. It -//! decodes any symbol the range coder's threshold lands in, updates the -//! frequency table (with the same `+4` increment + `MAX_FREQ` rescale), -//! and promotes the chosen state when its frequency overtakes its -//! predecessor. This is exactly the behaviour of a fresh model that has -//! never seen `UpdateModel` extend the tree, i.e. the moment immediately -//! after `Ppmd7_Init`. -//! -//! The full PPMII machinery — the per-order context tree built up by -//! `CreateSuccessors`, the binary-context special case for `NumStats==1` -//! contexts, the masked-escape walk through the suffix chain, the SEE -//! (secondary escape estimation) adaptation, and `Rescale` over the -//! entire tree — is **not** implemented in this build. Payloads from -//! real PPMd encoders exercise all of these paths almost immediately, -//! so this decoder is *not* a drop-in replacement for `7z x` or -//! `ppmd-cffi.decompress`. Tests use hand-built order-0 fixtures. - -extern crate alloc; - -use crate::error::Error; - -use super::arena::{ - Arena, STATE_SIZE, UNIT_SIZE, ctx_num_stats, ctx_set_num_stats, ctx_set_stats, ctx_set_suffix, - ctx_set_summ_freq, ctx_summ_freq, state_freq, state_set_freq, state_store, state_symbol, - swap_states, -}; -use super::range_dec::{ByteSource, RangeDec}; - -const MAX_FREQ: u8 = 124; - -/// Order-0 PPMII model. One 256-state context lives at offset -/// [`Model::CTX_OFFSET`] in the arena; states follow it. -pub(super) struct Model { - arena: Arena, - /// Offset of the order-0 context block (always zero in this build). - ctx_off: u32, - /// Offset of the state array (256 entries × `STATE_SIZE`). - stats_off: u32, -} - -impl Model { - /// Pre-computed arena layout. The order-0 context starts at byte 0 - /// (one UNIT = 12 bytes) and the state array starts at byte - /// `UNIT_SIZE`. With 256 states × 6 bytes = 1536 bytes of stats, the - /// minimum usable arena is `UNIT_SIZE + 256 * STATE_SIZE = 1548` - /// bytes. We round up to 2 KiB so the layout has some slack for any - /// future model-update extension. - const MIN_ARENA: usize = 2048; - - pub fn new(order: u32, mem_size_bytes: usize) -> Result { - if !(2..=16).contains(&order) { - return Err(Error::BadHeader); - } - // The header's advertised memory size (`mem_size_bytes`, up to - // 255 MiB) describes the arena the *full* PPMII tree would need. - // This build only implements the order-0 subset, whose arena never - // grows past root (`MIN_ARENA` = 2 KiB) — `arena.rs` has no growth - // path and every offset stays within that window. So we cap the - // eager `vec![0u8; size]` allocation to the actual working size - // rather than honouring the advertised figure, which would let an - // 11-byte header force a 255 MiB allocation (L9). The parsed header - // value is still validated and retained by the decoder; only the - // allocation size is capped here. - let _ = mem_size_bytes; - let size = Self::MIN_ARENA; - let _ = order; // captured in the framing header only; the order-0 - // subset doesn't grow the tree past root. - let mut m = Self { - arena: Arena::new(size), - ctx_off: 0, - stats_off: UNIT_SIZE as u32, - }; - m.restart()?; - Ok(m) - } - - /// Initialise the order-0 context with 256 equally-weighted states. - pub fn restart(&mut self) -> Result<(), Error> { - self.arena.clear(); - // Build context at offset 0. - ctx_set_num_stats(&mut self.arena, self.ctx_off, 256).ok_or(Error::Corrupt)?; - ctx_set_summ_freq(&mut self.arena, self.ctx_off, 256 + 1).ok_or(Error::Corrupt)?; - ctx_set_stats(&mut self.arena, self.ctx_off, self.stats_off).ok_or(Error::Corrupt)?; - ctx_set_suffix(&mut self.arena, self.ctx_off, 0).ok_or(Error::Corrupt)?; - for i in 0..256u32 { - let st = self.stats_off + i * STATE_SIZE as u32; - state_store(&mut self.arena, st, i as u8, 1, 0).ok_or(Error::Corrupt)?; - } - Ok(()) - } - - /// Decode one symbol from the range coder. Mirrors `Ppmd7_DecodeSymbol` - /// for the order-0 case (`NumStats == 256`, no binary fast path, no - /// suffix walk, no SEE). - pub fn decode_symbol( - &mut self, - rd: &mut RangeDec, - src: &mut ByteSource<'_>, - ) -> Result { - // Snapshot for "need more input" rollback. - let rd_snap = rd.clone(); - let pos_snap = src.pos; - - match self.decode_inner(rd, src) { - Ok(sym) => Ok(sym), - Err(Error::UnexpectedEnd) => { - *rd = rd_snap; - src.pos = pos_snap; - Err(Error::UnexpectedEnd) - } - Err(e) => Err(e), - } - } - - fn decode_inner(&mut self, rd: &mut RangeDec, src: &mut ByteSource<'_>) -> Result { - let nstats = ctx_num_stats(&self.arena, self.ctx_off).ok_or(Error::Corrupt)?; - if nstats == 0 { - return Err(Error::Corrupt); - } - let summ_freq = ctx_summ_freq(&self.arena, self.ctx_off).ok_or(Error::Corrupt)?; - let total = summ_freq as u32; - if total == 0 { - return Err(Error::Corrupt); - } - let hi_count = rd.get_threshold(total); - - let mut acc: u32 = 0; - for i in 0..nstats as u32 { - let st = self.stats_off + i * STATE_SIZE as u32; - let f = state_freq(&self.arena, st).ok_or(Error::Corrupt)? as u32; - if acc + f > hi_count { - rd.decode(src, acc, f)?; - let sym = state_symbol(&self.arena, st).ok_or(Error::Corrupt)?; - self.bump_freq(i, f, summ_freq)?; - return Ok(sym); - } - acc += f; - } - Err(Error::Corrupt) - } - - fn bump_freq(&mut self, i: u32, freq: u32, summ_freq: u16) -> Result<(), Error> { - let new_freq = freq + 4; - let new_summ = summ_freq as u32 + 4; - if new_freq > MAX_FREQ as u32 || new_summ > 0xFFFF { - self.rescale()?; - return Ok(()); - } - let st = self.stats_off + i * STATE_SIZE as u32; - state_set_freq(&mut self.arena, st, new_freq as u8).ok_or(Error::Corrupt)?; - ctx_set_summ_freq(&mut self.arena, self.ctx_off, new_summ as u16).ok_or(Error::Corrupt)?; - // Promote: if this state's frequency now exceeds its predecessor's, - // swap them (keeps the array roughly sorted by freq). - if i > 0 { - let prev = self.stats_off + (i - 1) * STATE_SIZE as u32; - let prev_f = state_freq(&self.arena, prev).ok_or(Error::Corrupt)? as u32; - if new_freq > prev_f { - swap_states(&mut self.arena, st, prev).ok_or(Error::Corrupt)?; - } - } - Ok(()) - } - - fn rescale(&mut self) -> Result<(), Error> { - // Halve every frequency, dropping zero-frequency states (the - // reference's behaviour). We never go below one because order-0 - // must keep every symbol decodable. - let nstats = ctx_num_stats(&self.arena, self.ctx_off).ok_or(Error::Corrupt)? as u32; - let mut new_summ: u32 = 0; - for i in 0..nstats { - let st = self.stats_off + i * STATE_SIZE as u32; - let f = state_freq(&self.arena, st).ok_or(Error::Corrupt)? as u32; - let new_f = ((f + 1) >> 1).max(1) as u8; - state_set_freq(&mut self.arena, st, new_f).ok_or(Error::Corrupt)?; - new_summ += new_f as u32; - } - ctx_set_summ_freq(&mut self.arena, self.ctx_off, new_summ.min(0xFFFF) as u16) - .ok_or(Error::Corrupt)?; - Ok(()) - } -} diff --git a/src/ppmd/ppmd7.rs b/src/ppmd/ppmd7.rs new file mode 100644 index 0000000..1957959 --- /dev/null +++ b/src/ppmd/ppmd7.rs @@ -0,0 +1,1294 @@ +//! PPMII variant H context model (`Ppmd7`) — full decoder. +//! +//! Safe-Rust port of Igor Pavlov's public-domain `Ppmd7.{c,h}` (LZMA SDK, +//! itself based on Dmitry Shkarin's PPMd var.H, 2001 — both public domain). +//! The reference packs every `CPpmd7_Context` (12 bytes) and `CPpmd_State` +//! (6 bytes) into a single `Byte *Base` arena and refers to them by 32-bit +//! offset; we keep that byte-exact layout in one `Vec` and reach it +//! through little-endian accessors. Offsets ("refs") are `u32`; ref `0` is +//! reserved as the null pointer (the arena's live region starts at +//! `align_offset`, always `>= 1`). +//! +//! The range coder is external (see [`super::range_dec::RangeDec`]); the +//! model calls `get_threshold` / `decode` / `decode_bit` on it, so the same +//! model core drives both the 7z framing (standalone `.ppmd`) and RAR's +//! carry-less range decoder. +//! +//! This is a faithful, spec-derived port — not a transliteration of any +//! license-restricted RAR source. `Ppmd7.c` is public domain and may be +//! followed closely; libarchive's BSD RAR reader was consulted only for the +//! RAR range-decoder *description*, not copied. + +// The model walks a flat byte arena by 32-bit offset; explicit index loops +// (`for i in 0..n { ... base[off + i] ... }`) mirror the reference's pointer +// arithmetic and read more clearly than iterator adapters here. +#![allow(clippy::needless_range_loop)] + +extern crate alloc; + +use alloc::vec; +use alloc::vec::Vec; + +use crate::error::Error; + +use super::range_dec::RangeDec; + +const UNIT_SIZE: u32 = 12; +const MAX_FREQ: u8 = 124; +const PPMD_NUM_INDEXES: usize = 38; // N1+N2+N3+N4 with N1=N2=N3=4 +const PPMD_BIN_SCALE: u32 = 1 << 14; +const PPMD_INT_BITS: u32 = 7; +const PPMD_PERIOD_BITS: u32 = 7; +const MAX_ORDER: usize = 64; + +const K_INIT_BIN_ESC: [u16; 8] = [ + 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051, +]; +const K_EXP_ESCAPE: [u8; 16] = [25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2]; + +#[inline] +fn get_mean(summ: u32) -> u32 { + (summ + (1 << (PPMD_PERIOD_BITS - 2))) >> PPMD_PERIOD_BITS +} + +/// One SEE (secondary escape estimation) cell. +#[derive(Clone, Copy, Default)] +struct See { + summ: u16, + shift: u8, + count: u8, +} + +impl See { + /// `Ppmd_See_Update`. + #[inline] + fn update(&mut self) { + if (self.shift as u32) < PPMD_PERIOD_BITS && { + self.count = self.count.wrapping_sub(1); + self.count == 0 + } { + self.summ = self.summ.wrapping_shl(1); + self.count = 3u8.wrapping_shl(self.shift as u32); + self.shift += 1; + } + } +} + +/// Which SEE cell `make_esc_freq` selected — used so the caller can update +/// the same cell after decoding. +#[derive(Clone, Copy)] +enum SeeRef { + Dummy, + Cell(usize, usize), +} + +pub(crate) struct Ppmd7 { + base: Vec, + size: u32, + align_offset: u32, + err: bool, + + // Pointers (byte offsets into `base`). + min_context: u32, + max_context: u32, + found_state: u32, + text: u32, + lo_unit: u32, + hi_unit: u32, + units_start: u32, + glue_count: u32, + + order_fall: u32, + init_esc: u32, + prev_success: u32, + max_order: u32, + hi_bits_flag: u32, + run_length: i32, + init_rl: i32, + + free_list: [u32; PPMD_NUM_INDEXES], + + indx2units: [u8; PPMD_NUM_INDEXES], + units2indx: [u8; 128], + ns2indx: [u8; 256], + ns2bsindx: [u8; 256], + hb2flag: [u8; 256], + + see: [[See; 16]; 25], + dummy_see: See, + bin_summ: [[u16; 64]; 128], +} + +impl Ppmd7 { + /// Build the model for `mem_size` bytes of arena. The order is supplied + /// to [`Ppmd7::init`]. `mem_size` must be at least `UNIT_SIZE`. + pub(crate) fn new(mem_size: u32) -> Result { + if mem_size < UNIT_SIZE { + return Err(Error::BadHeader); + } + let align_offset = 4 - (mem_size & 3); + let total = (align_offset + mem_size + UNIT_SIZE) as usize; + let mut p = Self { + base: vec![0u8; total], + size: mem_size, + align_offset, + err: false, + min_context: 0, + max_context: 0, + found_state: 0, + text: 0, + lo_unit: 0, + hi_unit: 0, + units_start: 0, + glue_count: 0, + order_fall: 0, + init_esc: 0, + prev_success: 0, + max_order: 0, + hi_bits_flag: 0, + run_length: 0, + init_rl: 0, + free_list: [0; PPMD_NUM_INDEXES], + indx2units: [0; PPMD_NUM_INDEXES], + units2indx: [0; 128], + ns2indx: [0; 256], + ns2bsindx: [0; 256], + hb2flag: [0; 256], + see: [[See::default(); 16]; 25], + dummy_see: See::default(), + bin_summ: [[0u16; 64]; 128], + }; + p.construct_tables(); + Ok(p) + } + + /// `Ppmd7_Construct` — the fixed lookup tables. + fn construct_tables(&mut self) { + let mut k = 0usize; + for i in 0..PPMD_NUM_INDEXES { + let mut step = if i >= 12 { 4 } else { (i >> 2) + 1 }; + while step > 0 { + self.units2indx[k] = i as u8; + k += 1; + step -= 1; + } + self.indx2units[i] = k as u8; + } + + self.ns2bsindx[0] = 0; + self.ns2bsindx[1] = 2; + for v in self.ns2bsindx[2..11].iter_mut() { + *v = 4; + } + for v in self.ns2bsindx[11..256].iter_mut() { + *v = 6; + } + + for i in 0..3 { + self.ns2indx[i] = i as u8; + } + let mut m = 3u8; + let mut kk = 1i32; + for i in 3..256 { + self.ns2indx[i] = m; + kk -= 1; + if kk == 0 { + m += 1; + kk = (m as i32) - 2; + } + } + + for v in self.hb2flag[0..0x40].iter_mut() { + *v = 0; + } + for v in self.hb2flag[0x40..256].iter_mut() { + *v = 8; + } + } + + // ─── raw arena accessors (little-endian, OOB-safe) ────────────────── + + #[inline] + fn gu8(&mut self, off: u32) -> u8 { + match self.base.get(off as usize) { + Some(&b) => b, + None => { + self.err = true; + 0 + } + } + } + #[inline] + fn pu8(&mut self, off: u32, v: u8) { + match self.base.get_mut(off as usize) { + Some(b) => *b = v, + None => self.err = true, + } + } + #[inline] + fn gu16(&mut self, off: u32) -> u16 { + let o = off as usize; + match self.base.get(o..o + 2) { + Some(s) => u16::from_le_bytes([s[0], s[1]]), + None => { + self.err = true; + 0 + } + } + } + #[inline] + fn pu16(&mut self, off: u32, v: u16) { + let o = off as usize; + match self.base.get_mut(o..o + 2) { + Some(s) => s.copy_from_slice(&v.to_le_bytes()), + None => self.err = true, + } + } + #[inline] + fn gu32(&mut self, off: u32) -> u32 { + let o = off as usize; + match self.base.get(o..o + 4) { + Some(s) => u32::from_le_bytes([s[0], s[1], s[2], s[3]]), + None => { + self.err = true; + 0 + } + } + } + #[inline] + fn pu32(&mut self, off: u32, v: u32) { + let o = off as usize; + match self.base.get_mut(o..o + 4) { + Some(s) => s.copy_from_slice(&v.to_le_bytes()), + None => self.err = true, + } + } + + // ─── context / state field accessors ──────────────────────────────── + + #[inline] + fn ctx_num_stats(&mut self, c: u32) -> u32 { + self.gu16(c) as u32 + } + #[inline] + fn ctx_set_num_stats(&mut self, c: u32, v: u32) { + self.pu16(c, v as u16); + } + #[inline] + fn ctx_summ_freq(&mut self, c: u32) -> u32 { + self.gu16(c + 2) as u32 + } + #[inline] + fn ctx_set_summ_freq(&mut self, c: u32, v: u32) { + self.pu16(c + 2, v as u16); + } + #[inline] + fn ctx_stats(&mut self, c: u32) -> u32 { + self.gu32(c + 4) + } + #[inline] + fn ctx_set_stats(&mut self, c: u32, v: u32) { + self.pu32(c + 4, v); + } + #[inline] + fn ctx_suffix(&mut self, c: u32) -> u32 { + self.gu32(c + 8) + } + #[inline] + fn ctx_set_suffix(&mut self, c: u32, v: u32) { + self.pu32(c + 8, v); + } + /// `Ppmd7Context_OneState` — the embedded single state overlaps + /// SummFreq+Stats, i.e. offset `c + 2`. + #[inline] + fn one_state(c: u32) -> u32 { + c + 2 + } + + #[inline] + fn st_symbol(&mut self, s: u32) -> u8 { + self.gu8(s) + } + #[inline] + fn st_set_symbol(&mut self, s: u32, v: u8) { + self.pu8(s, v); + } + #[inline] + fn st_freq(&mut self, s: u32) -> u8 { + self.gu8(s + 1) + } + #[inline] + fn st_set_freq(&mut self, s: u32, v: u8) { + self.pu8(s + 1, v); + } + /// `st.Freq += d` (read-modify-write; avoids `setter(getter())`). + #[inline] + fn st_add_freq(&mut self, s: u32, d: u8) { + let f = self.st_freq(s); + self.st_set_freq(s, f.wrapping_add(d)); + } + /// `ctx.SummFreq += d`. + #[inline] + fn ctx_add_summ_freq(&mut self, c: u32, d: u32) { + let f = self.ctx_summ_freq(c); + self.ctx_set_summ_freq(c, f + d); + } + #[inline] + fn st_successor(&mut self, s: u32) -> u32 { + (self.gu16(s + 2) as u32) | ((self.gu16(s + 4) as u32) << 16) + } + #[inline] + fn st_set_successor(&mut self, s: u32, v: u32) { + self.pu16(s + 2, (v & 0xFFFF) as u16); + self.pu16(s + 4, ((v >> 16) & 0xFFFF) as u16); + } + + /// Copy state `src` onto state `dst` (6 bytes) — one range check, one + /// block copy, like [`Ppmd7::copy_units`] (rescale's insertion sort + /// calls this in a loop). + #[inline] + fn st_copy(&mut self, dst: u32, src: u32) { + let (d, s) = (dst as usize, src as usize); + if d + 6 <= self.base.len() && s + 6 <= self.base.len() { + self.base.copy_within(s..s + 6, d); + } else { + self.err = true; + } + } + /// Swap two 6-byte states. + #[inline] + fn st_swap(&mut self, a: u32, b: u32) { + let (a, b) = (a as usize, b as usize); + if a + 6 <= self.base.len() && b + 6 <= self.base.len() { + for i in 0..6 { + self.base.swap(a + i, b + i); + } + } else { + self.err = true; + } + } + + #[inline] + fn i2u(&self, indx: usize) -> u32 { + self.indx2units[indx] as u32 + } + #[inline] + fn u2i(&self, nu: u32) -> usize { + // nu in 1..=128 + self.units2indx[(nu as usize).wrapping_sub(1).min(127)] as usize + } + #[inline] + fn u2b(nu: u32) -> u32 { + nu * UNIT_SIZE + } + + /// Copy `nu` units (12 bytes each) from `src` to `dst`. + fn copy_units(&mut self, dst: u32, src: u32, nu: u32) { + let n = (nu * UNIT_SIZE) as usize; + let (d, s) = (dst as usize, src as usize); + if d + n <= self.base.len() && s + n <= self.base.len() { + self.base.copy_within(s..s + n, d); + } else { + self.err = true; + } + } + + // ─── allocator ────────────────────────────────────────────────────── + + fn insert_node(&mut self, node: u32, indx: usize) { + let head = self.free_list[indx]; + self.pu32(node, head); + self.free_list[indx] = node; + } + + fn remove_node(&mut self, indx: usize) -> u32 { + let node = self.free_list[indx]; + let next = self.gu32(node); + self.free_list[indx] = next; + node + } + + fn split_block(&mut self, ptr: u32, old_indx: usize, new_indx: usize) { + let nu = self.i2u(old_indx) - self.i2u(new_indx); + let ptr = ptr + Self::u2b(self.i2u(new_indx)); + let mut i = self.u2i(nu); + if self.i2u(i) != nu { + i -= 1; + let k = self.i2u(i); + self.insert_node(ptr + Self::u2b(k), (nu - k - 1) as usize); + } + self.insert_node(ptr, i); + } + + fn glue_free_blocks(&mut self) { + // Node layout: Stamp u16 @0, NU u16 @2, Next u32 @4, Prev u32 @8. + // The reference builds a doubly-linked list of all free blocks, sets + // a head sentinel just past the arena, coalesces adjacent blocks, + // and re-buckets them. We follow it directly. + // + // KNOWN LIMITATION (fails closed): this ports the classic LZMA SDK + // GlueFreeBlocks. When a stream actually exhausts the suballocator — + // e.g. a high-order model over a large, high-entropy payload in a + // small (1 MiB) arena — a coalescing/re-bucketing difference from the + // exact encoder build can make our allocator fail (and thus restart + // the model) at a slightly different symbol than the encoder did, + // desyncing the range coder. That surfaces as `Error::Corrupt`, never + // wrong bytes and never an out-of-bounds access. Streams that don't + // fill the arena (the overwhelming majority) are unaffected. Reaching + // full allocator parity across every arena-exhausting input is + // tracked as Phase 3 hardening. + let head = self.align_offset + self.size; // sentinel node ref + let mut n = head; + self.glue_count = 255; + + for i in 0..PPMD_NUM_INDEXES { + let nu = self.i2u(i) as u16; + let mut next = self.free_list[i]; + self.free_list[i] = 0; + while next != 0 { + let node = next; + // node->Next = n + self.pu32(node + 4, n); + // n = node->Prev = node (NODE(n)->Prev = node; then n = node) + self.pu32(n + 8, node); + n = node; + next = self.gu32(node); // *(Ref*)node — original free-list link + self.pu16(node, 0); // Stamp = 0 + self.pu16(node + 2, nu); // NU + } + } + + // head node + self.pu16(head, 1); // Stamp + self.pu32(head + 4, n); // Next + self.pu32(n + 8, head); // NODE(n)->Prev = head + if self.lo_unit != self.hi_unit { + self.pu16(self.lo_unit, 1); // Stamp guard + } + + // Glue adjacent free blocks (walk head->Next round to head). + let mut n = self.gu32(head + 4); + while n != head { + let node = n; + let mut nu = self.gu16(node + 2) as u32; + loop { + let node2 = node + nu * UNIT_SIZE; + let stamp = self.gu16(node2); + let nu2 = self.gu16(node2 + 2) as u32; + if stamp != 0 || nu + nu2 >= 0x10000 { + break; + } + nu += nu2; + // unlink node2 + let prev = self.gu32(node2 + 8); + let nxt = self.gu32(node2 + 4); + self.pu32(prev + 4, nxt); + self.pu32(nxt + 8, prev); + self.pu16(node + 2, nu as u16); + } + n = self.gu32(node + 4); + if self.err { + return; + } + } + + // Refill free lists. + let mut n = self.gu32(head + 4); + while n != head { + let mut node = n; + let next = self.gu32(node + 4); + let mut nu = self.gu16(node + 2) as u32; + while nu > 128 { + self.insert_node(node, PPMD_NUM_INDEXES - 1); + nu -= 128; + node += 128 * UNIT_SIZE; + } + let mut i = self.u2i(nu); + if self.i2u(i) != nu { + i -= 1; + let k = self.i2u(i); + self.insert_node(node + k * UNIT_SIZE, (nu - k - 1) as usize); + } + self.insert_node(node, i); + n = next; + if self.err { + return; + } + } + } + + fn alloc_units_rare(&mut self, indx: usize) -> u32 { + if self.glue_count == 0 { + self.glue_free_blocks(); + if self.free_list[indx] != 0 { + return self.remove_node(indx); + } + } + let mut i = indx; + loop { + i += 1; + if i == PPMD_NUM_INDEXES { + let num_bytes = Self::u2b(self.i2u(indx)); + self.glue_count = self.glue_count.wrapping_sub(1); + return if self.units_start - self.text > num_bytes { + self.units_start -= num_bytes; + self.units_start + } else { + 0 + }; + } + if self.free_list[i] != 0 { + break; + } + } + let ret = self.remove_node(i); + self.split_block(ret, i, indx); + ret + } + + fn alloc_units(&mut self, indx: usize) -> u32 { + if self.free_list[indx] != 0 { + return self.remove_node(indx); + } + let num_bytes = Self::u2b(self.i2u(indx)); + if num_bytes <= self.hi_unit - self.lo_unit { + let ret = self.lo_unit; + self.lo_unit += num_bytes; + return ret; + } + self.alloc_units_rare(indx) + } + + fn shrink_units(&mut self, old_ptr: u32, old_nu: u32, new_nu: u32) -> u32 { + let i0 = self.u2i(old_nu); + let i1 = self.u2i(new_nu); + if i0 == i1 { + return old_ptr; + } + if self.free_list[i1] != 0 { + let ptr = self.remove_node(i1); + self.copy_units(ptr, old_ptr, new_nu); + self.insert_node(old_ptr, i0); + ptr + } else { + self.split_block(old_ptr, i0, i1); + old_ptr + } + } + + // ─── init / restart ───────────────────────────────────────────────── + + pub(crate) fn init(&mut self, max_order: u32) { + self.max_order = max_order; + self.restart_model(); + self.dummy_see.shift = PPMD_PERIOD_BITS as u8; + self.dummy_see.summ = 0; + self.dummy_see.count = 64; + } + + /// Seed `InitEsc` (RAR's header carries it explicitly). Only used by + /// the RAR3/4 PPMd path. + #[cfg_attr(not(feature = "rar3"), allow(dead_code))] + pub(crate) fn set_init_esc(&mut self, v: u32) { + self.init_esc = v; + } + + fn restart_model(&mut self) { + self.free_list = [0; PPMD_NUM_INDEXES]; + self.text = self.align_offset; + self.hi_unit = self.text + self.size; + let reserved = self.size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; + self.lo_unit = self.hi_unit - reserved; + self.units_start = self.lo_unit; + self.glue_count = 0; + + self.order_fall = self.max_order; + let mo = if self.max_order < 12 { + self.max_order + } else { + 12 + }; + self.init_rl = -(mo as i32) - 1; + self.run_length = self.init_rl; + self.prev_success = 0; + + self.hi_unit -= UNIT_SIZE; + let mc = self.hi_unit; + self.min_context = mc; + self.max_context = mc; + self.ctx_set_suffix(mc, 0); + self.ctx_set_num_stats(mc, 256); + self.ctx_set_summ_freq(mc, 256 + 1); + self.found_state = self.lo_unit; + self.lo_unit += Self::u2b(256 / 2); + let fs = self.found_state; + self.ctx_set_stats(mc, fs); + for i in 0..256u32 { + let s = fs + i * 6; + self.st_set_symbol(s, i as u8); + self.st_set_freq(s, 1); + self.st_set_successor(s, 0); + } + + for i in 0..128 { + for k in 0..8 { + let val = (PPMD_BIN_SCALE - (K_INIT_BIN_ESC[k] as u32) / (i as u32 + 2)) as u16; + let mut m = 0; + while m < 64 { + self.bin_summ[i][k + m] = val; + m += 8; + } + } + } + + for i in 0..25 { + for k in 0..16 { + let shift = (PPMD_PERIOD_BITS - 4) as u8; + self.see[i][k] = See { + summ: ((5 * i as u32 + 10) << shift) as u16, + shift, + count: 4, + }; + } + } + } + + // ─── model update helpers ─────────────────────────────────────────── + + /// `CreateSuccessors`. Returns `Some(ctx)` or `None` (NULL → restart). + fn create_successors(&mut self, skip: bool) -> Option { + let mut c = self.min_context; + let up_branch = self.st_successor(self.found_state); + let mut ps: [u32; MAX_ORDER] = [0; MAX_ORDER]; + let mut num_ps = 0usize; + + if !skip { + ps[num_ps] = self.found_state; + num_ps += 1; + } + + let found_sym = self.st_symbol(self.found_state); + while self.ctx_suffix(c) != 0 { + c = self.ctx_suffix(c); + let s = if self.ctx_num_stats(c) != 1 { + let mut ss = self.ctx_stats(c); + while self.st_symbol(ss) != found_sym { + ss += 6; + } + ss + } else { + Self::one_state(c) + }; + let successor = self.st_successor(s); + if successor != up_branch { + c = successor; + if num_ps == 0 { + return Some(c); + } + break; + } + if num_ps >= MAX_ORDER { + self.err = true; + return None; + } + ps[num_ps] = s; + num_ps += 1; + } + + // upState + let up_symbol = self.gu8(up_branch); + let up_successor = up_branch + 1; + let up_freq: u8 = if self.ctx_num_stats(c) == 1 { + self.st_freq(Self::one_state(c)) + } else { + let mut s = self.ctx_stats(c); + while self.st_symbol(s) != up_symbol { + s += 6; + } + let cf = self.st_freq(s) as u32 - 1; + let s0 = self.ctx_summ_freq(c) - self.ctx_num_stats(c) - cf; + (1 + if 2 * cf <= s0 { + (5 * cf > s0) as u32 + } else { + (2 * cf + 3 * s0 - 1) / (2 * s0) + }) as u8 + }; + + while num_ps != 0 { + // AllocContext + let c1 = if self.hi_unit != self.lo_unit { + self.hi_unit -= UNIT_SIZE; + self.hi_unit + } else if self.free_list[0] != 0 { + self.remove_node(0) + } else { + let r = self.alloc_units_rare(0); + if r == 0 { + return None; + } + r + }; + self.ctx_set_num_stats(c1, 1); + let os = Self::one_state(c1); + self.st_set_symbol(os, up_symbol); + self.st_set_freq(os, up_freq); + self.st_set_successor(os, up_successor); + self.ctx_set_suffix(c1, c); + num_ps -= 1; + self.st_set_successor(ps[num_ps], c1); + c = c1; + } + Some(c) + } + + fn update_model(&mut self) { + let f_successor = self.st_successor(self.found_state); + let found_sym = self.st_symbol(self.found_state); + let found_freq = self.st_freq(self.found_state); + + if found_freq < MAX_FREQ / 4 && self.ctx_suffix(self.min_context) != 0 { + let c = self.ctx_suffix(self.min_context); + if self.ctx_num_stats(c) == 1 { + let s = Self::one_state(c); + if self.st_freq(s) < 32 { + self.st_add_freq(s, 1); + } + } else { + let mut s = self.ctx_stats(c); + if self.st_symbol(s) != found_sym { + loop { + s += 6; + if self.st_symbol(s) == found_sym { + break; + } + } + if self.st_freq(s) >= self.st_freq(s - 6) { + self.st_swap(s, s - 6); + s -= 6; + } + } + if self.st_freq(s) < MAX_FREQ - 9 { + self.st_add_freq(s, 2); + self.ctx_add_summ_freq(c, 2); + } + } + } + + if self.order_fall == 0 { + match self.create_successors(true) { + Some(mc) => { + self.min_context = mc; + self.max_context = mc; + self.st_set_successor(self.found_state, mc); + } + None => self.restart_model(), + } + return; + } + + // *Text++ = found symbol + self.pu8(self.text, found_sym); + self.text += 1; + let successor = self.text; + if self.text >= self.units_start { + self.restart_model(); + return; + } + + let mut f_successor = f_successor; + let mut successor = successor; + if f_successor != 0 { + if f_successor <= self.text { + // f_successor points into the text region → materialise it. + match self.create_successors(false) { + Some(cs) => f_successor = cs, + None => { + self.restart_model(); + return; + } + } + } + self.order_fall -= 1; + if self.order_fall == 0 { + successor = f_successor; + if self.max_context != self.min_context { + self.text -= 1; + } + } + } else { + self.st_set_successor(self.found_state, successor); + f_successor = self.min_context; + } + + let ns = self.ctx_num_stats(self.min_context); + let s0 = self.ctx_summ_freq(self.min_context) - ns - (found_freq as u32 - 1); + + let mut c = self.max_context; + while c != self.min_context { + let ns1 = self.ctx_num_stats(c); + if ns1 != 1 { + if ns1 & 1 == 0 { + // grow the stats block by one unit + let old_nu = ns1 >> 1; + let i = self.u2i(old_nu); + if i != self.u2i(old_nu + 1) { + let ptr = self.alloc_units(i + 1); + if ptr == 0 { + self.restart_model(); + return; + } + let old_ptr = self.ctx_stats(c); + self.copy_units(ptr, old_ptr, old_nu); + self.insert_node(old_ptr, i); + self.ctx_set_stats(c, ptr); + } + } + let add = (2 * ns1 < ns) as u32 + + 2 * ((4 * ns1 <= ns) as u32 & (self.ctx_summ_freq(c) <= 8 * ns1) as u32); + self.ctx_add_summ_freq(c, add); + } else { + let s = self.alloc_units(0); + if s == 0 { + self.restart_model(); + return; + } + self.st_copy(s, Self::one_state(c)); + self.ctx_set_stats(c, s); + let mut fr = self.st_freq(s); + if fr < MAX_FREQ / 4 - 1 { + fr <<= 1; + } else { + fr = MAX_FREQ - 4; + } + self.st_set_freq(s, fr); + self.ctx_set_summ_freq(c, fr as u32 + self.init_esc + (ns > 3) as u32); + } + + let cf = 2 * (found_freq as u32) * (self.ctx_summ_freq(c) + 6); + let sf = s0 + self.ctx_summ_freq(c); + let new_freq; + if cf < 6 * sf { + new_freq = 1 + (cf > sf) as u32 + (cf >= 4 * sf) as u32; + self.ctx_add_summ_freq(c, 3); + } else { + new_freq = + 4 + (cf >= 9 * sf) as u32 + (cf >= 12 * sf) as u32 + (cf >= 15 * sf) as u32; + self.ctx_add_summ_freq(c, new_freq); + } + let stats = self.ctx_stats(c); + let s = stats + ns1 * 6; + self.st_set_successor(s, successor); + self.st_set_symbol(s, found_sym); + self.st_set_freq(s, new_freq as u8); + self.ctx_set_num_stats(c, ns1 + 1); + + c = self.ctx_suffix(c); + if self.err { + return; + } + } + + self.max_context = f_successor; + self.min_context = f_successor; + } + + fn rescale(&mut self) { + let stats = self.ctx_stats(self.min_context); + let mut s = self.found_state; + // Move found state to front (rotate). + { + let mut tmp = [0u8; 6]; + for i in 0..6 { + tmp[i] = self.gu8(s + i as u32); + } + while s != stats { + self.st_copy(s, s - 6); + s -= 6; + } + for i in 0..6 { + self.pu8(s + i as u32, tmp[i]); + } + } + let mut esc_freq = self.ctx_summ_freq(self.min_context) - self.st_freq(s) as u32; + self.st_add_freq(s, 4); + let adder = (self.order_fall != 0) as u32; + { + let nf = ((self.st_freq(s) as u32 + adder) >> 1) as u8; + self.st_set_freq(s, nf); + } + let mut sum_freq = self.st_freq(s) as u32; + + let mut i = self.ctx_num_stats(self.min_context) - 1; + loop { + s += 6; + esc_freq -= self.st_freq(s) as u32; + { + let nf = ((self.st_freq(s) as u32 + adder) >> 1) as u8; + self.st_set_freq(s, nf); + } + sum_freq += self.st_freq(s) as u32; + if self.st_freq(s) > self.st_freq(s - 6) { + let mut s1 = s; + let mut tmp = [0u8; 6]; + for j in 0..6 { + tmp[j] = self.gu8(s1 + j as u32); + } + let tmp_freq = tmp[1]; + loop { + self.st_copy(s1, s1 - 6); + s1 -= 6; + if s1 == stats || tmp_freq <= self.st_freq(s1 - 6) { + break; + } + } + for j in 0..6 { + self.pu8(s1 + j as u32, tmp[j]); + } + } + i -= 1; + if i == 0 { + break; + } + } + + if self.st_freq(s) == 0 { + let mut cnt = 0u32; + loop { + cnt += 1; + s -= 6; + if self.st_freq(s) != 0 { + break; + } + } + esc_freq += cnt; + let num_stats = self.ctx_num_stats(self.min_context); + let new_ns = num_stats - cnt; + self.ctx_set_num_stats(self.min_context, new_ns); + if new_ns == 1 { + let mut tmp = [0u8; 6]; + for j in 0..6 { + tmp[j] = self.gu8(stats + j as u32); + } + let mut tmp_freq = tmp[1]; + loop { + tmp_freq = tmp_freq - (tmp_freq >> 1); + esc_freq >>= 1; + if esc_freq <= 1 { + break; + } + } + tmp[1] = tmp_freq; + self.insert_node(stats, self.u2i((num_stats + 1) >> 1)); + let fs = Self::one_state(self.min_context); + self.found_state = fs; + for j in 0..6 { + self.pu8(fs + j as u32, tmp[j]); + } + return; + } + let n0 = (num_stats + 1) >> 1; + let n1 = (new_ns + 1) >> 1; + if n0 != n1 { + let ns = self.shrink_units(stats, n0, n1); + self.ctx_set_stats(self.min_context, ns); + } + } + let stats = self.ctx_stats(self.min_context); + self.ctx_set_summ_freq(self.min_context, sum_freq + esc_freq - (esc_freq >> 1)); + self.found_state = stats; + } + + /// `Ppmd7_MakeEscFreq`. Returns the SEE ref and writes `esc_freq`. + fn make_esc_freq(&mut self, num_masked: u32) -> (SeeRef, u32) { + let num_stats = self.ctx_num_stats(self.min_context); + if num_stats != 256 { + let non_masked = num_stats - num_masked; + let suffix = self.ctx_suffix(self.min_context); + let suffix_ns = self.ctx_num_stats(suffix) as i64; + let idx_row = self.ns2indx[(non_masked - 1) as usize] as usize; + let diff = suffix_ns - num_stats as i64; + let idx_col = ((non_masked as i64) < diff) as usize + + 2 * (self.ctx_summ_freq(self.min_context) < 11 * num_stats) as usize + + 4 * (num_masked > non_masked) as usize + + self.hi_bits_flag as usize; + let see = &mut self.see[idx_row][idx_col]; + let r = (see.summ >> see.shift) as u32; + see.summ = see.summ.wrapping_sub(r as u16); + let esc = r + (r == 0) as u32; + (SeeRef::Cell(idx_row, idx_col), esc) + } else { + (SeeRef::Dummy, 1) + } + } + + #[inline] + fn see_update(&mut self, sr: SeeRef) { + if let SeeRef::Cell(i, k) = sr { + self.see[i][k].update(); + } + } + #[inline] + fn see_add_summ(&mut self, sr: SeeRef, v: u32) { + if let SeeRef::Cell(i, k) = sr { + let s = &mut self.see[i][k]; + s.summ = s.summ.wrapping_add(v as u16); + } + } + + fn next_context(&mut self) { + let c = self.st_successor(self.found_state); + if self.order_fall == 0 && c > self.text { + self.min_context = c; + self.max_context = c; + } else { + self.update_model(); + } + } + + fn update1(&mut self) { + let s = self.found_state; + self.st_add_freq(s, 4); + let c = self.min_context; + self.ctx_add_summ_freq(c, 4); + if self.st_freq(s) > self.st_freq(s - 6) { + self.st_swap(s, s - 6); + self.found_state = s - 6; + if self.st_freq(s - 6) > MAX_FREQ { + self.rescale(); + } + } + self.next_context(); + } + + fn update1_0(&mut self) { + let s = self.found_state; + self.prev_success = + (2 * self.st_freq(s) as u32 > self.ctx_summ_freq(self.min_context)) as u32; + self.run_length += self.prev_success as i32; + let c = self.min_context; + self.ctx_add_summ_freq(c, 4); + self.st_add_freq(s, 4); + if self.st_freq(s) > MAX_FREQ { + self.rescale(); + } + self.next_context(); + } + + fn update_bin(&mut self) { + let s = self.found_state; + let f = self.st_freq(s); + self.st_set_freq(s, f + (f < 128) as u8); + self.prev_success = 1; + self.run_length += 1; + self.next_context(); + } + + fn update2(&mut self) { + let s = self.found_state; + let c = self.min_context; + self.ctx_add_summ_freq(c, 4); + self.st_add_freq(s, 4); + if self.st_freq(s) > MAX_FREQ { + self.rescale(); + } + self.run_length = self.init_rl; + self.update_model(); + } + + // ─── decode ───────────────────────────────────────────────────────── + + /// Decode one byte symbol. `Err(Corrupt)` on model/stream inconsistency + /// (the reference's `-1`/`-2` returns) or arena OOB. + pub(crate) fn decode_symbol(&mut self, rc: &mut RangeDec) -> Result { + // Initialised on entry to the escape path; the fast paths below + // return without ever reading it. + let mut char_mask: [u8; 256]; + + // `num_stats` is a raw arena u16: a context ref that has been + // corrupted (or dangles into freed/zeroed arena) can present 0 or + // an impossible count (> 256 — the alphabet is bytes). Both would + // wrap the `- 1` walks below; fail closed instead. + let root_stats = self.ctx_num_stats(self.min_context); + if root_stats == 0 || root_stats > 256 { + return Err(Error::Corrupt); + } + + if root_stats != 1 { + let mut s = self.ctx_stats(self.min_context); + let count = rc.get_threshold(self.ctx_summ_freq(self.min_context)); + let mut hi_cnt = self.st_freq(s) as u32; + if count < hi_cnt { + rc.decode(0, hi_cnt); + self.found_state = s; + let sym = self.st_symbol(s); + self.update1_0(); + return self.finish(sym); + } + self.prev_success = 0; + let mut i = self.ctx_num_stats(self.min_context) - 1; + loop { + s += 6; + let f = self.st_freq(s) as u32; + hi_cnt += f; + if hi_cnt > count { + rc.decode(hi_cnt - f, f); + self.found_state = s; + let sym = self.st_symbol(s); + self.update1(); + return self.finish(sym); + } + i -= 1; + if i == 0 { + break; + } + } + if count >= self.ctx_summ_freq(self.min_context) { + return Err(Error::Corrupt); + } + self.hi_bits_flag = self.hb2flag[self.st_symbol(self.found_state) as usize] as u32; + rc.decode(hi_cnt, self.ctx_summ_freq(self.min_context) - hi_cnt); + char_mask = [0xFF; 256]; + char_mask[self.st_symbol(s) as usize] = 0; + let mut i = self.ctx_num_stats(self.min_context) - 1; + while i != 0 { + s -= 6; + char_mask[self.st_symbol(s) as usize] = 0; + i -= 1; + } + } else { + let (row, col) = self.bin_summ_index(); + let prob = self.bin_summ[row][col] as u32; + let bit = rc.decode_bit(prob); + if self.err || rc.err() { + return Err(Error::Corrupt); + } + if bit == 0 { + self.bin_summ[row][col] = (prob + (1 << PPMD_INT_BITS) - get_mean(prob)) as u16; + let s = Self::one_state(self.min_context); + self.found_state = s; + let sym = self.st_symbol(s); + self.update_bin(); + return self.finish(sym); + } + let newp = prob - get_mean(prob); + self.bin_summ[row][col] = newp as u16; + self.init_esc = K_EXP_ESCAPE[(newp >> 10) as usize & 0xF] as u32; + char_mask = [0xFF; 256]; + let os = Self::one_state(self.min_context); + char_mask[self.st_symbol(os) as usize] = 0; + self.prev_success = 0; + } + + // Escape loop. + let mut ps: [u32; 256] = [0; 256]; + loop { + if self.err || rc.err() { + return Err(Error::Corrupt); + } + let mut num_masked = self.ctx_num_stats(self.min_context); + loop { + self.order_fall += 1; + let suffix = self.ctx_suffix(self.min_context); + if suffix == 0 { + return Err(Error::Corrupt); + } + self.min_context = suffix; + if self.ctx_num_stats(self.min_context) != num_masked { + break; + } + } + let mut hi_cnt = 0u32; + let mut s = self.ctx_stats(self.min_context); + // In a consistent model a suffix context is a superset of its + // children, so its `num_stats` exceeds the masked count and + // never tops 256 (byte alphabet). A corrupt tree can violate + // both; the subtraction would wrap and the `ps` walk below + // would run past the array. Fail closed. + let cur_stats = self.ctx_num_stats(self.min_context); + if cur_stats <= num_masked || cur_stats > 256 { + return Err(Error::Corrupt); + } + let num = cur_stats - num_masked; + let mut i = 0usize; + loop { + let sym = self.st_symbol(s) as usize; + let masked = char_mask[sym]; + if masked != 0 { + hi_cnt += self.st_freq(s) as u32; + ps[i] = s; + i += 1; + } + s += 6; + if i as u32 == num { + break; + } + if self.err { + return Err(Error::Corrupt); + } + } + + let (see_ref, mut freq_sum) = self.make_esc_freq(num_masked); + freq_sum += hi_cnt; + let count = rc.get_threshold(freq_sum); + + if count < hi_cnt { + let mut acc = 0u32; + let mut idx = 0usize; + loop { + acc += self.st_freq(ps[idx]) as u32; + if acc > count { + break; + } + idx += 1; + } + let sstate = ps[idx]; + let f = self.st_freq(sstate) as u32; + rc.decode(acc - f, f); + self.see_update(see_ref); + self.found_state = sstate; + let sym = self.st_symbol(sstate); + self.update2(); + return self.finish(sym); + } + if count >= freq_sum { + return Err(Error::Corrupt); + } + rc.decode(hi_cnt, freq_sum - hi_cnt); + self.see_add_summ(see_ref, freq_sum); + let mut j = i; + while j != 0 { + j -= 1; + char_mask[self.st_symbol(ps[j]) as usize] = 0; + } + num_masked = self.ctx_num_stats(self.min_context); + let _ = num_masked; + } + } + + #[inline] + fn finish(&mut self, sym: u8) -> Result { + if self.err { + Err(Error::Corrupt) + } else { + Ok(sym) + } + } + + /// `Ppmd7_GetBinSumm` index computation (also sets `hi_bits_flag`). + fn bin_summ_index(&mut self) -> (usize, usize) { + let os = Self::one_state(self.min_context); + let os_freq = self.st_freq(os); + let suffix = self.ctx_suffix(self.min_context); + let suffix_ns = self.ctx_num_stats(suffix); + let found_sym = self.st_symbol(self.found_state); + self.hi_bits_flag = self.hb2flag[found_sym as usize] as u32; + let os_sym = self.st_symbol(os); + let row = (os_freq - 1) as usize; + let col = self.prev_success as usize + + self.ns2bsindx[(suffix_ns - 1) as usize] as usize + + self.hi_bits_flag as usize + + 2 * self.hb2flag[os_sym as usize] as usize + + ((self.run_length >> 26) & 0x20) as usize; + (row.min(127), col.min(63)) + } +} diff --git a/src/ppmd/range_dec.rs b/src/ppmd/range_dec.rs index 5f0a374..5d51d76 100644 --- a/src/ppmd/range_dec.rs +++ b/src/ppmd/range_dec.rs @@ -1,137 +1,206 @@ -//! Carry-less 7z range decoder used by PPMd variant H. +//! Carry-less range decoder for PPMd variant H, in both flavours: //! -//! Differences from the LZMA range decoder: +//! - **7z** (`Ppmd7z_RangeDec`) — used by 7-Zip's `PPMd` method and the +//! standalone `.ppmd` framing in this crate. Init consumes a leading +//! `0x00` byte plus four big-endian bytes; `decode` subtracts from +//! `code`; `Bottom == 0`. +//! - **RAR** (`PpmdRAR_RangeDec`) — used by RAR3/4 PPMd blocks. Init +//! consumes four big-endian bytes (no leading zero); `decode` adds to a +//! tracked `low`; `Bottom == 0x8000`; bit decoding routes through +//! `get_threshold` + `decode` rather than the 7z fast path. //! -//! - **No "first byte must be zero" rule** in the encoder/decoder -//! protocol per se, but the reference `Ppmd7z_RangeDec_Init` requires -//! the first byte read to be `0x00`. (Hand-rolled fixtures and every -//! 7z-/RAR-/ZIP-produced stream do start with `0x00`.) -//! - **`code` is initialised from the next four bytes** big-endian, and -//! `range` starts at `0xFFFF_FFFF`. The decoder calls -//! `Range_Normalize` which conditionally pulls one byte at a time and -//! handles two consecutive shifts. -//! - **`Range_GetThreshold(total)` divides `range /= total`** (mutating -//! `range`!), then returns `code / range`. The decoder uses that -//! quotient as the symbol index into the frequency table. -//! - **`Range_DecodeBit(size0)`** uses an explicit 14-bit shift -//! (`range >> 14`) and never updates probabilities (the model owns -//! that — see `PPMD_UPDATE_PROB_*`). -//! -//! Streaming: callers pull bytes from an internal buffered byte source. -//! When the decoder needs a byte and the buffer is empty, the symbol -//! decode aborts upward via `NeedInput` so the outer loop can refill. +//! Both share `get_threshold` (`range /= total; (code - low) / range`) and +//! the `low`/`bottom`-aware normalisation. Derived from the public-domain +//! `Ppmd7Dec.c` (LZMA SDK) and the RAR variant described in libarchive's +//! BSD RAR reader; no license-restricted code was copied. use crate::error::Error; const K_TOP_VALUE: u32 = 1 << 24; +const PPMD_BIN_SCALE: u32 = 1 << 14; +/// Safety cap on normalisation iterations (a well-formed stream needs at +/// most a few); prevents a crafted/truncated stream from spinning. +const MAX_NORMALIZE_STEPS: u32 = 64; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Mode { + SevenZip, + /// RAR3/4 PPMd blocks. Only constructed behind the `rar3` feature. + #[cfg_attr(not(feature = "rar3"), allow(dead_code))] + Rar, +} -/// Trait-free byte source so the decoder can be driven either from a -/// pre-buffered slice (during a decode call) or from a "we already read -/// past the end" sentinel during init. -pub(super) struct ByteSource<'a> { - pub buf: &'a [u8], - pub pos: usize, +pub(crate) struct RangeDec<'a> { + range: u32, + code: u32, + low: u32, + bottom: u32, + mode: Mode, + input: &'a [u8], + pos: usize, + err: bool, } -impl<'a> ByteSource<'a> { - pub fn new(buf: &'a [u8], pos: usize) -> Self { - Self { buf, pos } +impl<'a> RangeDec<'a> { + /// Construct and initialise from `input[start..]`. Returns the decoder + /// and the number of input bytes the init consumed. + pub(crate) fn init(mode: Mode, input: &'a [u8], start: usize) -> Result<(Self, usize), Error> { + let mut d = RangeDec { + range: 0xFFFF_FFFF, + code: 0, + low: 0, + bottom: 0, + mode, + input, + pos: start, + err: false, + }; + let consumed = match mode { + Mode::SevenZip => { + // First byte must be zero, then 4 big-endian bytes. + if input.len() < start + 5 { + return Err(Error::UnexpectedEnd); + } + if input[start] != 0 { + return Err(Error::Corrupt); + } + d.pos = start + 1; + for _ in 0..4 { + d.code = (d.code << 8) | d.read_byte() as u32; + } + d.bottom = 0; + 5 + } + Mode::Rar => { + if input.len() < start + 4 { + return Err(Error::UnexpectedEnd); + } + for _ in 0..4 { + d.code = (d.code << 8) | d.read_byte() as u32; + } + d.bottom = 0x8000; + 4 + } + }; + Ok((d, consumed)) } - /// Returns the next byte and advances `pos`. `Err(UnexpectedEnd)` - /// when starved — the outer streaming machinery is expected to - /// translate that into a `NeedInput` rollback by snapshotting state - /// before the symbol decode begins. #[inline] - pub fn read(&mut self) -> Result { - let b = *self.buf.get(self.pos).ok_or(Error::UnexpectedEnd)?; - self.pos += 1; - Ok(b) + pub(crate) fn err(&self) -> bool { + self.err } -} -#[derive(Clone, Debug)] -pub(super) struct RangeDec { - pub range: u32, - pub code: u32, - /// Position into the caller's buffered input where the next byte - /// will be read from. - pub pos: usize, -} + /// The stream read past the end of the available input. + #[inline] + pub(crate) fn overran(&self) -> bool { + self.pos > self.input.len() + } -impl RangeDec { - pub fn new() -> Self { - Self { - range: 0, - code: 0, - pos: 0, - } + /// Byte position in the input the decoder has consumed up to (including + /// the init bytes and any normalisation look-ahead). RAR3 resumes its + /// bit-domain block headers at this offset when a PPMd block ends. + #[inline] + #[cfg_attr(not(feature = "rar3"), allow(dead_code))] + pub(crate) fn pos(&self) -> usize { + self.pos.min(self.input.len()) } - /// Initialise from the first 5 bytes of the PPMd payload. First byte - /// must be zero; the remaining four are big-endian and form the - /// initial `code`. - /// - /// Returns `Ok(true)` on successful init, `Ok(false)` if input was - /// short. On `code == 0xFFFF_FFFF` (reference rejects it) returns - /// `Err(Corrupt)`. - pub fn init(&mut self, buf: &[u8]) -> Result { - if buf.len() < self.pos + 5 { - return Ok(false); - } - if buf[self.pos] != 0 { - return Err(Error::Corrupt); - } - let b1 = buf[self.pos + 1] as u32; - let b2 = buf[self.pos + 2] as u32; - let b3 = buf[self.pos + 3] as u32; - let b4 = buf[self.pos + 4] as u32; - self.code = (b1 << 24) | (b2 << 16) | (b3 << 8) | b4; - self.range = 0xFFFF_FFFF; - self.pos += 5; - if self.code == 0xFFFF_FFFF { - return Err(Error::Corrupt); - } - Ok(true) + #[inline] + fn read_byte(&mut self) -> u8 { + let b = self.input.get(self.pos).copied().unwrap_or(0); + // Advance regardless so `pos`/`overran` reflect demand; a real + // stream never reads past the last symbol's bytes. + self.pos += 1; + b } - /// `range /= total; return code / range`. Mutates `self.range`. + /// `range /= total; return (code - low) / range`. #[inline] - pub fn get_threshold(&mut self, total: u32) -> u32 { + pub(crate) fn get_threshold(&mut self, total: u32) -> u32 { + if total == 0 { + self.err = true; + return 0; + } self.range /= total; - self.code / self.range + if self.range == 0 { + self.err = true; + return 0; + } + self.code.wrapping_sub(self.low) / self.range } - /// `range *= size`; advance `code` by `start * range_before`. - /// `range` has already been divided by `total` by `get_threshold`. + /// Advance past a decoded interval `[start, start+size)`. #[inline] - pub fn decode(&mut self, src: &mut ByteSource<'_>, start: u32, size: u32) -> Result<(), Error> { - self.code = self.code.wrapping_sub(start.wrapping_mul(self.range)); + pub(crate) fn decode(&mut self, start: u32, size: u32) { + match self.mode { + Mode::SevenZip => { + self.code = self.code.wrapping_sub(start.wrapping_mul(self.range)); + } + Mode::Rar => { + self.low = self.low.wrapping_add(start.wrapping_mul(self.range)); + } + } self.range = self.range.wrapping_mul(size); - self.normalize(src) + self.normalize(); } - /// Pull bytes while `range < 1<<24`. The reference loops at most - /// twice (`range` is shifted by 8 per iteration, so two iterations - /// take `range` from anywhere in `1..1<<24` to `>=1<<24`). + /// Decode one binary decision with probability `size0` (out of + /// `PPMD_BIN_SCALE`). Returns the bit. #[inline] - pub fn normalize(&mut self, src: &mut ByteSource<'_>) -> Result<(), Error> { - if self.range < K_TOP_VALUE { - self.code = (self.code << 8) | src.read()? as u32; + pub(crate) fn decode_bit(&mut self, size0: u32) -> u32 { + match self.mode { + Mode::SevenZip => { + let new_bound = (self.range >> 14) * size0; + if self.code < new_bound { + self.range = new_bound; + self.normalize(); + 0 + } else { + self.code -= new_bound; + self.range -= new_bound; + self.normalize(); + 1 + } + } + Mode::Rar => { + let value = self.get_threshold(PPMD_BIN_SCALE); + if value < size0 { + self.decode(0, size0); + 0 + } else { + self.decode(size0, PPMD_BIN_SCALE - size0); + 1 + } + } + } + } + + #[inline] + fn normalize(&mut self) { + let mut steps = 0u32; + loop { + if (self.low ^ self.low.wrapping_add(self.range)) >= K_TOP_VALUE { + if self.range >= self.bottom { + break; + } + // range too small: clamp to the bottom window (RAR path). + self.range = self.low.wrapping_neg() & self.bottom.wrapping_sub(1); + } + self.code = (self.code << 8) | self.read_byte() as u32; self.range <<= 8; - if self.range < K_TOP_VALUE { - self.code = (self.code << 8) | src.read()? as u32; - self.range <<= 8; + self.low <<= 8; + steps += 1; + if steps > MAX_NORMALIZE_STEPS { + self.err = true; + break; } } - Ok(()) } - /// Reference's `Ppmd7z_RangeDec_IsFinishedOK`. After draining the - /// final symbol the encoder leaves `code == 0`; anything else means - /// the stream was truncated or the model dropped a symbol. + /// 7z terminal check: after the final symbol the encoder leaves + /// `code == 0`. #[inline] - pub fn is_finished_ok(&self) -> bool { + pub(crate) fn is_finished_ok(&self) -> bool { self.code == 0 } } diff --git a/src/rar3/bits.rs b/src/rar3/bits.rs index 5c9ee94..050bdbb 100644 --- a/src/rar3/bits.rs +++ b/src/rar3/bits.rs @@ -84,10 +84,29 @@ impl BitReader { Ok(()) } + /// Consume `n` bits that a prior `peek(m >= n)` on the *same* buffered + /// state already proved are available — no availability check. The caller + /// must guarantee `n <= nbits` (true immediately after a successful + /// `peek`); misuse corrupts the bit position, so keep it to the hot paths + /// that peek-then-consume. `n == 0` is a no-op (the shift below is only + /// valid for `1..=64`). + #[inline] + pub fn consume(&mut self, n: u32) { + debug_assert!(n <= self.nbits); + if n == 0 { + return; + } + self.acc <<= n; + self.nbits -= n; + } + /// Read and consume `n` bits in a single call. + #[inline] pub fn read_bits(&mut self, n: u32) -> Result { + // `peek` guarantees `nbits >= n` on success, so the consume is + // check-free. let v = self.peek(n)?; - self.drop_bits(n)?; + self.consume(n); Ok(v) } @@ -102,6 +121,28 @@ impl BitReader { let _ = self.drop_bits(drop); } } + + /// Reposition the reader to an absolute byte offset in the fed buffer, + /// discarding any buffered look-ahead bits. Used when the PPMd path + /// hands control back to the bit domain: the range decoder consumed raw + /// bytes past the reader's position, and the next block header starts + /// at the byte where the range-coded data ended. + pub fn seek_byte(&mut self, pos: usize) { + self.byte_pos = pos.min(self.buf.len()); + self.acc = 0; + self.nbits = 0; + } + + /// Number of source bytes logically consumed so far. Only meaningful on + /// a byte boundary (call [`byte_align`] first). Used by the PPMd path to + /// hand the raw byte stream after the block header to the RAR range + /// decoder. + pub fn consumed_bytes(&self) -> usize { + // `byte_pos` is the next byte to pull into `acc`; `nbits` bits are + // buffered ahead but unconsumed. On a byte boundary `nbits` is a + // multiple of 8. + self.byte_pos - (self.nbits as usize) / 8 + } } #[cfg(test)] diff --git a/src/rar3/decoder.rs b/src/rar3/decoder.rs index b04dd0c..3aeac93 100644 --- a/src/rar3/decoder.rs +++ b/src/rar3/decoder.rs @@ -14,30 +14,50 @@ //! //! ## What's supported //! -//! - Non-PPMd blocks (the "LZ77 + Huffman" path used by the vast majority -//! of RAR3 archives). -//! - All five Huffman codes (precode + main + offset + low-offset + length). -//! - The 4-deep rolling-offset buffer, short offsets (codes 263..=270), and -//! the full match-length / offset machinery. -//! - The keep-table flag — successive blocks may reuse the previous code -//! lengths. +//! - The **LZ77 + Huffman path** used by the vast majority of RAR3 +//! archives: all five Huffman codes (precode, main, offset, low-offset, +//! length), the 4-deep rolling-offset buffer, short offsets (codes 263 +//! through 270), the full match-length / offset machinery, and the +//! keep-table flag (successive blocks reusing the previous code lengths). +//! - **In-band standard filters** (main symbol 257, or escape code 3 in a +//! PPMd block): Delta and x86 E8/E8E9 declarations are recognized by +//! their bytecode fingerprint and run natively over their declared +//! output windows — see `super::filters` for the recognition scheme and +//! provenance. +//! - **PPMd-II variant H blocks** (bit-0 of the block header): the full +//! PPMII model in [`crate::ppmd`] driven by the RAR range decoder, with +//! the RAR escape layer (literals, LZ matches, filters, end-of-data) on +//! top — see [`run_ppmd`]. Mid-stream switches between LZ and PPMd (the +//! `start-new-table` paths in both domains) are followed, including PPMd +//! continuation headers that reuse the live model. +//! - **Solid groups** ([`Decoder::with_solid`] + +//! [`Decoder::begin_solid_member`]): the LZ window, code tables, offset +//! history, filter programs and PPMd model persist across members. Each +//! member's payload is decoded as its own byte-aligned stream, with the +//! end-of-member markers consumed through the shared state (a PPMd +//! member's marker updates the model, exactly as the encoder's did). //! - The standalone E8/E9 post-pass filter when enabled via //! [`Decoder::with_e8_filter`]. //! //! ## What's refused //! -//! - **PPMd-II blocks** (the bit-0 flag in the block header). PPMd-II is a -//! ~1500-line context-mixed arithmetic coder; implementing it faithfully -//! is out of scope for this build. Streams containing a PPMd block fail -//! with `Error::Unsupported`. -//! - **In-band VM filter declarations** (main symbols 257..=261 that emit -//! bytecode for the RarVM interpreter). These also fail with -//! `Error::Unsupported`. The standalone E8/E9 filter remains available. -//! - **Dictionary sizes** other than the default 4 MiB. Streams compressed -//! with smaller dictionaries decode correctly with the larger window — -//! the larger window doesn't change semantics. +//! - **Cross-member range-coder state**: a PPMd block whose range coder +//! would have to straddle a solid member boundary (a member ending +//! without an end-of-data marker mid-PPMd, or a new PPMd block starting +//! exactly at the boundary via an inline table announcement). rar 6.24 +//! always ends PPMd members with a marker and starts PPMd blocks with a +//! header in the member that uses them, so these arise only in crafted +//! streams; they fail with `Error::Unsupported`. +//! - **Filter declarations carrying any other VM program** (custom +//! bytecode, or legacy standard programs no current archiver emits — +//! Itanium, RGB, the audio predictor). These fail with +//! `Error::Unsupported` rather than interpreting RarVM bytecode. +//! - **Dictionary sizes** other than the default 4 MiB for the LZ path. +//! Streams compressed with smaller dictionaries decode correctly with the +//! larger window — the larger window doesn't change semantics. use alloc::boxed::Box; +use alloc::collections::VecDeque; use alloc::vec; use alloc::vec::Vec; @@ -45,13 +65,16 @@ use crate::error::Error; use crate::traits::{RawDecoder, RawProgress}; use super::bits::BitReader; -use super::filters::apply_e8_filter; +use super::filters::{ + PendingFilter, StdProgram, apply_e8_filter, apply_pending, recognize_program, +}; use super::huffman::Huffman; use super::tables::{ DICT_DEFAULT_SIZE, HUFF_TABLE_SIZE, LENGTH_BASE, LENGTH_EXTRA_BITS, LENGTH_SIZE, LOW_OFFSET_SIZE, MAIN_SIZE, OFFSET_BASE, OFFSET_EXTRA_BITS, OFFSET_SIZE, PRECODE_SIZE, SHORT_BASE, SHORT_EXTRA_BITS, }; +use crate::ppmd::{Ppmd7, RangeDec, RangeMode}; /// Streaming RAR 3.x decoder. See module docs for the calling convention. pub struct Decoder { @@ -71,6 +94,14 @@ pub struct Decoder { e8_translate_e9: bool, /// Set on any irrecoverable error. poisoned: bool, + /// Solid-group mode: end-of-member markers are consumed (through the + /// PPMd model where applicable) and the decode context persists across + /// [`Decoder::begin_solid_member`] calls. + solid: bool, + /// The persistent decode context (window, code tables, offset history, + /// filter programs, PPMd model). Created on the first `finish`; dropped + /// after each stream unless `solid`. + ctx: Option>, } enum State { @@ -99,20 +130,16 @@ impl Decoder { e8_enabled: false, e8_translate_e9: false, poisoned: false, + solid: false, + ctx: None, } } /// Construct a decoder that will produce at most `n` uncompressed bytes. pub fn with_unpack_size(n: u64) -> Self { - Self { - state: State::Buffering { input: Vec::new() }, - out_buf: Vec::new(), - out_drained: 0, - unpack_size: n, - e8_enabled: false, - e8_translate_e9: false, - poisoned: false, - } + let mut d = Self::new(); + d.unpack_size = n; + d } /// Enable the standalone E8 (and optionally E9) filter as a post-pass. @@ -122,6 +149,40 @@ impl Decoder { self } + /// Enable solid-group mode. In a RAR3 **solid** archive the members of + /// a solid group share one compression history: the LZ window, code + /// tables, offset history, declared filter programs and any live PPMd + /// model all persist from one member to the next, while each member's + /// compressed payload is its own byte-aligned stream. Decode the first + /// member as usual, then call [`Decoder::begin_solid_member`] before + /// feeding each subsequent member. + /// + /// Solid mode also makes truncation a hard error: a member whose stream + /// ends before its declared unpacked size poisons the whole group (the + /// shared history would desync every later member), where the default + /// mode returns the short output and leaves the verdict to the caller. + pub fn with_solid(mut self) -> Self { + self.solid = true; + self + } + + /// Prepare to decode the next member of a solid group: keeps the shared + /// compression history and expects `unpack_size` uncompressed bytes from + /// the next member's compressed payload (fed via `decode`/`finish` as + /// usual). Only valid on a [`Decoder::with_solid`] decoder whose current + /// member has fully drained. + pub fn begin_solid_member(&mut self, unpack_size: u64) -> Result<(), Error> { + if self.poisoned { + return Err(Error::Corrupt); + } + if !self.solid || !matches!(self.state, State::Done) { + return Err(Error::Unsupported); + } + self.unpack_size = unpack_size; + self.state = State::Buffering { input: Vec::new() }; + Ok(()) + } + fn poison(&mut self, e: Error) -> Result { self.poisoned = true; Err(e) @@ -189,15 +250,25 @@ impl RawDecoder for Decoder { // If we still need to decode, do it now. if let State::Buffering { input } = &mut self.state { let input = core::mem::take(input); - // Move into a separate scope so the `match` borrow ends before - // we mutate `self.state`. - match run_decode( - input, - self.unpack_size, - self.e8_enabled, - self.e8_translate_e9, - ) { - Ok(out) => { + let ctx = self + .ctx + .get_or_insert_with(|| Box::new(RunCtx::new(self.unpack_size))); + ctx.unpack_size = self.unpack_size; + let result = if self.unpack_size == 0 { + Ok(Vec::new()) + } else { + run_member(ctx, &input, self.solid) + }; + match result { + Ok(mut out) => { + if self.e8_enabled { + apply_e8_filter(&mut out, 0, self.e8_translate_e9); + } + if !self.solid { + // Match the pre-solid memory profile: a one-shot + // stream has no further use for the 4 MiB window. + self.ctx = None; + } self.out_buf = out; self.out_drained = 0; self.state = State::Draining; @@ -219,59 +290,115 @@ impl RawDecoder for Decoder { self.out_buf.clear(); self.out_drained = 0; self.poisoned = false; - // unpack_size and filter flags are configuration; preserved across - // reset to match the LZX/Quantum conventions. + // A reset starts a fresh stream: any solid history is gone. + self.ctx = None; + // unpack_size, solid and filter flags are configuration; preserved + // across reset to match the LZX/Quantum conventions. } } // ─── Internal decode pipeline ───────────────────────────────────────────── -fn run_decode( - input: Vec, - unpack_size: u64, - e8_enabled: bool, - e8_translate_e9: bool, -) -> Result, Error> { - if unpack_size == 0 { - return Ok(Vec::new()); - } - let mut br = BitReader::new(); - br.feed_slice(&input); - - let mut ctx = Box::new(RunCtx { - bits: br, - // The length table survives across blocks: a successive block can - // signal "keep table" with a single header bit and reuse what was - // most recently decoded. - lengths: vec![0u8; HUFF_TABLE_SIZE], - main: None, - offset: None, - low_offset: None, - length: None, - old_offsets: [1u32, 1, 1, 1], - last_offset: 0, - last_length: 0, - last_low_offset: 0, - num_low_offset_repeats: 0, - out: Vec::new(), - window: vec![0u8; DICT_DEFAULT_SIZE], - wmask: { - debug_assert!(DICT_DEFAULT_SIZE.is_power_of_two()); - DICT_DEFAULT_SIZE - 1 - }, - window_pos: 0, - unpack_size, - }); +/// Decode one member's compressed payload against the (possibly carried- +/// over) context. In solid mode the end-of-member marker is consumed so the +/// persistent state is exactly what the next member's stream expects. +/// Cap on speculative preallocation of the output buffer, so a hostile +/// `unpack_size` can't drive a huge up-front allocation. Real members past +/// this still decode; `out` just grows the rest of the way. +const OUT_PREALLOC_CAP: u64 = 64 << 20; + +/// Bytes to reserve up front for a member's output, from its declared +/// `unpack_size`. Avoids ~log2(size) grow-and-copy reallocations while a +/// large member decodes. The `u64::MAX` sentinel means "unknown length" +/// (no declared size) — reserve nothing rather than speculate. +fn out_prealloc(unpack_size: u64) -> usize { + if unpack_size == u64::MAX { + 0 + } else { + unpack_size.min(OUT_PREALLOC_CAP) as usize + } +} + +fn run_member(ctx: &mut RunCtx, input: &[u8], solid: bool) -> Result, Error> { + // Each member's payload is its own byte-aligned stream (the container + // resets the bit input at every member boundary), so the reader is + // rebuilt even when the rest of the context carries over. + ctx.bits = BitReader::new(); + ctx.bits.feed_slice(input); + ctx.out = Vec::with_capacity(out_prealloc(ctx.unpack_size)); + // Filter *programs* persist across solid members, but scheduled filter + // instances never span a member boundary. + ctx.pending_filters.clear(); - // The decoder starts by parsing the first block header. - parse_block_header(&mut ctx)?; - expand(&mut ctx)?; + // A fresh stream — or a previous member that announced new tables — + // starts with a block header; otherwise symbol decoding continues + // directly under the carried-over tables (or PPMd model). + if !ctx.tables_read { + parse_block_header(ctx)?; + } + loop { + let seg = match ctx.block { + BlockKind::Lz => expand(ctx, solid)?, + BlockKind::Ppm => run_ppmd(ctx, input, solid)?, + }; + match seg { + Segment::MemberEnd => break, + Segment::NewTable => parse_block_header(ctx)?, + Segment::NewTableThenEnd => { + parse_block_header(ctx)?; + if matches!(ctx.block, BlockKind::Ppm) { + // A PPMd block starting exactly at the member boundary + // would prime its range coder from this member's tail + // bytes and keep pulling from the next member's payload + // — cross-member coder state we don't support (rar 6.24 + // starts such blocks with a header in the next member + // instead). + return Err(Error::Unsupported); + } + break; + } + } + } + + if solid && (ctx.out.len() as u64) < ctx.unpack_size { + // A short member desyncs the shared history for every member after + // it; fail the group rather than hand back silently-short output. + return Err(Error::UnexpectedEnd); + } - let mut out = core::mem::take(&mut ctx.out); - if e8_enabled { - apply_e8_filter(&mut out, 0, e8_translate_e9); + // Run any in-band filters whose windows the stream completed. A filter + // still pending here declared a window the stream never finished + // producing — a truncated or malformed stream. unrar in that situation + // writes the raw bytes and relies on the container CRC to flag the + // file; this crate's policy is to surface the error instead of + // returning pre-filter bytes as a success. + ctx.flush_completed_filters()?; + if !ctx.pending_filters.is_empty() { + return Err(Error::Corrupt); } - Ok(out) + + Ok(core::mem::take(&mut ctx.out)) +} + +/// How a run of symbol decoding ended. +enum Segment { + /// The member is complete (declared size produced and, in solid mode, + /// the end-of-member marker consumed). + MemberEnd, + /// An in-band "new code tables follow" boundary mid-member: parse a + /// block header and continue decoding this member. + NewTable, + /// "New code tables follow" arrived exactly at the member's declared + /// size: the tables land in this member's tail bytes and the *next* + /// member continues under them without a header of its own. + NewTableThenEnd, +} + +/// Which decoding mode the current block uses. Persists across solid +/// members: a member may continue a block the previous member started. +enum BlockKind { + Lz, + Ppm, } struct RunCtx { @@ -301,9 +428,82 @@ struct RunCtx { wmask: usize, window_pos: usize, unpack_size: u64, + /// RarVM program slots declared so far (recognized standard programs + /// only) with the per-slot remembered block length — a declaration may + /// omit the length and reuse the slot's previous one. + programs: Vec, + /// Slot used by the most recent declaration; a declaration without an + /// explicit slot field reuses it. + last_filter_slot: usize, + /// Scheduled filter instances, in declaration order. Applied (and + /// popped from the front) as soon as their windows are fully decoded — + /// see [`RunCtx::flush_completed_filters`]. + pending_filters: VecDeque, + /// The live PPMd model, if any block has created one. Persists across + /// blocks and solid members: a later PPMd block header without the + /// reset flag reuses it (with a freshly initialised range coder). + ppmd: Option>, + /// Decoding mode of the current block (LZ+Huffman or PPMd). + block: BlockKind, + /// Whether valid LZ code tables are in effect, i.e. whether the next + /// member of a solid group starts decoding symbols directly instead of + /// parsing a block header first. Cleared by PPMd block headers (a + /// member after a PPMd block always re-reads a header) and by an + /// end-of-member marker announcing new tables. + tables_read: bool, +} + +/// A live PPMd model plus the RAR escape layer's current escape byte. +struct PpmdBlock { + model: Ppmd7, + /// The RAR-layer escape byte (a decoded symbol equal to this introduces + /// a control code rather than a literal). Persists across blocks and + /// members; updated by headers carrying an explicit escape byte. + escape: u8, +} + +/// A declared filter program plus its per-slot remembered block length. +#[derive(Debug, Clone, Copy)] +struct ProgramSlot { + program: StdProgram, + last_block_length: u32, } impl RunCtx { + fn new(unpack_size: u64) -> Self { + RunCtx { + bits: BitReader::new(), + // The length table survives across blocks (and solid members): + // a successive block can signal "keep table" with a single + // header bit and delta-code against what was most recently + // decoded. + lengths: vec![0u8; HUFF_TABLE_SIZE], + main: None, + offset: None, + low_offset: None, + length: None, + old_offsets: [1u32, 1, 1, 1], + last_offset: 0, + last_length: 0, + last_low_offset: 0, + num_low_offset_repeats: 0, + out: Vec::with_capacity(out_prealloc(unpack_size)), + window: vec![0u8; DICT_DEFAULT_SIZE], + wmask: { + debug_assert!(DICT_DEFAULT_SIZE.is_power_of_two()); + DICT_DEFAULT_SIZE - 1 + }, + window_pos: 0, + unpack_size, + programs: Vec::new(), + last_filter_slot: 0, + pending_filters: VecDeque::new(), + ppmd: None, + block: BlockKind::Lz, + tables_read: false, + } + } + fn emit_literal(&mut self, b: u8) { self.out.push(b); self.window[self.window_pos] = b; @@ -329,24 +529,32 @@ impl RunCtx { self.out.reserve(length); if off == 1 { - // Distance-1 run: one repeated byte. Fill directly. + // Distance-1 run: one repeated byte, read once before any + // window write (src is behind window_pos). Bulk-fill both the + // output and the window in wrap-capped segments. let b = self.window[src]; - for _ in 0..length { - self.out.push(b); - self.window[self.window_pos] = b; - self.window_pos = (self.window_pos + 1) & wmask; + self.out.resize(self.out.len() + length, b); + let mut done = 0usize; + while done < length { + let run = (length - done).min(wlen - self.window_pos); + let sp = self.window_pos; + self.window[sp..sp + run].fill(b); + self.window_pos = (self.window_pos + run) & wmask; + done += run; } } else if off >= length { - // Non-overlapping: src and dst regions are disjoint. Copy in - // contiguous window segments (no per-byte recompute of `src`). + // Non-overlapping: src and dst regions are disjoint (or dst + // precedes src in the ring, where a forward copy is still + // exact). `off >= length >= run` and the run is capped against + // ring wrap on both cursors, so the bulk copies match the + // per-byte writes exactly — same transformation as the rar2/ + // rar5 match-copy vectorization (#115). let mut done = 0usize; while done < length { let run = (length - done).min(wlen - src).min(wlen - self.window_pos); - for k in 0..run { - let b = self.window[src + k]; - self.out.push(b); - self.window[self.window_pos + k] = b; - } + let sp = self.window_pos; + self.out.extend_from_slice(&self.window[src..src + run]); + self.window.copy_within(src..src + run, sp); src = (src + run) & wmask; self.window_pos = (self.window_pos + run) & wmask; done += run; @@ -367,8 +575,35 @@ impl RunCtx { fn done(&self) -> bool { (self.out.len() as u64) >= self.unpack_size } + + /// Apply and drop every filter at the *front* of the pending queue + /// whose window `[start, start + length)` is fully decoded. + /// + /// Only a prefix is flushed: a filter behind one whose window is still + /// incomplete stays queued even if its own window is complete, so + /// overlapping windows (filter chains) always apply in declaration + /// order — the same order unrar's stack executes them. `out` is + /// append-only and LZ back-references read the (unfiltered) window, + /// not `out`, so applying a filter as soon as its window completes is + /// equivalent to applying it at the end of the stream. + fn flush_completed_filters(&mut self) -> Result<(), Error> { + while let Some(&f) = self.pending_filters.front() { + let end = f.start + f.length as u64; + if end > self.out.len() as u64 { + break; + } + apply_pending(&f, &mut self.out[f.start as usize..end as usize])?; + self.pending_filters.pop_front(); + } + Ok(()) + } } +/// Cap on concurrently scheduled filters, matching unrar's +/// `MAX_UNPACK_FILTERS` (8192). Without a cap, a stream of tiny reused +/// declarations could grow the pending queue without bound. +const MAX_PENDING_FILTERS: usize = 8192; + // ─── Block header parsing ──────────────────────────────────────────────── fn parse_block_header(ctx: &mut RunCtx) -> Result<(), Error> { @@ -376,14 +611,61 @@ fn parse_block_header(ctx: &mut RunCtx) -> Result<(), Error> { // including the very first one (where alignment is a no-op since we // start on a byte boundary). ctx.bits.byte_align(); - // 1 bit: PPMd-block flag. We reject PPMd unconditionally. + // 1 bit: PPMd-block flag. let is_ppmd = ctx.bits.read_bits(1)?; if is_ppmd != 0 { - // PPMd-II would consume 7 more flag bits and possibly 2 more - // bytes here; we don't bother reading them since we're refusing - // the stream. - return Err(Error::Unsupported); + // PPMd block header: 7 flag bits, then (per flags) a memory byte + // and an escape byte. + // flag 0x20: model reset — read 8-bit mem → suballocator = + // (mem+1)<<20, and order = (flags & 0x1F) + 1 (values + // > 16 expand as 16 + (order-16)*3). Without 0x20 the + // block *continues* the live model from an earlier + // block or solid member (fresh range coder, same + // statistics); there must be one. + // flag 0x40: read 8-bit escape/InitEsc seed (else escape = 2 on + // reset; unchanged on continuation). + let flags = ctx.bits.read_bits(7)?; + if flags & 0x20 != 0 { + let mem_mb = ctx.bits.read_bits(8)?; + let mem_size = (mem_mb + 1).saturating_mul(1 << 20); + let mut max_order = (flags & 0x1F) + 1; + if max_order > 16 { + max_order = 16 + (max_order - 16) * 3; + } + if max_order < 2 { + return Err(Error::Corrupt); + } + let (escape, init_esc) = if flags & 0x40 != 0 { + let e = ctx.bits.read_bits(8)? as u8; + (e, Some(e)) + } else { + (2u8, None) + }; + let mut model = Ppmd7::new(mem_size)?; + model.init(max_order); + if let Some(e) = init_esc { + model.set_init_esc(e as u32); + } + ctx.ppmd = Some(Box::new(PpmdBlock { model, escape })); + } else { + let ppmd = ctx.ppmd.as_deref_mut().ok_or(Error::Corrupt)?; + if flags & 0x40 != 0 { + // An explicit escape byte updates the escape layer; the + // live model's InitEsc is a model-creation parameter and + // stays as-is. + ppmd.escape = ctx.bits.read_bits(8)? as u8; + } + // The order bits are informational on a continuation — the + // live model keeps the order it was built with. + } + ctx.bits.byte_align(); + ctx.block = BlockKind::Ppm; + // A PPMd block never leaves LZ tables in effect: after a PPMd + // member, the next member always starts with its own header. + ctx.tables_read = false; + return Ok(()); } + ctx.block = BlockKind::Lz; // 1 bit: keep-table flag. 0 ⇒ reset the persistent length table. let keep_table = ctx.bits.read_bits(1)? != 0; if !keep_table { @@ -491,15 +773,19 @@ fn parse_block_header(ctx: &mut RunCtx) -> Result<(), Error> { &ctx.lengths[MAIN_SIZE + OFFSET_SIZE + LOW_OFFSET_SIZE ..MAIN_SIZE + OFFSET_SIZE + LOW_OFFSET_SIZE + LENGTH_SIZE], )?)); + ctx.tables_read = true; Ok(()) } // ─── Expansion ─────────────────────────────────────────────────────────── -fn expand(ctx: &mut RunCtx) -> Result<(), Error> { +fn expand(ctx: &mut RunCtx, solid: bool) -> Result { loop { if ctx.done() { - return Ok(()); + if solid { + return read_member_end_lz(ctx); + } + return Ok(Segment::MemberEnd); } // Decode the next main-tree symbol. @@ -507,9 +793,14 @@ fn expand(ctx: &mut RunCtx) -> Result<(), Error> { let sym = match main_tree.decode(&mut ctx.bits) { Ok(s) => s, Err(Error::UnexpectedEnd) => { + if solid { + // A short member desyncs the group; `run_member` turns + // this into a hard error rather than short output. + return Err(Error::UnexpectedEnd); + } // Stream ran out before we've reached unpack_size. The // caller's count of output bytes is authoritative. - return Ok(()); + return Ok(Segment::MemberEnd); } Err(e) => return Err(e), }; @@ -521,24 +812,32 @@ fn expand(ctx: &mut RunCtx) -> Result<(), Error> { match sym { 256 => { - // End-of-block marker; followed by a single bit deciding - // between "this is the end of the stream" and "a new code - // table follows". The PPMd-vs-Huffman test makes one more - // bit (the "new file" flag) optional in libarchive's port, - // but unarr just reads `start_new_table` directly. We - // follow unarr here: one bit = start_new_table. + // End-of-block marker. One bit: set ⇒ new code tables + // follow immediately (the caller parses a block header — + // which may also switch this member to PPMd — and decoding + // continues). Clear ⇒ this member's data ends here; a + // second bit then announces whether the *next* member of a + // solid group starts with its own table header or keeps + // decoding under the current tables. (Framing per the + // libarchive/unarr RAR readers' descriptions, validated + // against the solid corpus.) let new_table = ctx.bits.read_bits(1)? != 0; if new_table { - parse_block_header(ctx)?; - } else { - // End of stream marker: any further bytes belong to a - // separate stream. - return Ok(()); + return Ok(Segment::NewTable); } + if solid { + let next_has_header = ctx.bits.read_bits(1)? != 0; + ctx.tables_read = !next_has_header; + } + // For a one-shot stream any further bytes belong to a + // separate stream; the second marker bit is irrelevant. + return Ok(Segment::MemberEnd); } 257 => { - // Filter program declaration — refuse. - return Err(Error::Unsupported); + // Filter declaration: a standard-program instance gets + // scheduled over a window of upcoming output; anything we + // can't run natively fails the stream (inside the parser). + read_filter_declaration(ctx)?; } 258 => { // Repeat last (offset, length). @@ -662,6 +961,385 @@ fn expand(ctx: &mut RunCtx) -> Result<(), Error> { } } +/// In solid mode, consume the end-of-member marker once the member's +/// declared size has been produced, so `tables_read` reflects what the +/// next member's stream expects. A well-formed member ends with the 256 +/// marker; a stream that simply stops (no marker) leaves the tables in +/// effect, and anything else is data beyond the declared size, which we +/// leave unread (the next member's payload is a fresh stream regardless). +fn read_member_end_lz(ctx: &mut RunCtx) -> Result { + let main_tree = ctx.main.as_ref().ok_or(Error::InvalidHuffmanTree)?; + let sym = match main_tree.decode(&mut ctx.bits) { + Ok(s) => s, + Err(Error::UnexpectedEnd) => return Ok(Segment::MemberEnd), + Err(e) => return Err(e), + }; + if sym != 256 { + return Ok(Segment::MemberEnd); + } + if ctx.bits.read_bits(1)? != 0 { + // New tables land in this member's tail; the next member continues + // symbol decoding under them directly. + return Ok(Segment::NewTableThenEnd); + } + let next_has_header = ctx.bits.read_bits(1)? != 0; + ctx.tables_read = !next_has_header; + Ok(Segment::MemberEnd) +} + +// ─── PPMd-II variant H block ───────────────────────────────────────────── + +/// Drive a RAR PPMd block: the range-coded payload (starting at the bit +/// reader's current byte position) feeds the live [`Ppmd7`] model through +/// the RAR range decoder. Decoded byte symbols are literals unless they +/// equal the escape byte, which introduces a control code (end-of-data, an +/// LZ match, a filter declaration, a new table, or a literal-escape). +/// Matches copy through the same sliding window as the LZ path so they +/// interleave seamlessly. On return the bit reader is repositioned to the +/// byte where the range-coded data ended. +fn run_ppmd(ctx: &mut RunCtx, input: &[u8], solid: bool) -> Result { + ctx.bits.byte_align(); + let payload_start = ctx.bits.consumed_bytes(); + if payload_start > input.len() { + return Err(Error::UnexpectedEnd); + } + // Take the model out of the context so the emit helpers can borrow the + // context mutably alongside it; it is put back on every path. + let mut pb = ctx.ppmd.take().ok_or(Error::Corrupt)?; + let result = run_ppmd_inner(ctx, input, payload_start, &mut pb, solid); + ctx.ppmd = Some(pb); + let (seg, end_pos) = result?; + ctx.bits.seek_byte(end_pos); + Ok(seg) +} + +/// Decode one PPMd symbol, failing closed on range-coder errors and on +/// payload overrun (a truncated payload makes the coder read fabricated +/// zero bytes). +fn ppmd_symbol(m: &mut Ppmd7, rc: &mut RangeDec) -> Result { + let s = m.decode_symbol(rc)?; + if rc.err() { + return Err(Error::Corrupt); + } + if rc.overran() { + return Err(Error::UnexpectedEnd); + } + Ok(s) +} + +fn run_ppmd_inner( + ctx: &mut RunCtx, + input: &[u8], + payload_start: usize, + pb: &mut PpmdBlock, + solid: bool, +) -> Result<(Segment, usize), Error> { + let (mut rc, _) = RangeDec::init(RangeMode::Rar, input, payload_start)?; + + loop { + if ctx.done() { + if !solid { + return Ok((Segment::MemberEnd, rc.pos())); + } + // The encoder coded this member's end marker through the model; + // decode it the same way or the shared statistics desync from + // the encoder's for every later member. If the payload is + // exhausted right here instead, the member boundary splits a + // still-running range coder across payloads — cross-member + // coder state we don't support (rar 6.24 ends PPMd members + // with an explicit marker). + let s = ppmd_symbol(&mut pb.model, &mut rc).map_err(|e| match e { + Error::UnexpectedEnd => Error::Unsupported, + other => other, + })?; + if s != pb.escape { + return Err(Error::Corrupt); + } + let code = ppmd_symbol(&mut pb.model, &mut rc)?; + return match code { + 2 => Ok((Segment::MemberEnd, rc.pos())), + 0 => Ok((Segment::NewTableThenEnd, rc.pos())), + _ => Err(Error::Corrupt), + }; + } + let s = ppmd_symbol(&mut pb.model, &mut rc)?; + if s != pb.escape { + ctx.emit_literal(s); + continue; + } + let code = ppmd_symbol(&mut pb.model, &mut rc)?; + match code { + 0 => { + // start-new-table: a fresh block header follows in the bit + // domain at the coder's byte position (it may keep PPMd + // with or without a model reset, or switch back to LZ). + return Ok((Segment::NewTable, rc.pos())); + } + 2 => { + // End of PPMd data before the declared size: short member. + // Solid mode turns this into an error in `run_member`. + return Ok((Segment::MemberEnd, rc.pos())); + } + 3 => { + // A filter declaration carried in the PPMd stream: the same + // wire layout as main symbol 257, with every byte decoded + // through the model. + read_filter_declaration_ppmd(ctx, &mut pb.model, &mut rc)?; + } + 4 => { + // 24-bit distance from three symbols (big-endian), then a + // length symbol. Distance +2, length +32. + let mut dist = 0u32; + for i in (0..3).rev() { + let b = ppmd_symbol(&mut pb.model, &mut rc)? as u32; + dist |= b << (i * 8); + } + let len = ppmd_symbol(&mut pb.model, &mut rc)? as u32; + ctx.emit_match(dist + 2, len + 32)?; + } + 5 => { + // Distance-1 run: length symbol, length +4. + let len = ppmd_symbol(&mut pb.model, &mut rc)? as u32; + ctx.emit_match(1, len + 4)?; + } + _ => { + // Any other control code encodes a literal equal to the + // escape byte (the control symbol is consumed and dropped). + ctx.emit_literal(pb.escape); + } + } + } +} + +/// Parse a filter declaration whose bytes arrive as PPMd symbols (escape +/// code 3): an 8-bit flags byte, a 1/2/3-byte length field, then `length` +/// payload bytes forming the same self-contained declaration payload the +/// bit-domain parser (main symbol 257) reads. +fn read_filter_declaration_ppmd( + ctx: &mut RunCtx, + model: &mut Ppmd7, + rc: &mut RangeDec, +) -> Result<(), Error> { + let flags = ppmd_symbol(model, rc)? as u32; + let mut decl_len = (flags & 0x07) + 1; + if decl_len == 7 { + decl_len = ppmd_symbol(model, rc)? as u32 + 7; + } else if decl_len == 8 { + let hi = ppmd_symbol(model, rc)? as u32; + let lo = ppmd_symbol(model, rc)? as u32; + decl_len = (hi << 8) | lo; + } + if decl_len == 0 { + return Err(Error::Corrupt); + } + let mut payload = vec![0u8; decl_len as usize]; + for b in payload.iter_mut() { + *b = ppmd_symbol(model, rc)?; + } + let mut db = BitReader::new(); + db.feed_slice(&payload); + parse_declaration_payload(ctx, flags, &mut db).map_err(|e| match e { + Error::UnexpectedEnd => Error::Corrupt, + other => other, + }) +} + +// ─── In-band filter declarations (main symbol 257) ────────────────────── + +/// Upper bound on a filter's block length: the RarVM memory the standard +/// programs operate in (0x40000 bytes — modern unrar lets a block use all +/// of it). Delta needs separate source and destination halves, so its +/// windows are capped at half that. Real encoders stay far below both caps +/// and split large regions into several filter blocks. Beyond the cap +/// UnRAR 7.23 skips the transform and emits the raw bytes with success; +/// this crate fails closed instead (same policy as unfinished windows). +const FILTER_MAX_BLOCK: u32 = 0x40000; +const FILTER_MAX_BLOCK_DELTA: u32 = 0x20000; + +/// Read a RarVM variable-length number: a 2-bit tag selects a 4-, 8- +/// (with a sign-extension-style escape for values below 16), 16- or 32-bit +/// payload. +fn read_vm_number(bits: &mut BitReader) -> Result { + Ok(match bits.read_bits(2)? { + 0 => bits.read_bits(4)?, + 1 => { + let v = bits.read_bits(8)?; + if v >= 16 { + v + } else { + 0xFFFF_FF00 | (v << 4) | bits.read_bits(4)? + } + } + 2 => bits.read_bits(16)?, + _ => bits.read_bits(32)?, + }) +} + +/// Parse the declaration that follows main symbol 257 and schedule the +/// filter it describes. +/// +/// Wire layout (validated bit-exact against rar 6.24 archives; see the +/// module docs in `filters.rs` for provenance): an 8-bit flags byte and a +/// 1/2/3-byte length field are read from the main bitstream, then `length` +/// payload bytes (8 bits each, unaligned). The payload forms its own +/// MSB-first bit domain containing, in order: +/// +/// 1. flags bit 7: a program-slot number (RarVM number; 0 resets all +/// declared programs and selects slot 0, n>0 selects slot n-1). Absent → +/// reuse the most recent slot. +/// 2. Window start relative to the current output position (RarVM number; +/// flags bit 6 adds 258). +/// 3. flags bit 5: explicit window length (RarVM number). Absent → the +/// slot's remembered length. +/// 4. flags bit 4: a 7-bit register mask followed by a RarVM number per set +/// bit (registers r0..r6; Delta receives its channel count in r0). +/// 5. For a first-use slot: bytecode as a RarVM number length plus that +/// many bytes, the first being an XOR checksum of the rest. +/// 6. flags bit 3: trailing global data — not needed by any standard +/// program, ignored here. +fn read_filter_declaration(ctx: &mut RunCtx) -> Result<(), Error> { + let flags = ctx.bits.read_bits(8)?; + let mut decl_len = (flags & 0x07) + 1; + if decl_len == 7 { + decl_len = ctx.bits.read_bits(8)? + 7; + } else if decl_len == 8 { + decl_len = ctx.bits.read_bits(16)?; + } + if decl_len == 0 { + return Err(Error::Corrupt); + } + let mut payload = vec![0u8; decl_len as usize]; + for b in payload.iter_mut() { + *b = ctx.bits.read_bits(8)? as u8; + } + // The payload is its own bit domain; running out of payload bits means + // the declaration is malformed, not that the caller should feed more + // input, so map UnexpectedEnd to Corrupt. + let mut db = BitReader::new(); + db.feed_slice(&payload); + parse_declaration_payload(ctx, flags, &mut db).map_err(|e| match e { + Error::UnexpectedEnd => Error::Corrupt, + other => other, + }) +} + +fn parse_declaration_payload( + ctx: &mut RunCtx, + flags: u32, + db: &mut BitReader, +) -> Result<(), Error> { + let slot = if flags & 0x80 != 0 { + let v = read_vm_number(db)?; + if v == 0 { + // Full reset (unrar's InitFilters30): cancel every scheduled + // filter — *including* ones whose windows are complete but not + // yet applied. unrar executes filters only when it flushes + // decoded output, which lags decoding by up to a window, so a + // reset discards them and their windows stay raw bytes. (For + // multi-window outputs the reference may have flushed — and + // applied — earlier filters before the reset; real encoders + // only reset at stream start, so that corner stays unmodeled.) + ctx.pending_filters.clear(); + ctx.programs.clear(); + 0 + } else { + (v - 1) as usize + } + } else { + ctx.last_filter_slot + }; + // A slot may reference an existing program or append exactly one new + // one; skipping ahead is malformed. + if slot > ctx.programs.len() { + return Err(Error::Corrupt); + } + ctx.last_filter_slot = slot; + + let mut start = read_vm_number(db)? as u64; + if flags & 0x40 != 0 { + start += 258; + } + let start = ctx.out.len() as u64 + start; + + let explicit_length = if flags & 0x20 != 0 { + Some(read_vm_number(db)?) + } else { + None + }; + + // Registers r0..r6. Only r0 matters to the standard transforms (Delta's + // channel count), but all present values must be consumed to stay in + // sync with the fields that follow. + let mut r0 = 0u32; + if flags & 0x10 != 0 { + let mask = db.read_bits(7)?; + for r in 0..7 { + if mask & (1 << r) != 0 { + let v = read_vm_number(db)?; + if r == 0 { + r0 = v; + } + } + } + } + + if slot == ctx.programs.len() { + // First use of this slot: bytecode follows. + let code_len = read_vm_number(db)?; + if code_len == 0 || code_len >= 0x1_0000 { + return Err(Error::Corrupt); + } + let mut code = vec![0u8; code_len as usize]; + for b in code.iter_mut() { + *b = db.read_bits(8)? as u8; + } + // The first bytecode byte is an XOR checksum of the rest. + let checksum = code[1..].iter().fold(0u8, |acc, &b| acc ^ b); + if checksum != code[0] { + return Err(Error::Corrupt); + } + let program = recognize_program(&code).ok_or(Error::Unsupported)?; + ctx.programs.push(ProgramSlot { + program, + last_block_length: 0, + }); + } + // (flags bit 3: global data would follow here; no standard program + // reads it, so it stays unparsed — the payload is self-contained.) + + let length = explicit_length.unwrap_or(ctx.programs[slot].last_block_length); + ctx.programs[slot].last_block_length = length; + if length == 0 { + // A zero-length window is a no-op declaration. + return Ok(()); + } + let program = ctx.programs[slot].program; + let cap = if program == StdProgram::Delta { + FILTER_MAX_BLOCK_DELTA + } else { + FILTER_MAX_BLOCK + }; + if length > cap { + return Err(Error::Corrupt); + } + ctx.pending_filters.push_back(PendingFilter { + start, + length, + program, + channels: r0, + }); + if ctx.pending_filters.len() > MAX_PENDING_FILTERS { + // Match unrar's cap on concurrently scheduled filters: try to + // drain completed windows first; a stream that still exceeds the + // cap is hostile or malformed. + ctx.flush_completed_filters()?; + if ctx.pending_filters.len() > MAX_PENDING_FILTERS { + return Err(Error::Corrupt); + } + } + Ok(()) +} + /// Promote the offset at `idx` (in the rolling buffer) to position 0, /// shifting everything above it down. Used by symbols 259..=262. fn promote_offset(ctx: &mut RunCtx, idx: usize, offs: u32) { @@ -681,26 +1359,56 @@ mod tests { extern crate std; use std::vec; - #[test] - fn unpack_size_zero_is_immediate_done() { - let mut dec = Decoder::with_unpack_size(0); - let mut out = [0u8; 8]; - let (p, status) = dec.finish(&mut out).unwrap(); - assert_eq!(p.written, 0); - assert!(matches!(status, crate::Status::StreamEnd)); + /// MSB-first bit writer used to hand-build declaration payloads. + struct BitWriter { + bytes: std::vec::Vec, + nbits: u32, + } + impl BitWriter { + fn new() -> Self { + Self { + bytes: std::vec::Vec::new(), + nbits: 0, + } + } + fn push(&mut self, value: u32, n: u32) { + for i in (0..n).rev() { + let bit = ((value >> i) & 1) as u8; + if self.nbits.is_multiple_of(8) { + self.bytes.push(0); + } + let last = self.bytes.len() - 1; + self.bytes[last] |= bit << (7 - (self.nbits % 8)); + self.nbits += 1; + } + } + /// Push a value in RarVM variable-number encoding (shortest form). + fn push_vm_number(&mut self, v: u32) { + if v < 16 { + self.push(0, 2); + self.push(v, 4); + } else if v < 256 { + self.push(1, 2); + self.push(v, 8); + } else if v < 0x1_0000 { + self.push(2, 2); + self.push(v, 16); + } else { + self.push(3, 2); + self.push(v, 32); + } + } } - #[test] - fn promote_offset_rotates_correctly() { - // Construct a context-shaped struct just to test the helper. - let mut ctx = RunCtx { + fn test_ctx() -> RunCtx { + RunCtx { bits: BitReader::new(), lengths: vec![], main: None, offset: None, low_offset: None, length: None, - old_offsets: [10, 20, 30, 40], + old_offsets: [1, 1, 1, 1], last_offset: 0, last_length: 0, last_low_offset: 0, @@ -710,7 +1418,282 @@ mod tests { wmask: 15, window_pos: 0, unpack_size: 0, - }; + programs: vec![], + last_filter_slot: 0, + pending_filters: VecDeque::new(), + ppmd: None, + block: BlockKind::Lz, + tables_read: false, + } + } + + #[test] + fn vm_number_all_tag_widths() { + let mut w = BitWriter::new(); + w.push_vm_number(9); // tag 0, 4-bit + w.push_vm_number(200); // tag 1, 8-bit (>= 16) + w.push_vm_number(0x1234); // tag 2, 16-bit + w.push_vm_number(0x0102_0304); // tag 3, 32-bit + // tag 1 with an 8-bit value below 16: extends to 0xFFFFFF00-form. + w.push(1, 2); + w.push(5, 8); + w.push(0xA, 4); + let mut r = BitReader::new(); + r.feed_slice(&w.bytes); + assert_eq!(read_vm_number(&mut r).unwrap(), 9); + assert_eq!(read_vm_number(&mut r).unwrap(), 200); + assert_eq!(read_vm_number(&mut r).unwrap(), 0x1234); + assert_eq!(read_vm_number(&mut r).unwrap(), 0x0102_0304); + assert_eq!(read_vm_number(&mut r).unwrap(), 0xFFFF_FF5A); + } + + /// Build the payload of a declaration introducing a fresh program. + /// `code` must carry a valid XOR checksum byte already. + fn new_program_payload(block_start: u32, block_len: u32, code: &[u8]) -> std::vec::Vec { + let mut w = BitWriter::new(); + w.push_vm_number(0); // slot field: reset-all + slot 0 + w.push_vm_number(block_start); + w.push_vm_number(block_len); + w.push_vm_number(code.len() as u32); + for &b in code { + w.push(b as u32, 8); + } + w.bytes + } + + /// Wrap arbitrary bytecode with its XOR checksum byte. + fn with_checksum(body: &[u8]) -> std::vec::Vec { + let mut code = std::vec::Vec::with_capacity(body.len() + 1); + code.push(body.iter().fold(0u8, |a, &b| a ^ b)); + code.extend_from_slice(body); + code + } + + /// The 29-byte standard Delta program as WinRAR emits it, lifted from + /// `tests/fixtures/rar3/filter_delta_gradient_bmp.bin` (rar 6.24 + /// archive of gradient.bmp). CRC-32 0x0E06077D; byte 0 is the XOR + /// checksum of the rest. + const DELTA_PROG: [u8; 29] = [ + 0x2F, 0x01, 0x9A, 0x41, 0x80, 0xEC, 0x27, 0x48, 0x2F, 0x09, 0x76, 0x6D, 0xD3, 0xEA, 0x41, + 0x5B, 0x59, 0x44, 0xE8, 0x17, 0x5C, 0xE1, 0x6C, 0x91, 0x4C, 0x4E, 0x3F, 0x77, 0x00, + ]; + + fn incomplete_filter(start: u64) -> PendingFilter { + PendingFilter { + start, + length: 100, + program: StdProgram::Delta, + channels: 1, + } + } + + #[test] + fn delta_program_is_recognized() { + assert_eq!(recognize_program(&DELTA_PROG), Some(StdProgram::Delta)); + } + + /// A reset declaration (slot field 0) cancels every scheduled filter — + /// including completed-but-unapplied windows. unrar's InitFilters30 + /// discards its whole filter stack (execution happens at output-flush + /// time, which lags decoding), so those windows stay raw bytes; a + /// canceled filter must never rewrite output. + #[test] + fn reset_cancels_all_pending_without_applying() { + let mut ctx = test_ctx(); + ctx.out = vec![1, 0, 0, 0, 9, 9, 9, 9]; + ctx.programs.push(ProgramSlot { + program: StdProgram::Delta, + last_block_length: 4, + }); + // A completed-but-unapplied window plus an incomplete one. + ctx.pending_filters.push_back(PendingFilter { + start: 0, + length: 4, + program: StdProgram::Delta, + channels: 1, + }); + ctx.pending_filters.push_back(incomplete_filter(4)); + + // Payload: slot reset, block_start 0, explicit length 4, r0 = 1 + // channel, then the (new, post-reset) Delta program bytecode. + let mut w = BitWriter::new(); + w.push_vm_number(0); + w.push_vm_number(0); + w.push_vm_number(4); + w.push(0x01, 7); // register mask: r0 only + w.push_vm_number(1); + w.push_vm_number(DELTA_PROG.len() as u32); + for &b in &DELTA_PROG { + w.push(b as u32, 8); + } + let mut db = BitReader::new(); + db.feed_slice(&w.bytes); + parse_declaration_payload(&mut ctx, 0xB0, &mut db).unwrap(); + + // Nothing ran: both prior filters were canceled outright. + assert_eq!(ctx.out, vec![1, 0, 0, 0, 9, 9, 9, 9]); + // Only the fresh declaration (window at out position 8) is + // scheduled against the fresh slot table. + assert_eq!(ctx.pending_filters.len(), 1); + assert_eq!(ctx.pending_filters[0].start, 8); + assert_eq!(ctx.programs.len(), 1); + } + + /// The pending queue is capped (unrar's MAX_UNPACK_FILTERS): once no + /// completed window can be drained, further declarations are corrupt. + #[test] + fn pending_filter_cap_is_enforced() { + let mut ctx = test_ctx(); + ctx.programs.push(ProgramSlot { + program: StdProgram::Delta, + last_block_length: 5, + }); + for _ in 0..MAX_PENDING_FILTERS { + ctx.pending_filters.push_back(incomplete_filter(1_000_000)); + } + // Reuse-slot declaration: block_start only, remembered length. + let mut w = BitWriter::new(); + w.push_vm_number(0); + let mut db = BitReader::new(); + db.feed_slice(&w.bytes); + assert_eq!( + parse_declaration_payload(&mut ctx, 0x00, &mut db), + Err(Error::Corrupt) + ); + } + + /// A declaration without an explicit length (flags bit 5 clear) reuses + /// the slot's remembered length from the previous declaration. + #[test] + fn slot_reuse_inherits_remembered_length() { + let mut ctx = test_ctx(); + // First declaration: fresh Delta program, explicit length 4. + let mut w = BitWriter::new(); + w.push_vm_number(0); + w.push_vm_number(0); + w.push_vm_number(4); + w.push(0x01, 7); + w.push_vm_number(1); + w.push_vm_number(DELTA_PROG.len() as u32); + for &b in &DELTA_PROG { + w.push(b as u32, 8); + } + let mut db = BitReader::new(); + db.feed_slice(&w.bytes); + parse_declaration_payload(&mut ctx, 0xB0, &mut db).unwrap(); + + // Second declaration: no slot field, no explicit length — inherits + // slot 0's remembered length; window 16 bytes further out. + let mut w = BitWriter::new(); + w.push_vm_number(16); + let mut db = BitReader::new(); + db.feed_slice(&w.bytes); + parse_declaration_payload(&mut ctx, 0x00, &mut db).unwrap(); + + assert_eq!(ctx.pending_filters.len(), 2); + assert_eq!(ctx.pending_filters[1].start, 16); + assert_eq!(ctx.pending_filters[1].length, 4); + } + + /// Only a *prefix* of completed windows may flush: a completed filter + /// queued behind an incomplete one must wait so that overlapping + /// windows always apply in declaration order. + #[test] + fn flush_is_prefix_ordered() { + let mut ctx = test_ctx(); + ctx.out = vec![7; 8]; + ctx.pending_filters.push_back(incomplete_filter(4)); + ctx.pending_filters.push_back(PendingFilter { + start: 0, + length: 4, + program: StdProgram::Delta, + channels: 1, + }); + ctx.flush_completed_filters().unwrap(); + assert_eq!(ctx.pending_filters.len(), 2, "nothing may flush"); + assert_eq!(ctx.out, vec![7; 8], "output must be untouched"); + } + + #[test] + fn unknown_program_is_unsupported() { + // Valid declaration framing around bytecode we don't recognize + // (flags: slot present + explicit length = 0xA0). + let code = with_checksum(&[0x12, 0x34, 0x56, 0x78]); + let payload = new_program_payload(0, 64, &code); + let mut ctx = test_ctx(); + let mut db = BitReader::new(); + db.feed_slice(&payload); + assert_eq!( + parse_declaration_payload(&mut ctx, 0xA0, &mut db), + Err(Error::Unsupported) + ); + } + + #[test] + fn bad_bytecode_checksum_is_corrupt() { + let mut code = with_checksum(&[0x12, 0x34]); + code[0] ^= 0xFF; // break the checksum + let payload = new_program_payload(0, 64, &code); + let mut ctx = test_ctx(); + let mut db = BitReader::new(); + db.feed_slice(&payload); + assert_eq!( + parse_declaration_payload(&mut ctx, 0xA0, &mut db), + Err(Error::Corrupt) + ); + } + + #[test] + fn slot_skipping_ahead_is_corrupt() { + // Slot field 3 => slot index 2 with no programs declared. + let mut w = BitWriter::new(); + w.push_vm_number(3); + let mut ctx = test_ctx(); + let mut db = BitReader::new(); + db.feed_slice(&w.bytes); + assert_eq!( + parse_declaration_payload(&mut ctx, 0x80, &mut db), + Err(Error::Corrupt) + ); + } + + #[test] + fn truncated_main_stream_mid_declaration_is_unexpected_end() { + // flags byte 0x86 declares (6&7)+1 = 7 → an extra 8-bit length + // field must follow, but the stream ends first. + let mut ctx = test_ctx(); + ctx.bits.feed_slice(&[0x86]); + assert!(matches!( + read_filter_declaration(&mut ctx), + Err(Error::UnexpectedEnd) + )); + } + + #[test] + fn payload_bits_running_out_is_corrupt() { + // flags 0xA0, decl_len 1, payload [0xFF]: the slot field's 2-bit + // tag reads 0b11 → a 32-bit number that the 1-byte payload can't + // hold. Inside the payload's own bit domain that's a malformed + // declaration, so the wrapper maps it to Corrupt. + let mut ctx = test_ctx(); + ctx.bits.feed_slice(&[0xA0, 0xFF]); + assert_eq!(read_filter_declaration(&mut ctx), Err(Error::Corrupt)); + } + + #[test] + fn unpack_size_zero_is_immediate_done() { + let mut dec = Decoder::with_unpack_size(0); + let mut out = [0u8; 8]; + let (p, status) = dec.finish(&mut out).unwrap(); + assert_eq!(p.written, 0); + assert!(matches!(status, crate::Status::StreamEnd)); + } + + #[test] + fn promote_offset_rotates_correctly() { + // Construct a context-shaped struct just to test the helper. + let mut ctx = test_ctx(); + ctx.old_offsets = [10, 20, 30, 40]; // Promote slot 2 (value 30) — result should be [30, 10, 20, 40]. promote_offset(&mut ctx, 2, 30); assert_eq!(ctx.old_offsets, [30, 10, 20, 40]); diff --git a/src/rar3/filters.rs b/src/rar3/filters.rs index d04f72d..0a6eed6 100644 --- a/src/rar3/filters.rs +++ b/src/rar3/filters.rs @@ -1,21 +1,112 @@ //! RAR 3.x post-decompression filters. //! -//! RAR3 supports a small set of "VM filters" that the encoder embeds as -//! bytecode programs in symbol 257 of the main code. The decoder is then -//! supposed to instantiate an interpreter for a stack-based RISC-like VM -//! ("RarVM"). Faithfully implementing RarVM is a large effort (the -//! upstream interpreter is several hundred lines plus a full instruction -//! set) and is out of scope for this build. +//! ## In-band standard filters (main symbol 257) //! -//! What we do support is the **stand-alone Intel E8/E9 x86 call translation -//! filter** which can be activated through an external selector -//! ([`Decoder::with_e8_filter`]). This filter is what the vast majority of -//! RAR3 streams over x86 executables actually use, and the operation is -//! the same as the LZX intel-call-translation post-pass. +//! RAR3 embeds filters as bytecode programs for a stack-based RISC-like VM +//! ("RarVM") in symbol 257 of the main code. We do **not** interpret +//! arbitrary programs: WinRAR's compressor only ever emits a fixed set of +//! standard programs, so — like libarchive and unrar in practice — we +//! recognize the standard programs (by bytecode length + CRC-32, +//! [`recognize_program`]) and run native transforms: //! -//! Future versions of this module may grow Itanium, RGB delta and audio -//! delta filters if there's demand; for now any in-band filter declaration -//! is refused with `Error::Unsupported`. +//! - **Delta** (channel de-interleave; emitted for bitmaps, WAV audio and +//! other channel-interleaved content; channel count arrives in VM +//! register 0), +//! - **x86 E8** and **E8/E9** call(-jump) translation. +//! +//! The transforms themselves live in [`crate::rar_filters`], shared with +//! the RAR5 decoder, and were validated byte-for-byte against WinRAR +//! archives (UnRAR 7.23 ground truth). Streams declaring any other program +//! (custom bytecode, or the legacy Itanium/RGB/audio-predictor standard +//! programs, which current archivers no longer emit) fail with +//! `Error::Unsupported` — never wrong bytes. +//! +//! ## Stand-alone E8/E9 pass +//! +//! Separately, the **stand-alone Intel E8/E9 call translation pass** +//! ([`apply_e8_filter`]) can be activated by the caller via +//! `Decoder::with_e8_filter`. It predates in-band filter support and is an +//! LZX-style whole-output transform, *not* the same arithmetic as the +//! in-band x86 filter; it is kept for callers that relied on it. + +use crate::checksum::Crc32; +use crate::error::Error; +use crate::rar_filters::{delta_decode, x86_e8_decode}; + +/// A standard RarVM filter program we recognize and can run natively. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum StdProgram { + /// Channel de-interleave; channel count comes from VM register 0. + Delta, + /// x86 `0xE8` (CALL) relative-address restore. + X86Call, + /// x86 `0xE8`/`0xE9` (CALL/JMP) relative-address restore. + X86CallJmp, +} + +/// Identify a standard filter program from its bytecode. +/// +/// WinRAR's compressor emits each standard filter as a fixed byte string, +/// so `(length, CRC-32)` is a stable fingerprint — the same recognition +/// scheme libarchive uses. The Delta and x86-E8 fingerprints below were +/// computed from programs extracted out of real rar 6.24 archives (this +/// crate's differential corpus); the E8E9 fingerprint (emitted by older +/// WinRAR 3.x builds) matches the value documented in libarchive +/// (BSD-licensed `archive_read_support_format_rar.c`). +/// +/// Returns `None` for anything else — including the legacy Itanium / RGB / +/// audio-predictor standard programs, which no current archiver emits. +pub(super) fn recognize_program(code: &[u8]) -> Option { + let mut crc = Crc32::new(); + crc.update(code); + match (code.len(), crc.finalize()) { + (29, 0x0E06_077D) => Some(StdProgram::Delta), + (53, 0xAD57_6887) => Some(StdProgram::X86Call), + (57, 0x3CD7_E57E) => Some(StdProgram::X86CallJmp), + _ => None, + } +} + +/// A parsed, scheduled instance of a standard filter: it rewrites the +/// window `[start, start + length)` of the unpacked stream. +#[derive(Debug, Clone, Copy)] +pub(super) struct PendingFilter { + /// Absolute byte offset in the unpacked stream. + pub start: u64, + pub length: u32, + pub program: StdProgram, + /// VM register 0 at declaration time — the Delta channel count. + pub channels: u32, +} + +/// Delta channel-count ceiling, matching unrar's `MAX3_UNPACK_CHANNELS` +/// (1024). unrar refuses to *run* the transform beyond it and emits the +/// raw bytes with success; this crate fails closed instead (same policy as +/// unfinished filter windows — surfacing an error beats returning bytes +/// that only a container CRC could flag). +const MAX_DELTA_CHANNELS: u32 = 1024; + +/// Run a scheduled filter over its region (already sliced by the caller). +/// +/// The x86 transforms use the **unmasked** 32-bit position base (RAR3 VM +/// semantics — see [`x86_e8_decode`]); `filter.start` is file-relative, +/// which for solid archives means member-relative (unrar seeds the VM with +/// its per-member written-size counter). +pub(super) fn apply_pending(filter: &PendingFilter, region: &mut [u8]) -> Result<(), Error> { + match filter.program { + StdProgram::Delta => { + if filter.channels == 0 || filter.channels > MAX_DELTA_CHANNELS { + return Err(Error::Corrupt); + } + // More channels than bytes is well-defined (trailing planes are + // empty) and unrar runs it; no length-based bound here. + delta_decode(filter.channels as usize, region); + } + StdProgram::X86Call => x86_e8_decode(filter.start, region, false, false), + StdProgram::X86CallJmp => x86_e8_decode(filter.start, region, true, false), + } + Ok(()) +} /// Apply the E8/E9 (x86 near-call) translation filter to `data` in place. /// diff --git a/src/rar3/huffman.rs b/src/rar3/huffman.rs index 140d9ec..3ce1452 100644 --- a/src/rar3/huffman.rs +++ b/src/rar3/huffman.rs @@ -155,6 +155,12 @@ impl Huffman { return Err(Error::InvalidHuffmanTree); } let max = self.max_length as u32; + // Slow-path scan lower bound. A LUT miss below (peek succeeded but no + // code ≤ PRIMARY_BITS matched) proves the code is longer than + // PRIMARY_BITS, so the canonical scan can skip lengths 1..=PRIMARY_BITS + // entirely. It stays 1 only when the fast-path peek itself failed + // (stream near end), where a short code may still be valid. + let mut min_len = 1u32; // Fast path: when we can peek PRIMARY_BITS bits, a single LUT // lookup resolves any code of length ≤ PRIMARY_BITS. @@ -162,17 +168,20 @@ impl Huffman { let entry = self.lut[idx as usize]; let len = (entry >> LUT_LEN_SHIFT) as u32; if len > 0 { - reader.drop_bits(len)?; + // `peek(PRIMARY_BITS)` succeeded and `len <= PRIMARY_BITS`, so + // the bits are buffered — consume without re-checking. + reader.consume(len); return Ok(entry & LUT_SYM_MASK); } // Long code (> PRIMARY_BITS) -- fall through to the slow path. + min_len = PRIMARY_BITS + 1; } // Peek `max` bits; if not enough, peek the remaining smaller widths // one at a time. For RAR3 trees this is unlikely to matter -- most // codes fit in the buffer easily. let lookahead = self.peek_padded(reader, max)?; - for length in 1..=max { + for length in min_len..=max { let code = lookahead >> (max - length); let count = self.counts[length as usize] as u32; if count > 0 { diff --git a/src/rar3/mod.rs b/src/rar3/mod.rs index e0b6244..2620f7c 100644 --- a/src/rar3/mod.rs +++ b/src/rar3/mod.rs @@ -18,11 +18,17 @@ //! 2. **PPMd-II** — an Order-N context-mixed arithmetic coder. Used by some //! text-heavy archives and `-m5` (best compression) runs. //! -//! This build implements the **LZ77 + Huffman path** in full. PPMd-II blocks -//! are refused with `Error::Unsupported` — see the private `decoder` submodule for details and -//! limitations. The standalone E8/E9 (x86 near-call) post-pass filter can -//! be enabled via [`Decoder::with_e8_filter`]; the in-band RarVM filter -//! mechanism (main symbols 257..=261) is refused. +//! This build implements the **LZ77 + Huffman path** in full, including the +//! in-band standard filters WinRAR declares via main symbol 257 or PPMd +//! escape code 3 (Delta and x86 E8/E8E9, recognized by bytecode fingerprint +//! and run natively — no RarVM interpreter; unknown programs are refused), +//! **PPMd-II variant H** blocks (the full PPMII model in [`crate::ppmd`], +//! driven by the RAR range decoder with the RAR literal/match/end-of-data +//! escape layer), mid-stream switches between the two, and **solid +//! multi-member groups** (shared LZ window, code tables, filter programs +//! and PPMd model across members — see [`Decoder::with_solid`] / +//! [`Decoder::begin_solid_member`]). The standalone E8/E9 (x86 near-call) +//! post-pass filter can also be enabled via [`Decoder::with_e8_filter`]. //! //! ## Calling convention //! @@ -48,6 +54,12 @@ //! } //! ``` //! +//! For a **solid group**, construct the decoder with +//! [`Decoder::with_solid`], decode the first member as above, then call +//! [`Decoder::begin_solid_member`] with the next member's unpacked size +//! before feeding its payload — each member's payload is its own +//! byte-aligned stream, but the compression history carries over. +//! //! ## References //! //! - libarchive `archive_read_support_format_rar.c` (BSD): structure and diff --git a/src/rar5/decoder.rs b/src/rar5/decoder.rs index af8a7ea..0cb0981 100644 --- a/src/rar5/decoder.rs +++ b/src/rar5/decoder.rs @@ -48,6 +48,11 @@ //! container framing themselves (header blocks, file headers, multi-volume //! continuations, etc.) and hand the inner compressed-data run to this //! decoder. +//! +//! When the run is a **solid group** (several files sharing one LZ +//! stream), the container should also register each member's starting +//! offset via [`Decoder::add_file_boundary`] so the x86 filters can +//! compute file-relative addresses, as unrar does. use alloc::boxed::Box; use alloc::collections::VecDeque; @@ -107,6 +112,12 @@ pub struct Decoder { /// Absolute offset (in the unpacked stream) of the first byte still in /// `out_queue`. Used to identify which pending filters fire when. out_queue_start: u64, + /// Sorted starting offsets (in the unpacked stream) of the files a + /// solid group concatenates, registered via + /// [`Decoder::add_file_boundary`]. Position-dependent filters (x86) + /// compute their addresses relative to the containing file, matching + /// unrar/libarchive. An implicit boundary at 0 always exists. + file_boundaries: Vec, } #[derive(Debug)] @@ -171,6 +182,39 @@ impl Decoder { pending_filters: Vec::new(), ready: VecDeque::new(), out_queue_start: 0, + file_boundaries: Vec::new(), + } + } + + /// Declare that a new file starts at absolute unpacked-stream offset + /// `pos` — needed when this decoder runs over a **solid group** (one + /// continuous LZ stream concatenating several files). + /// + /// The x86 E8/E8E9 filters rewrite call targets relative to the start + /// of the *containing file*, not the solid stream (unrar resets its + /// position base per extracted file; libarchive tracks the same value + /// as `solid_offset`). A container driving a solid group should + /// register each member's starting offset (the cumulative unpacked + /// sizes of the members before it) before decoding; boundaries may be + /// registered ahead of time and in any order. Without registrations + /// every filter is file-relative to offset 0, which is correct for the + /// single-file (non-solid) case. + /// + /// Like the unpack size, boundaries are treated as stream-shape + /// configuration: they survive [`reset`](crate::Decoder::reset). + pub fn add_file_boundary(&mut self, pos: u64) { + if let Err(idx) = self.file_boundaries.binary_search(&pos) { + self.file_boundaries.insert(idx, pos); + } + } + + /// Starting offset of the file containing unpacked-stream position + /// `pos` (0 when no boundary at or before `pos` is registered). + fn file_start_for(&self, pos: u64) -> u64 { + match self.file_boundaries.binary_search(&pos) { + Ok(_) => pos, + Err(0) => 0, + Err(idx) => self.file_boundaries[idx - 1], } } @@ -308,6 +352,9 @@ impl RawDecoder for Decoder { self.pending_filters.clear(); self.ready.clear(); self.out_queue_start = 0; + // A reset starts an unrelated stream: stale solid-group boundaries + // would skew every later x86 filter's position base. + self.file_boundaries.clear(); } } @@ -639,7 +686,16 @@ impl Decoder { // Propagate filter failures instead of silently emitting // the raw, unfiltered bytes. An unsupported or corrupt // filter would otherwise yield wrong output with no error. - super::filters::apply(&f, &mut region)?; + // + // The transform's position base is relative to the + // *containing file*, not the solid stream (see + // `add_file_boundary`); identical to `f.start` in the + // single-file case. + let file_rel = super::filters::Filter { + start: f.start - self.file_start_for(f.start), + ..f + }; + super::filters::apply(&file_rel, &mut region)?; for &b in ®ion { self.ready.push_back(b); } diff --git a/src/rar5/filters.rs b/src/rar5/filters.rs index e001b43..1501301 100644 --- a/src/rar5/filters.rs +++ b/src/rar5/filters.rs @@ -7,7 +7,7 @@ //! //! ## Filter types //! -//! - `0` — Delta. RGB pre-processing (channel deinterleaving). +//! - `0` — Delta. Channel de-interleave (bitmap/audio pre-processing). //! - `1` — x86 E8 call-translation. Rewrites the 4-byte relative target of //! every `0xE8` opcode. //! - `2` — x86 E8/E9 call+jump-translation. Same as `1` but also fires on @@ -16,9 +16,10 @@ //! - `4..=7` — Audio, RGB, Itanium, PPM. Not used in any RAR5 stream we have //! seen in the wild; treated as `Unsupported`. //! -//! This crate implements filters `1` and `2` (the most common) and rejects -//! the rest with `Error::Unsupported`. Adding more filters means extending -//! the dispatch in [`apply`]. +//! This crate implements filters `0`, `1` and `2` and rejects ARM and the +//! rest with `Error::Unsupported`. Adding more filters means extending the +//! dispatch in [`apply`]. The byte transforms themselves live in +//! [`crate::rar_filters`], shared with the RAR3 decoder. //! //! ## Activation //! @@ -29,10 +30,11 @@ //! block_length)`. use crate::error::Error; +use crate::rar_filters::{delta_decode, x86_e8_decode}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FilterKind { - /// 0 — RGB delta. `channels` is the channel count (1..=32). + /// 0 — Delta. `channels` is the channel count (1..=32). Delta { channels: u8 }, /// 1 — x86 `0xE8` (CALL) relative-address rewrite. X86Call, @@ -60,77 +62,30 @@ pub fn apply(filter: &Filter, buf: &mut [u8]) -> Result<(), Error> { if (buf.len() as u64) < filter.length as u64 { return Err(Error::Corrupt); } + let region = &mut buf[..filter.length as usize]; match filter.kind { - FilterKind::X86Call => apply_e8(filter.start, &mut buf[..filter.length as usize], false), - FilterKind::X86CallJmp => apply_e8(filter.start, &mut buf[..filter.length as usize], true), - // Filters we recognise on the wire but do not implement. Rejecting - // these is honest: the decoder is decoder-only and we can either - // surface "unsupported" up to the caller (so they can fall back to - // the official `unrar`) or silently mangle the stream. We pick - // honesty. - FilterKind::Delta { .. } => Err(Error::Unsupported), - FilterKind::Arm => Err(Error::Unsupported), - } -} - -/// RAR5 x86 call/jump filter. Operates on a 16 MiB virtual file-size window; -/// the relative target of each opcode is normalised so that the *absolute* -/// target is encoded instead, which compresses better. -/// -/// `start` is the absolute position of `buf[0]` in the unpacked stream. -/// When `extended` is true the filter fires on `0xE8` *and* `0xE9`; when -/// false it only fires on `0xE8`. The transform is its own inverse. -fn apply_e8(start: u64, buf: &mut [u8], extended: bool) -> Result<(), Error> { - const FILE_SIZE: u32 = 0x0100_0000; - if buf.len() < 5 { - // No room for a [opcode][4-byte rel] sequence. - return Ok(()); - } - let last = buf.len() - 4; - let mut i = 0; - while i < last { - let b = buf[i]; - let matches = b == 0xE8 || (extended && b == 0xE9); - if !matches { - i += 1; - continue; + FilterKind::X86Call => { + x86_e8_decode(filter.start, region, false, true); + Ok(()) + } + FilterKind::X86CallJmp => { + x86_e8_decode(filter.start, region, true, true); + Ok(()) } - // 4-byte little-endian relative target sitting at buf[i+1..i+5]. - let rel = u32::from_le_bytes([buf[i + 1], buf[i + 2], buf[i + 3], buf[i + 4]]); - // Libarchive: the offset is computed *after* the opcode byte has - // been consumed, so the relevant absolute position is start + i + 1. - let off = ((start + i as u64 + 1) as u32) & (FILE_SIZE - 1); - // RAR5 transform, decode direction. The two range checks are - // NESTED on the sign of `rel`, exactly as in unrar/libarchive: - // - // if (addr < 0) { if (addr + off >= 0) addr += FILE_SIZE; } - // else { if (addr < FILE_SIZE) addr -= off; } - // - // Flattening the second check into an `else if` is a bug: a - // negative `rel` that stays negative after adding `off` (a byte - // pattern the encoder never rewrote — e.g. a stray 0xE8 inside a - // ModRM/displacement sequence followed by high bytes) also passes - // `(rel - FILE_SIZE) & 0x8000_0000 != 0` and would be wrongly - // rewritten, corrupting real x86 code on decode. - let new = if (rel & 0x8000_0000) != 0 { - if (rel.wrapping_add(off) & 0x8000_0000) == 0 { - rel.wrapping_add(FILE_SIZE) - } else { - rel + FilterKind::Delta { channels } => { + if channels == 0 { + // The wire format encodes channels-1 in 5 bits, so 0 can't + // be parsed off the stream; guard against caller misuse. + return Err(Error::Corrupt); } - } else if (rel.wrapping_sub(FILE_SIZE) & 0x8000_0000) != 0 { - rel.wrapping_sub(off) - } else { - rel - }; - let nb = new.to_le_bytes(); - buf[i + 1] = nb[0]; - buf[i + 2] = nb[1]; - buf[i + 3] = nb[2]; - buf[i + 4] = nb[3]; - i += 5; + delta_decode(channels as usize, region); + Ok(()) + } + // Recognised on the wire but not implemented. Rejecting is honest: + // the caller can fall back to the official unrar instead of us + // silently mangling the stream. + FilterKind::Arm => Err(Error::Unsupported), } - Ok(()) } #[cfg(test)] @@ -235,14 +190,22 @@ mod tests { } #[test] - fn delta_and_arm_return_unsupported() { - let mut buf = alloc::vec![0; 16]; + fn delta_three_channels_reinterleaves() { + // Planar deltas: ch0=[1,1], ch1=[2,2], ch2=[3,3] over a 6-byte + // region. Decode integrates each channel with prev - delta. + let mut buf = alloc::vec![1u8, 1, 2, 2, 3, 3]; let f = Filter { start: 0, length: buf.len() as u32, kind: FilterKind::Delta { channels: 3 }, }; - assert_eq!(apply(&f, &mut buf), Err(Error::Unsupported)); + apply(&f, &mut buf).unwrap(); + assert_eq!(buf, [0xFF, 0xFE, 0xFD, 0xFE, 0xFC, 0xFA]); + } + + #[test] + fn arm_returns_unsupported() { + let mut buf = alloc::vec![0; 16]; let f = Filter { start: 0, length: buf.len() as u32, diff --git a/src/rar5/mod.rs b/src/rar5/mod.rs index d74736e..c14d4b0 100644 --- a/src/rar5/mod.rs +++ b/src/rar5/mod.rs @@ -19,9 +19,16 @@ //! - Single-volume RAR5 LZ77+Huffman compressed-data runs. //! - Cross-block table reuse (`table_present` bit set or clear). //! - The four-deep distance LRU and the "repeat last match" command. -//! - The x86 E8 and x86 E8/E9 post-decompression filters (filter types -//! 1 and 2). Delta (type 0), ARM (type 3), and the rare types 4–7 -//! are recognised on the wire but return [`Error::Unsupported`]. +//! - The Delta (type 0), x86 E8 (type 1) and x86 E8/E9 (type 2) +//! post-decompression filters. ARM (type 3) and the rare types 4–7 are +//! recognised on the wire but return [`Error::Unsupported`]. The +//! transforms live in the crate-internal `rar_filters` module, shared +//! with the RAR3 decoder. +//! - **Solid groups**, when the caller drives them as one continuous +//! stream: the LZ window carries across the group's members naturally, +//! and [`Decoder::add_file_boundary`] lets the container register each +//! member's starting offset so the x86 filters compute file-relative +//! addresses (unrar semantics). See the decoder docs. //! //! # What the decoder does *not* do //! @@ -29,11 +36,8 @@ //! main header, file headers, multi-volume continuations, encryption, //! recovery records, …) is not decoded here. Callers extract the inner //! compressed-data run from the container themselves and feed it to -//! the decoder's `decode()` method. -//! - **No solid-archive cross-file dictionary sharing.** RAR5's solid mode -//! keeps the LZ window alive across consecutive file entries; this -//! decoder treats every stream independently. -//! - **No filter chains for non-`X86Call` filter types.** +//! the decoder's `decode()` method. For solid groups that includes +//! concatenating the members' runs and slicing the decoded output. //! - **No CRC32/Blake2sp verification.** Those checksums live in the file //! header, not the compressed stream. //! diff --git a/src/rar_filters.rs b/src/rar_filters.rs new file mode 100644 index 0000000..193ef69 --- /dev/null +++ b/src/rar_filters.rs @@ -0,0 +1,197 @@ +//! Byte transforms for RAR's *standard* post-decompression filters, shared +//! by the RAR3 (in-band RarVM programs, main symbol 257) and RAR5 (filter +//! descriptors after main symbol 256) decoders. +//! +//! RAR compressors optionally pre-process content before LZ compression so +//! it compresses better (relative call targets in executables, interleaved +//! channel data in bitmaps/audio). The decoder undoes the transform over a +//! declared `(start, length)` window of the unpacked stream after +//! decompressing it. Both container generations use the same two transforms +//! implemented here: +//! +//! - **x86 E8 / E8E9** call(/jump) translation, [`x86_e8_decode`]. +//! - **Delta** channel de-interleave + integrate, [`delta_decode`]. +//! +//! ## Provenance +//! +//! Semantics derived from public format documentation and validated +//! byte-for-byte against WinRAR-produced archives (differential harness vs +//! `UnRAR.exe` 7.23: RAR4 delta ch=2/3/12 and E8 windows, RAR5 E8/E8E9). +//! Structure cross-checked against libarchive (BSD); no code copied from +//! RARLAB's unRAR or The Unarchiver. + +use alloc::vec::Vec; + +/// x86 call/jump filter, decode direction. Operates on a 16 MiB virtual +/// file-size window; the encoder rewrote each `E8` (and, for the extended +/// variant, `E9`) opcode's 4-byte relative target into absolute form, and +/// this pass restores the original relative value. +/// +/// `start` is the absolute position of `buf[0]` in the unpacked stream +/// (file-relative for both generations). When `also_e9` is true the filter +/// fires on `0xE8` *and* `0xE9`; when false only on `0xE8`. +/// +/// `wrap_16m` selects the position-base arithmetic, where the two container +/// generations differ: RAR5 reduces the position modulo the 16 MiB virtual +/// file size (validated against real WinRAR archives in the differential +/// harness), while RAR3's VM filter uses the unmasked 32-bit position (per +/// libarchive's RAR3 reader; the two agree below 16 MiB, so windows past +/// 16 MiB of a large executable are where masking would corrupt RAR3 +/// output). +pub(crate) fn x86_e8_decode(start: u64, buf: &mut [u8], also_e9: bool, wrap_16m: bool) { + const FILE_SIZE: u32 = 0x0100_0000; + if buf.len() < 5 { + // No room for a [opcode][4-byte rel] sequence. + return; + } + let last = buf.len() - 4; + let mut i = 0; + while i < last { + let b = buf[i]; + let matches = b == 0xE8 || (also_e9 && b == 0xE9); + if !matches { + i += 1; + continue; + } + // 4-byte little-endian relative target sitting at buf[i+1..i+5]. + let rel = u32::from_le_bytes([buf[i + 1], buf[i + 2], buf[i + 3], buf[i + 4]]); + // The offset is computed *after* the opcode byte has been consumed, + // so the relevant absolute position is start + i + 1. + let mut off = (start + i as u64 + 1) as u32; + if wrap_16m { + off &= FILE_SIZE - 1; + } + // Decode direction. The two range checks are NESTED on the sign of + // `rel`, exactly as in unrar/libarchive: + // + // if (addr < 0) { if (addr + off >= 0) addr += FILE_SIZE; } + // else { if (addr < FILE_SIZE) addr -= off; } + // + // Flattening the second check into an `else if` is a bug: a + // negative `rel` that stays negative after adding `off` (a byte + // pattern the encoder never rewrote — e.g. a stray 0xE8 inside a + // ModRM/displacement sequence followed by high bytes) also passes + // `(rel - FILE_SIZE) & 0x8000_0000 != 0` and would be wrongly + // rewritten, corrupting real x86 code on decode. + let new = if (rel & 0x8000_0000) != 0 { + if (rel.wrapping_add(off) & 0x8000_0000) == 0 { + rel.wrapping_add(FILE_SIZE) + } else { + rel + } + } else if (rel.wrapping_sub(FILE_SIZE) & 0x8000_0000) != 0 { + rel.wrapping_sub(off) + } else { + rel + }; + let nb = new.to_le_bytes(); + buf[i + 1] = nb[0]; + buf[i + 2] = nb[1]; + buf[i + 3] = nb[2]; + buf[i + 4] = nb[3]; + i += 5; + } +} + +/// Delta filter, decode direction. The encoder de-interleaved the region +/// into `channels` planes of successive byte differences; this pass +/// re-interleaves and integrates them: each output byte is the previous +/// output byte of the same channel *minus* the next source byte. +/// +/// `channels == 0` is a caller error and leaves the buffer untouched +/// (callers validate the channel count when parsing the declaration). +pub(crate) fn delta_decode(channels: usize, data: &mut [u8]) { + debug_assert!(channels >= 1); + if channels == 0 || data.is_empty() { + return; + } + // The source (planar deltas) is consumed sequentially while the + // destination is written strided, so a scratch copy of the source is + // needed. + let src: Vec = data.to_vec(); + let mut sp = 0usize; + for ch in 0..channels { + let mut prev = 0u8; + let mut i = ch; + while i < data.len() { + prev = prev.wrapping_sub(src[sp]); + data[i] = prev; + sp += 1; + i += channels; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::vec; + + #[test] + fn delta_two_channels_hand_computed() { + // Two channels, planar source: channel 0 deltas [1, 2], channel 1 + // deltas [3, 4]. Integration is prev - delta starting from 0. + // ch0: 0-1=0xFF, 0xFF-2=0xFD ; ch1: 0-3=0xFD, 0xFD-4=0xF9. + let mut data = vec![1u8, 2, 3, 4]; + delta_decode(2, &mut data); + assert_eq!(data, [0xFF, 0xFD, 0xFD, 0xF9]); + } + + #[test] + fn delta_single_channel_is_running_negated_sum() { + let mut data = vec![0u8, 0xFF, 0xFF]; + delta_decode(1, &mut data); + assert_eq!(data, [0, 1, 2]); + } + + #[test] + fn delta_length_not_divisible_by_channels() { + // 5 bytes, 2 channels: channel 0 covers indices 0,2,4 (3 source + // bytes), channel 1 covers 1,3 (2 source bytes) — source is planar + // in that order. + let mut data = vec![1u8, 1, 1, 2, 2]; + delta_decode(2, &mut data); + // ch0 deltas [1,1,1] -> FF,FE,FD at 0,2,4; ch1 deltas [2,2] -> FE,FC. + assert_eq!(data, [0xFF, 0xFE, 0xFE, 0xFC, 0xFD]); + } + + #[test] + fn e8_rewrites_call_target() { + let mut buf = vec![0x00, 0x00, 0xE8, 0x10, 0x00, 0x00, 0x00, 0x90, 0x90]; + x86_e8_decode(0, &mut buf, false, true); + // off = 2 + 1 = 3; rel = 0x10 in 0..FILE_SIZE => rel - off. + let expected = 0x10u32.wrapping_sub(3).to_le_bytes(); + assert_eq!(&buf[3..7], &expected); + } + + #[test] + fn e8_ignores_e9_unless_extended() { + let mut buf = vec![0xE9, 0x10, 0x00, 0x00, 0x00]; + let orig = buf.clone(); + x86_e8_decode(0, &mut buf, false, true); + assert_eq!(buf, orig); + x86_e8_decode(0, &mut buf, true, true); + assert_ne!(buf, orig); + } + + /// RAR3 (unmasked) and RAR5 (16 MiB-wrapped) position bases agree below + /// 16 MiB and diverge above — a >16 MiB window must subtract the full + /// offset on the RAR3 path. + #[test] + fn e8_base_masking_diverges_past_16mib() { + const START: u64 = 0x0100_0000; // exactly 16 MiB + let src = vec![0xE8, 0x10, 0x00, 0x00, 0x00]; + + let mut unmasked = src.clone(); + x86_e8_decode(START, &mut unmasked, false, false); + // off = 16 MiB + 1; rel = 0x10 < FILE_SIZE => rel - off (wrapping). + let want = 0x10u32.wrapping_sub(0x0100_0001).to_le_bytes(); + assert_eq!(&unmasked[1..5], &want); + + let mut masked = src.clone(); + x86_e8_decode(START, &mut masked, false, true); + // off wraps to 1 => rel - 1. + assert_eq!(&masked[1..5], &0x0Fu32.to_le_bytes()); + } +} diff --git a/tests/fixtures/ppmd/english.bin b/tests/fixtures/ppmd/english.bin new file mode 100644 index 0000000..41a4d51 --- /dev/null +++ b/tests/fixtures/ppmd/english.bin @@ -0,0 +1 @@ +the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. the quick brown fox jumps over lazy dog. \ No newline at end of file diff --git a/tests/fixtures/ppmd/english.ppmd b/tests/fixtures/ppmd/english.ppmd new file mode 100644 index 0000000..2a754a3 Binary files /dev/null and b/tests/fixtures/ppmd/english.ppmd differ diff --git a/tests/fixtures/ppmd/hello.bin b/tests/fixtures/ppmd/hello.bin new file mode 100644 index 0000000..95d09f2 --- /dev/null +++ b/tests/fixtures/ppmd/hello.bin @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/tests/fixtures/ppmd/hello.ppmd b/tests/fixtures/ppmd/hello.ppmd new file mode 100644 index 0000000..161a934 Binary files /dev/null and b/tests/fixtures/ppmd/hello.ppmd differ diff --git a/tests/fixtures/ppmd/mixed.bin b/tests/fixtures/ppmd/mixed.bin new file mode 100644 index 0000000..11aca8d Binary files /dev/null and b/tests/fixtures/ppmd/mixed.bin differ diff --git a/tests/fixtures/ppmd/mixed.ppmd b/tests/fixtures/ppmd/mixed.ppmd new file mode 100644 index 0000000..895dae3 Binary files /dev/null and b/tests/fixtures/ppmd/mixed.ppmd differ diff --git a/tests/fixtures/ppmd/repeat.bin b/tests/fixtures/ppmd/repeat.bin new file mode 100644 index 0000000..e7db635 --- /dev/null +++ b/tests/fixtures/ppmd/repeat.bin @@ -0,0 +1 @@ +the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox \ No newline at end of file diff --git a/tests/fixtures/ppmd/repeat.ppmd b/tests/fixtures/ppmd/repeat.ppmd new file mode 100644 index 0000000..f94e32e Binary files /dev/null and b/tests/fixtures/ppmd/repeat.ppmd differ diff --git a/tests/fixtures/ppmd/text.bin b/tests/fixtures/ppmd/text.bin new file mode 100644 index 0000000..13d0586 --- /dev/null +++ b/tests/fixtures/ppmd/text.bin @@ -0,0 +1 @@ +PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. PPMd context modelling test. \ No newline at end of file diff --git a/tests/fixtures/ppmd/text.ppmd b/tests/fixtures/ppmd/text.ppmd new file mode 100644 index 0000000..8fce48a Binary files /dev/null and b/tests/fixtures/ppmd/text.ppmd differ diff --git a/tests/fixtures/rar3/filter_delta_gradient_bmp.bin b/tests/fixtures/rar3/filter_delta_gradient_bmp.bin new file mode 100644 index 0000000..3cd8c10 Binary files /dev/null and b/tests/fixtures/rar3/filter_delta_gradient_bmp.bin differ diff --git a/tests/fixtures/rar3/filter_delta_ramp_wav.bin b/tests/fixtures/rar3/filter_delta_ramp_wav.bin new file mode 100644 index 0000000..478b3e3 Binary files /dev/null and b/tests/fixtures/rar3/filter_delta_ramp_wav.bin differ diff --git a/tests/fixtures/rar3/filter_x86_slice.bin b/tests/fixtures/rar3/filter_x86_slice.bin new file mode 100644 index 0000000..95f6d79 Binary files /dev/null and b/tests/fixtures/rar3/filter_x86_slice.bin differ diff --git a/tests/fixtures/rar3/m5_calls_delta12.bin b/tests/fixtures/rar3/m5_calls_delta12.bin new file mode 100644 index 0000000..e420c58 Binary files /dev/null and b/tests/fixtures/rar3/m5_calls_delta12.bin differ diff --git a/tests/fixtures/rar3/ppmd_notes.bin b/tests/fixtures/rar3/ppmd_notes.bin new file mode 100644 index 0000000..e1fd2a1 Binary files /dev/null and b/tests/fixtures/rar3/ppmd_notes.bin differ diff --git a/tests/fixtures/rar3/solid_m3_calls.bin b/tests/fixtures/rar3/solid_m3_calls.bin new file mode 100644 index 0000000..e0f7859 Binary files /dev/null and b/tests/fixtures/rar3/solid_m3_calls.bin differ diff --git a/tests/fixtures/rar3/solid_m3_gradient.bin b/tests/fixtures/rar3/solid_m3_gradient.bin new file mode 100644 index 0000000..dbd2947 Binary files /dev/null and b/tests/fixtures/rar3/solid_m3_gradient.bin differ diff --git a/tests/fixtures/rar3/solid_m3_notes.bin b/tests/fixtures/rar3/solid_m3_notes.bin new file mode 100644 index 0000000..80568b1 Binary files /dev/null and b/tests/fixtures/rar3/solid_m3_notes.bin differ diff --git a/tests/fixtures/rar3/solid_m3_photo.bin b/tests/fixtures/rar3/solid_m3_photo.bin new file mode 100644 index 0000000..a545c53 Binary files /dev/null and b/tests/fixtures/rar3/solid_m3_photo.bin differ diff --git a/tests/fixtures/rar3/solid_m3_ramp.bin b/tests/fixtures/rar3/solid_m3_ramp.bin new file mode 100644 index 0000000..577c5a7 Binary files /dev/null and b/tests/fixtures/rar3/solid_m3_ramp.bin differ diff --git a/tests/fixtures/rar3/solid_m3_x86slice.bin b/tests/fixtures/rar3/solid_m3_x86slice.bin new file mode 100644 index 0000000..678c99a Binary files /dev/null and b/tests/fixtures/rar3/solid_m3_x86slice.bin differ diff --git a/tests/fixtures/rar3/solid_ppmd_prose.bin b/tests/fixtures/rar3/solid_ppmd_prose.bin new file mode 100644 index 0000000..270e9ea Binary files /dev/null and b/tests/fixtures/rar3/solid_ppmd_prose.bin differ diff --git a/tests/fixtures/rar5/delta_gradient_bmp.bin b/tests/fixtures/rar5/delta_gradient_bmp.bin new file mode 100644 index 0000000..3c75d20 Binary files /dev/null and b/tests/fixtures/rar5/delta_gradient_bmp.bin differ diff --git a/tests/fixtures/rar5/solid_group_m3.bin b/tests/fixtures/rar5/solid_group_m3.bin new file mode 100644 index 0000000..d2d4ed2 Binary files /dev/null and b/tests/fixtures/rar5/solid_group_m3.bin differ diff --git a/tests/ppmd.rs b/tests/ppmd.rs index e9578b4..9c08071 100644 --- a/tests/ppmd.rs +++ b/tests/ppmd.rs @@ -1,24 +1,32 @@ -//! Integration tests for the PPMd decoder. +//! Integration tests for the PPMd (PPMII variant H) decoder. //! -//! PPMd is decoder-only in this crate (encoder always returns -//! [`Error::Unsupported`]) and the decoder implements *only* the -//! order-0 subset of PPMII variant H (see `src/ppmd/mod.rs` for the -//! documented gap). These tests: -//! -//! - confirm the algorithm metadata and Encoder-side Unsupported contract; -//! - decode hand-built order-0 fixtures (generated by the small encoder -//! helper in this file, which mirrors the decoder's order-0 model); -//! - exercise streaming with 1-byte input chunking; -//! - reject truncated input without panicking; -//! - confirm garbage doesn't panic; -//! - confirm `reset()` puts the decoder back to the Header phase; -//! - exercise the `#[cfg(feature = "factory")]` by-name lookup. +//! PPMd is decoder-only in this crate (the encoder always returns +//! [`Error::Unsupported`]). The round-trip fixtures are **real 7z Ppmd7 +//! streams** produced by `pyppmd.Ppmd7Encoder(order=6, mem=16 MiB)` and +//! wrapped in this crate's 11-byte framing header (order, mem_mb, +//! restoration, u64 length). They live in `tests/fixtures/ppmd/` alongside +//! their expected plaintext (`.bin`); see +//! `compcol-rar-corpus/probe/gen_ppmd_fixtures.py` for how they were +//! generated. Decoding them byte-for-byte exercises the full model: +//! context-tree construction, the binary-context path, masked escapes, SEE, +//! rescale, and the suballocator. #![cfg(feature = "ppmd")] use compcol::ppmd::{Decoder, Encoder, Ppmd}; use compcol::{Algorithm, Decoder as _, Encoder as _, Error, Status}; +// ─── fixtures ───────────────────────────────────────────────────────────── + +macro_rules! fixture { + ($name:literal) => { + ( + include_bytes!(concat!("fixtures/ppmd/", $name, ".ppmd")) as &[u8], + include_bytes!(concat!("fixtures/ppmd/", $name, ".bin")) as &[u8], + ) + }; +} + // ─── helpers ───────────────────────────────────────────────────────────── /// Build a framing header: order, mem_mb, restoration, len_le_u64. @@ -31,8 +39,8 @@ fn make_header(order: u8, mem_mb: u8, restoration: u8, len: u64) -> Vec { h } -/// Drive the decoder to completion, using a small output buffer to -/// exercise the OutputFull/InputEmpty back-pressure paths. +/// Drive the decoder to completion using a small output buffer to exercise +/// the OutputFull / InputEmpty back-pressure paths. fn drive_to_end(dec: &mut Decoder, input: &[u8]) -> Result, Error> { let mut out = Vec::new(); let mut buf = vec![0u8; 4096]; @@ -55,12 +63,8 @@ fn drive_to_end(dec: &mut Decoder, input: &[u8]) -> Result, Error> { break; } spin += 1; - if spin > 100_000 { - panic!( - "decoder spin (consumed={}, out_len={})", - consumed, - out.len() - ); + if spin > 1_000_000 { + panic!("decoder spin (consumed={consumed}, out_len={})", out.len()); } } loop { @@ -76,164 +80,6 @@ fn drive_to_end(dec: &mut Decoder, input: &[u8]) -> Result, Error> { Ok(out) } -// ─── reference order-0 encoder (test-only) ─────────────────────────────── -// -// The public PPMd encoder is permanently Unsupported. To exercise the -// decoder we need byte-perfect fixtures. We embed a tiny test-only -// order-0 PPMII encoder that mirrors the model in `src/ppmd/model.rs` -// (same +4 frequency increment, same MAX_FREQ rescale, same swap-with- -// predecessor promotion) and emits via the 7z carry-less range encoder. -// This keeps the production crate decoder-only while still letting -// `cargo test` round-trip without external tools. - -const MAX_FREQ: u32 = 124; - -struct OrderZeroModel { - // 256 (symbol, freq) entries kept roughly sorted by freq desc. - states: Vec<(u8, u32)>, - summ_freq: u32, -} - -impl OrderZeroModel { - fn new() -> Self { - let states: Vec<(u8, u32)> = (0..=255).map(|s| (s as u8, 1)).collect(); - Self { - states, - summ_freq: 256 + 1, - } - } - - fn find(&self, sym: u8) -> usize { - self.states - .iter() - .position(|&(s, _)| s == sym) - .expect("order-0") - } - - /// Returns (cum, freq, total). - fn encode_lookup(&self, sym: u8) -> (u32, u32, u32) { - let mut acc = 0u32; - for &(s, f) in &self.states { - if s == sym { - return (acc, f, self.summ_freq); - } - acc += f; - } - unreachable!() - } - - fn update(&mut self, sym: u8) { - let i = self.find(sym); - let new_f = self.states[i].1 + 4; - if new_f > MAX_FREQ { - self.rescale(); - return; - } - self.states[i].1 = new_f; - self.summ_freq += 4; - if i > 0 && new_f > self.states[i - 1].1 { - self.states.swap(i, i - 1); - } - } - - fn rescale(&mut self) { - let mut new_summ = 0u32; - for s in &mut self.states { - s.1 = ((s.1 + 1) >> 1).max(1); - new_summ += s.1; - } - self.summ_freq = new_summ; - } -} - -struct RangeEnc { - low: u64, - range: u32, - cache_size: u32, - cache: u8, - out: Vec, -} - -impl RangeEnc { - fn new() -> Self { - Self { - low: 0, - range: 0xFFFF_FFFF, - cache_size: 1, - cache: 0, - out: Vec::new(), - } - } - - /// Encode `(start, size, total)`: `range /= total; low += start*range; range *= size;`. - fn encode(&mut self, start: u32, size: u32, total: u32) { - self.range /= total; - self.low = self.low.wrapping_add(start as u64 * self.range as u64); - self.range = self.range.wrapping_mul(size); - self.normalize(); - } - - fn normalize(&mut self) { - while self.range < (1 << 24) { - self.shift_low(); - self.range <<= 8; - } - } - - fn shift_low(&mut self) { - // Pavlov's PPMd 7z `ShiftLow`. - if (self.low as u32) < 0xFF00_0000 || self.low >> 32 != 0 { - let mut temp = self.cache; - loop { - self.out.push(temp.wrapping_add((self.low >> 32) as u8)); - temp = 0xFF; - self.cache_size -= 1; - if self.cache_size == 0 { - break; - } - } - self.cache = (self.low as u32 >> 24) as u8; - } - self.cache_size += 1; - self.low = (self.low << 8) & 0xFFFF_FFFF; - } - - fn flush(mut self) -> Vec { - for _ in 0..5 { - self.shift_low(); - } - self.out - } -} - -fn encode_payload(input: &[u8]) -> Vec { - // 7z PPMd format: payload starts with a leading 0x00 byte that the - // decoder consumes during `init`. - let mut m = OrderZeroModel::new(); - let mut e = RangeEnc::new(); - for &b in input { - let (start, size, total) = m.encode_lookup(b); - e.encode(start, size, total); - m.update(b); - } - let mut bytes = e.flush(); - // The first byte of the encoded stream is the high-order byte that - // `shift_low`'s first iteration emits *after* `cache` (a leading 0). - // The carry-less PPMd-7z stream always starts with `0x00`, which the - // decoder requires (`Ppmd7z_RangeDec_Init`). We prepend `0x00` if our - // shift_low didn't already emit one. - if bytes.first() != Some(&0) { - bytes.insert(0, 0); - } - bytes -} - -fn make_stream(input: &[u8], order: u8) -> Vec { - let mut s = make_header(order, 1, 0, input.len() as u64); - s.extend_from_slice(&encode_payload(input)); - s -} - // ─── algorithm metadata ────────────────────────────────────────────────── #[test] @@ -279,65 +125,54 @@ fn encoder_reset_is_a_noop() { assert_eq!(enc.encode(b"x", &mut out).unwrap_err(), Error::Unsupported); } -// ─── round-trip via the test-only order-0 encoder ─────────────────────── +// ─── round-trip against real Ppmd7 fixtures ────────────────────────────── -#[test] -fn rt_empty() { - let stream = make_stream(b"", 4); +fn check(stream: &[u8], expected: &[u8]) { let mut dec = Decoder::new(); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, b""); + let out = drive_to_end(&mut dec, stream).unwrap(); + assert_eq!(out.len(), expected.len(), "length mismatch"); + assert_eq!(out, expected, "byte mismatch"); } #[test] -fn rt_single_byte() { - let stream = make_stream(b"A", 4); - let mut dec = Decoder::new(); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, b"A"); +fn rt_hello() { + let (stream, expected) = fixture!("hello"); + check(stream, expected); } #[test] -fn rt_hello_world() { - let stream = make_stream(b"hello world", 6); - let mut dec = Decoder::new(); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, b"hello world"); +fn rt_repeat() { + // Highly repetitive text — heavy match / context reuse. + let (stream, expected) = fixture!("repeat"); + check(stream, expected); } #[test] -fn rt_64k_repeating() { - let mut payload = Vec::with_capacity(64 * 1024); - let pattern = b"the quick brown fox jumps over the lazy dog "; - while payload.len() < 64 * 1024 { - payload.extend_from_slice(pattern); - } - payload.truncate(64 * 1024); - let stream = make_stream(&payload, 8); - let mut dec = Decoder::new(); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, payload); +fn rt_text() { + let (stream, expected) = fixture!("text"); + check(stream, expected); } #[test] -fn rt_mixed_corpus() { - let mut payload = Vec::new(); - payload.extend_from_slice(b"ASCII prefix.\n"); - payload.extend((0u8..=255u8).cycle().take(4096)); - payload.extend_from_slice(b"\nASCII suffix."); - let stream = make_stream(&payload, 4); - let mut dec = Decoder::new(); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, payload); +fn rt_english() { + // Word-structured input — exercises the model's typical operating point. + let (stream, expected) = fixture!("english"); + check(stream, expected); +} + +#[test] +fn rt_mixed_high_entropy() { + // 20 KiB of deterministic pseudo-random bytes — drives the escape / + // masked-suffix / SEE paths and rescales hard. + let (stream, expected) = fixture!("mixed"); + check(stream, expected); } // ─── streaming: one byte at a time ─────────────────────────────────────── #[test] fn streaming_one_byte_at_a_time() { - let payload = b"the quick brown fox"; - let stream = make_stream(payload, 4); - + let (stream, expected) = fixture!("english"); let mut dec = Decoder::new(); let mut out = Vec::new(); let mut buf = [0u8; 128]; @@ -351,10 +186,11 @@ fn streaming_one_byte_at_a_time() { consumed += p.consumed; out.extend_from_slice(&buf[..p.written]); if matches!(status, Status::StreamEnd) { - return assert_eq!(out, payload); + assert_eq!(out, expected); + return; } spin += 1; - if spin > 100_000 { + if spin > 1_000_000 { panic!("streaming spin"); } } @@ -365,77 +201,111 @@ fn streaming_one_byte_at_a_time() { break; } } - assert_eq!(out, payload); + assert_eq!(out, expected); } // ─── error cases ───────────────────────────────────────────────────────── #[test] -fn truncated_header_returns_input_empty() { +fn truncated_header_returns_unexpected_end_on_finish() { let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - let (p, status) = dec.decode(&[4, 1, 0, 0, 0], &mut buf).unwrap(); + let (p, _status) = dec.decode(&[6, 16, 0, 0, 0], &mut buf).unwrap(); assert_eq!(p.consumed, 5); - assert!(matches!(status, Status::InputEmpty)); - // finish should fail UnexpectedEnd. let r = dec.finish(&mut buf); assert_eq!(r, Err(Error::UnexpectedEnd)); } #[test] fn header_order_too_small_is_bad_header() { + let (stream, _) = fixture!("hello"); + let mut bad = make_header(1, 16, 0, 11); + bad.extend_from_slice(&stream[11..]); let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - let stream = make_header(1, 1, 0, 0); - let r = dec.decode(&stream, &mut buf); - assert_eq!(r, Err(Error::BadHeader)); + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::BadHeader)); } #[test] -fn header_order_too_large_is_bad_header() { +fn header_zero_mem_is_bad_header() { + let mut bad = make_header(6, 0, 0, 0); + bad.extend_from_slice(&[0, 0, 0, 0, 0]); let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - let stream = make_header(17, 1, 0, 0); - let r = dec.decode(&stream, &mut buf); - assert_eq!(r, Err(Error::BadHeader)); + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::BadHeader)); } #[test] -fn header_zero_mem_is_bad_header() { +fn header_bad_restoration_is_bad_header() { + let mut bad = make_header(6, 16, 9, 0); + bad.extend_from_slice(&[0, 0, 0, 0, 0]); let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - let stream = make_header(4, 0, 0, 0); - let r = dec.decode(&stream, &mut buf); - assert_eq!(r, Err(Error::BadHeader)); + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::BadHeader)); } +/// A hostile stream with an obviously invalid header must be rejected as +/// soon as the 11 header bytes have arrived — from `decode` itself — not +/// after the caller has piped (and the decoder buffered) the whole payload. #[test] -fn header_bad_restoration_is_bad_header() { +fn bad_header_is_rejected_at_decode_time() { + let mut bad = make_header(0, 16, 0, 100); // order 0: invalid + bad.extend_from_slice(&[0u8; 64]); let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - let stream = make_header(4, 1, 9, 0); - let r = dec.decode(&stream, &mut buf); - assert_eq!(r, Err(Error::BadHeader)); + assert_eq!(dec.decode(&bad, &mut buf).unwrap_err(), Error::BadHeader); + // The error is sticky and stays the same across calls. + assert_eq!(dec.finish(&mut buf), Err(Error::BadHeader)); +} + +#[test] +fn unknown_declared_length_is_refused() { + // PPMd has no in-band end-of-stream marker, so a stream framed with the + // "unknown length" sentinel (u64::MAX) has no reliable stopping point: + // decoding until the range coder exhausts input appends its finalisation + // bytes as extra garbage symbols (reframing the 1280-byte `repeat` + // fixture this way used to return 1466 bytes). It must be refused. + let (stream, _) = fixture!("repeat"); + let mut bad = stream.to_vec(); + bad[3..11].copy_from_slice(&u64::MAX.to_le_bytes()); + let mut dec = Decoder::new(); + let mut buf = [0u8; 256]; + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::Unsupported)); +} + +#[test] +fn absurd_declared_length_is_rejected_not_oomed() { + // Regression: a fuzz-found stream with a valid header but a wildly + // oversized declared length (~71 quadrillion bytes) used to drive the + // known-length decode loop toward OOM — a high-probability PPMd symbol + // can decode repeatedly without consuming input, so the range coder + // never overruns. The decoder must reject it up front. + let mut bad = make_header(6, 16, 0, 71_213_169_107_795_979); + bad.extend_from_slice(&[0x00, 0xfd, 0x00, 0x00, 0x67, 0xfb, 0x83, 0x7d]); + let mut dec = Decoder::new(); + let mut buf = [0u8; 16]; + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::OutputLimitExceeded)); } #[test] fn payload_first_byte_must_be_zero() { + let mut bad = make_header(6, 16, 0, 8); + bad.extend_from_slice(&[0xFF; 16]); let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - let mut stream = make_header(4, 1, 0, 8); - // Payload starts with 0xFF — the range decoder rejects. - stream.extend_from_slice(&[0xFF; 16]); - let r = dec.decode(&stream, &mut buf); - assert_eq!(r, Err(Error::Corrupt)); + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::Corrupt)); } #[test] -fn truncated_payload_returns_unexpected_end_on_finish() { - // Build a known-good stream then chop bytes off the tail. - let payload = b"hello there ppmd world"; - let stream = make_stream(payload, 4); +fn truncated_payload_is_unexpected_end() { + let (stream, _) = fixture!("text"); let truncated = &stream[..stream.len() - 3]; - let mut dec = Decoder::new(); let mut buf = [0u8; 256]; let _ = dec.decode(truncated, &mut buf); @@ -445,12 +315,11 @@ fn truncated_payload_returns_unexpected_end_on_finish() { #[test] fn garbage_after_header_does_not_panic() { - let mut stream = make_header(4, 1, 0, 16); - stream.extend_from_slice(&[0u8; 4]); - stream.extend_from_slice(&[0xAA; 64]); // garbage entropy stream + let mut stream = make_header(6, 16, 0, 64); + stream.extend_from_slice(&[0u8; 1]); + stream.extend_from_slice(&[0xAA; 128]); let mut dec = Decoder::new(); let mut buf = [0u8; 32]; - // Don't care about the result — just that it doesn't panic. let _ = dec.decode(&stream, &mut buf); let _ = dec.finish(&mut buf); } @@ -459,30 +328,30 @@ fn garbage_after_header_does_not_panic() { #[test] fn reset_returns_to_header_phase() { - let stream = make_stream(b"reset test", 4); + let (stream, expected) = fixture!("hello"); let mut dec = Decoder::new(); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, b"reset test"); + let out = drive_to_end(&mut dec, stream).unwrap(); + assert_eq!(out, expected); dec.reset(); - let out2 = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out2, b"reset test"); + let out2 = drive_to_end(&mut dec, stream).unwrap(); + assert_eq!(out2, expected); } #[test] fn reset_after_error_recovers() { let mut dec = Decoder::new(); let mut buf = [0u8; 16]; - // Bad header poisons the decoder. - let r = dec.decode(&[99, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], &mut buf); - assert_eq!(r, Err(Error::BadHeader)); - // Without reset, further calls should error. + let mut bad = make_header(99, 16, 0, 0); + bad.extend_from_slice(&[0, 0, 0, 0, 0]); + let _ = dec.decode(&bad, &mut buf); + assert_eq!(dec.finish(&mut buf), Err(Error::BadHeader)); + // Poisoned until reset. assert!(dec.decode(b"x", &mut buf).is_err()); dec.reset(); - // After reset, the decoder is usable again. - let stream = make_stream(b"", 4); - let out = drive_to_end(&mut dec, &stream).unwrap(); - assert_eq!(out, b""); + let (stream, expected) = fixture!("hello"); + let out = drive_to_end(&mut dec, stream).unwrap(); + assert_eq!(out, expected); } // ─── factory (only if the feature is enabled) ──────────────────────────── diff --git a/tests/rar3.rs b/tests/rar3.rs index 6a0c5cc..3c4f8a2 100644 --- a/tests/rar3.rs +++ b/tests/rar3.rs @@ -180,18 +180,23 @@ fn decodes_libarchive_test_txt_tight_output_buffer() { assert_eq!(out, TESTDIR_TEST_TXT_EXPECTED); } -// ─── PPMd rejection ────────────────────────────────────────────────────── +// ─── PPMd continuation without a live model ────────────────────────────── #[test] -fn ppmd_block_is_unsupported() { - // A block whose first bit (after byte-align) is 1 = PPMd flag. - // 0x80 = 1000_0000 → byte-aligned start, top bit = 1 ⇒ PPMd. +fn ppmd_continuation_without_model_is_corrupt() { + // Top bit set ⇒ PPMd block; the next 7 flag bits are 0, so flag 0x20 + // (fresh model + memory/order) is clear. That marks a continuation + // reusing a live model from an earlier block or solid member — which a + // standalone first block can't have, so the stream is malformed. (A + // real, self-contained PPMd block decodes — see `ppmd_block_decodes`; + // a real continuation across a solid group decodes — see the solid + // fixtures.) let ppmd_marker = [0x80u8, 0x00, 0x00, 0x00]; let mut dec = Decoder::with_unpack_size(32); let (_p, _status) = dec.decode(&ppmd_marker, &mut []).unwrap(); let mut buf = [0u8; 16]; let err = dec.finish(&mut buf).unwrap_err(); - assert_eq!(err, Error::Unsupported); + assert_eq!(err, Error::Corrupt); } // ─── error path: malformed inputs ──────────────────────────────────────── @@ -287,6 +292,267 @@ fn decoder_with_e8_filter_round_trip_on_synthetic_data() { assert_eq!(out, TESTDIR_TEST_TXT_EXPECTED); } +// ─── In-band standard filters (real-archive fixtures) ──────────────────── +// +// Payloads lifted out of archives created with RARLAB `rar` 6.24 (the last +// encoder able to write RAR4) over synthetic content: the fixture bytes are +// exactly what follows the RAR4 file header. The expected CRC-32 is the +// archive's own FILE_CRC header field (CRC-32 of the uncompressed file, +// written by the encoder); extraction was additionally cross-checked +// byte-identical against WinRAR `UnRAR.exe` 7.23 by the differential +// harness. Each stream opens with a main-symbol-257 declaration carrying a +// standard RarVM program that the decoder must recognize and run natively. + +/// gradient.bmp — Delta filter, 3 channels, window 49152 of 49206 bytes. +static FILTER_DELTA_BMP: &[u8] = include_bytes!("fixtures/rar3/filter_delta_gradient_bmp.bin"); +/// ramp.wav — Delta filter, 2 channels (rar 6.24 uses Delta for WAV, not +/// the legacy audio predictor), window 16384 of 16428 bytes. +static FILTER_DELTA_WAV: &[u8] = include_bytes!("fixtures/rar3/filter_delta_ramp_wav.bin"); +/// calls.bin at -m5 — Delta filter, 12 channels. +static FILTER_DELTA12_CALLS: &[u8] = include_bytes!("fixtures/rar3/m5_calls_delta12.bin"); +/// x86slice.bin — x86 E8 (call-only) filter over the whole 32 KiB. +static FILTER_X86_SLICE: &[u8] = include_bytes!("fixtures/rar3/filter_x86_slice.bin"); + +/// Bitwise CRC-32 (IEEE, reflected 0xEDB88320) — small and table-free; +/// test-only, so speed is irrelevant. +fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFFu32; + for &b in data { + crc ^= b as u32; + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc +} + +fn decode_and_check_crc(block: &[u8], unpack_size: u64, want_crc: u32) -> Vec { + let out = decode_full(block, unpack_size); + assert_eq!(out.len() as u64, unpack_size, "unpacked size mismatch"); + assert_eq!( + crc32(&out), + want_crc, + "decoded bytes differ from the archive's FILE_CRC" + ); + out +} + +#[test] +fn inband_delta_filter_bmp_three_channels() { + let out = decode_and_check_crc(FILTER_DELTA_BMP, 49206, 0x2347_E5ED); + // Spot-check: it really is the bitmap (BMP magic survives filtering). + assert_eq!(&out[..2], b"BM"); +} + +#[test] +fn inband_delta_filter_wav_two_channels() { + let out = decode_and_check_crc(FILTER_DELTA_WAV, 16428, 0x0E8F_2810); + assert_eq!(&out[..4], b"RIFF"); +} + +#[test] +fn inband_delta_filter_twelve_channels() { + decode_and_check_crc(FILTER_DELTA12_CALLS, 6146, 0x6C08_D7DF); +} + +#[test] +fn inband_x86_e8_filter() { + decode_and_check_crc(FILTER_X86_SLICE, 32768, 0x6188_0029); +} + +/// A stream that declares a filter window it never finishes producing is +/// malformed: returning the raw pre-filter bytes as a success would be +/// wrong output. (unrar emits the raw bytes and relies on the container +/// CRC to flag the file; this crate surfaces the error directly.) +#[test] +fn unfinished_filter_window_is_corrupt() { + // The delta fixture declares a 49152-byte window up front; capping the + // unpack size below that truncates the window. + let mut dec = Decoder::with_unpack_size(1000); + let (_p, _s) = dec.decode(FILTER_DELTA_BMP, &mut []).unwrap(); + let mut buf = [0u8; 64]; + assert_eq!(dec.finish(&mut buf).unwrap_err(), Error::Corrupt); +} + +// ─── PPMd-II variant H block (real-archive fixture) ────────────────────── +// +// The raw v29 payload of `notes.txt` from a `rar 6.24 -mct+` (forced +// text/PPMd) archive — a self-contained PPMd block (bit-0 of the block +// header set) that decodes to 20001 bytes. Expected CRC-32 is the archive's +// own FILE_CRC; extraction was cross-checked byte-identical against UnRAR +// 7.23. This exercises the RAR range decoder plus the full PPMII model +// (context tree, escapes, SEE, rescale) through the rar3 entry point. +static PPMD_NOTES: &[u8] = include_bytes!("fixtures/rar3/ppmd_notes.bin"); + +#[test] +fn ppmd_block_decodes() { + decode_and_check_crc(PPMD_NOTES, 20001, 0x0E1A_EC07); +} + +/// A truncated PPMd payload must fail, not fabricate output. Once the range +/// coder reads past the end of the compressed block, `read_byte` supplies +/// zeroes and every further symbol is invented; without an overrun check the +/// decoder would still reach the declared 20001-byte size (with a wrong CRC) +/// and report success. Dropping the tail bytes must surface an error instead. +#[test] +fn ppmd_truncated_payload_errors_not_fabricates() { + let truncated = &PPMD_NOTES[..PPMD_NOTES.len() - 14]; + let mut dec = Decoder::with_unpack_size(20001); + // Buffer-then-decode: the failure surfaces on the first finish/drain. + let _ = dec.decode(truncated, &mut []); + let mut buf = [0u8; 4096]; + let mut err = None; + let mut produced = 0usize; + loop { + match dec.finish(&mut buf) { + Ok((p, status)) => { + produced += p.written; + if matches!(status, Status::StreamEnd) || p.written == 0 { + break; + } + } + Err(e) => { + err = Some(e); + break; + } + } + } + assert!( + matches!(err, Some(Error::UnexpectedEnd) | Some(Error::Corrupt)), + "truncated PPMd should error (got err={err:?}, produced={produced} bytes)" + ); +} + +// ─── Solid groups (real-archive fixtures) ───────────────────────────────── +// +// Raw v29 member payloads of `rar4_m3_solid.rar` and `rar4_ppmd_solid.rar` +// (RARLAB `rar 6.24 -s`), in archive order. Members of a solid group share +// one compression history: the LZ window, code tables, offset history, +// filter programs and any live PPMd model carry from member to member, +// while each payload is its own byte-aligned stream. Expected CRC-32s are +// the archives' own FILE_CRC fields; extraction was cross-checked +// byte-identical against UnRAR 7.23 by the differential harness. +// +// The m3 group exercises every member-boundary shape rar 6.24 emits: +// members that open with their own block header (fresh or `keep_table` +// delta-coded), members that continue the previous member's stream with no +// header at all (photo.jpg, ramp.wav — announced by the previous member's +// end marker), in-band filters inside a solid stream, and LZ matches +// reaching across member boundaries. The PPMd group exercises model +// persistence: prose.txt's header has no reset flag and reuses notes.txt's +// live model (with a freshly initialised range coder). + +/// (payload, unpacked size, FILE_CRC) for each member, in archive order. +static SOLID_M3_GROUP: &[(&[u8], u64, u32)] = &[ + ( + include_bytes!("fixtures/rar3/solid_m3_calls.bin"), + 6146, + 0x6C08_D7DF, + ), + ( + include_bytes!("fixtures/rar3/solid_m3_x86slice.bin"), + 32768, + 0x6188_0029, + ), + ( + include_bytes!("fixtures/rar3/solid_m3_gradient.bin"), + 49206, + 0x2347_E5ED, + ), + ( + include_bytes!("fixtures/rar3/solid_m3_photo.bin"), + 8198, + 0x8420_E285, + ), + ( + include_bytes!("fixtures/rar3/solid_m3_notes.bin"), + 20001, + 0x0E1A_EC07, + ), + ( + include_bytes!("fixtures/rar3/solid_m3_ramp.bin"), + 16428, + 0x0E8F_2810, + ), +]; + +/// prose.txt — the second member of `rar4_ppmd_solid.rar`; its first member +/// is the same payload as `PPMD_NOTES`. +static SOLID_PPMD_PROSE: &[u8] = include_bytes!("fixtures/rar3/solid_ppmd_prose.bin"); + +/// Decode a whole solid group member-by-member, asserting each member's +/// size and FILE_CRC. +fn decode_solid_group(members: &[(&[u8], u64, u32)]) { + let mut dec = Decoder::with_unpack_size(members[0].1).with_solid(); + for (i, &(payload, unp, want_crc)) in members.iter().enumerate() { + if i > 0 { + dec.begin_solid_member(unp).unwrap(); + } + let (_p, _s) = dec.decode(payload, &mut []).unwrap(); + let mut out = Vec::with_capacity(unp as usize); + drain(&mut dec, &mut out); + assert_eq!(out.len() as u64, unp, "member {i}: unpacked size mismatch"); + assert_eq!( + crc32(&out), + want_crc, + "member {i}: decoded bytes differ from the archive's FILE_CRC" + ); + } +} + +#[test] +fn solid_lz_group_decodes() { + decode_solid_group(SOLID_M3_GROUP); +} + +#[test] +fn solid_ppmd_group_decodes() { + decode_solid_group(&[ + (PPMD_NOTES, 20001, 0x0E1A_EC07), + (SOLID_PPMD_PROSE, 236737, 0x028E_1AC1), + ]); +} + +/// Feeding a solid continuation without its group head must fail closed: +/// prose.txt's block header reuses a live PPMd model that a fresh decoder +/// doesn't have. +#[test] +fn solid_ppmd_continuation_without_head_fails() { + let mut dec = Decoder::with_unpack_size(236737).with_solid(); + let (_p, _s) = dec.decode(SOLID_PPMD_PROSE, &mut []).unwrap(); + let mut buf = [0u8; 4096]; + assert_eq!(dec.finish(&mut buf).unwrap_err(), Error::Corrupt); +} + +/// A truncated member inside a solid group is a hard error — the shared +/// history would silently desync every member after it. +#[test] +fn solid_truncated_member_poisons_group() { + let mut dec = Decoder::with_unpack_size(6146).with_solid(); + let head = SOLID_M3_GROUP[0].0; + let (_p, _s) = dec.decode(&head[..head.len() / 2], &mut []).unwrap(); + let mut buf = [0u8; 4096]; + assert!(matches!( + dec.finish(&mut buf).unwrap_err(), + Error::UnexpectedEnd | Error::Corrupt + )); + // The group is poisoned: the next member can't be started. + assert!(dec.begin_solid_member(32768).is_err()); +} + +/// `begin_solid_member` is only valid on a solid decoder whose current +/// member has fully drained. +#[test] +fn begin_solid_member_misuse_is_rejected() { + // Not a solid decoder. + let mut dec = Decoder::with_unpack_size(4); + assert_eq!(dec.begin_solid_member(4).unwrap_err(), Error::Unsupported); + // Solid, but the current member hasn't been decoded yet. + let mut dec = Decoder::with_unpack_size(6146).with_solid(); + assert_eq!(dec.begin_solid_member(4).unwrap_err(), Error::Unsupported); +} + // ─── factory (only if compiled in) ─────────────────────────────────────── #[cfg(feature = "factory")] diff --git a/tests/rar5.rs b/tests/rar5.rs index 4942ee9..49e6ae0 100644 --- a/tests/rar5.rs +++ b/tests/rar5.rs @@ -63,7 +63,13 @@ const FIXTURE_E8_UNPACK: u64 = 1506; /// Drive a freshly-constructed decoder to completion against a single input /// slice and return the produced bytes. fn decode_once(comp: &[u8], unpack: u64, window: usize) -> Result, Error> { - let mut dec = Decoder::with_unpack_size_and_window(unpack, window); + let dec = Decoder::with_unpack_size_and_window(unpack, window); + drive(dec, comp, unpack) +} + +/// Drive an already-configured decoder (e.g. one with file boundaries +/// registered) to completion against a single input slice. +fn drive(mut dec: Decoder, comp: &[u8], unpack: u64) -> Result, Error> { let mut out = vec![0u8; unpack as usize]; let mut total = 0usize; @@ -393,6 +399,93 @@ fn reset_clears_state() { assert_eq!(&out2[..total2], expected.as_slice()); } +// ─── Real-archive fixtures: Delta filter + solid-group boundaries ──────── +// +// Packed runs lifted out of archives produced by WinRAR 7.23 (`rar5` +// format, m3, 128 KiB window) over synthetic content — the fixture bytes +// are exactly the member's compressed run following its file header. The +// expected CRC-32s are the archives' own per-file data-CRC header fields; +// extraction is additionally cross-checked byte-identical against +// `UnRAR.exe` 7.23 by the differential harness. + +/// gradient.bmp, non-solid — carries a Delta filter (3 channels). +static RAR5_DELTA_GRADIENT: &[u8] = include_bytes!("fixtures/rar5/delta_gradient_bmp.bin"); + +/// A whole solid group: six members sharing one continuous LZ stream, in +/// this order. x86slice.bin carries x86 E8 filters, gradient.bmp/ramp.wav/ +/// calls.bin carry Delta filters. +static RAR5_SOLID_GROUP: &[u8] = include_bytes!("fixtures/rar5/solid_group_m3.bin"); + +/// (name, start offset in the group's output, unpacked size, CRC-32) per +/// member, from the archive's file headers. +const SOLID_MEMBERS: [(&str, u64, u64, u32); 6] = [ + ("notes.txt", 0, 20001, 0x0E1A_EC07), + ("calls.bin", 20001, 6146, 0x6C08_D7DF), + ("x86slice.bin", 26147, 32768, 0x6188_0029), + ("gradient.bmp", 58915, 49206, 0x2347_E5ED), + ("ramp.wav", 108121, 16428, 0x0E8F_2810), + ("photo.jpg", 124549, 8198, 0x8420_E285), +]; +const SOLID_TOTAL: u64 = 132_747; +const SOLID_WINDOW: usize = 0x20000; + +/// Bitwise CRC-32 (IEEE, reflected 0xEDB88320) — table-free, test-only. +fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFFu32; + for &b in data { + crc ^= b as u32; + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc +} + +#[test] +fn delta_filter_end_to_end() { + let out = decode_once(RAR5_DELTA_GRADIENT, 49206, 0x20000).unwrap(); + assert_eq!(out.len(), 49206); + assert_eq!(&out[..2], b"BM", "BMP magic must survive de-filtering"); + assert_eq!( + crc32(&out), + 0x2347_E5ED, + "must match the archive's data CRC" + ); +} + +#[test] +fn solid_group_with_file_boundaries_end_to_end() { + let mut dec = Decoder::with_unpack_size_and_window(SOLID_TOTAL, SOLID_WINDOW); + for (_, start, _, _) in SOLID_MEMBERS { + dec.add_file_boundary(start); + } + let out = drive(dec, RAR5_SOLID_GROUP, SOLID_TOTAL).unwrap(); + assert_eq!(out.len() as u64, SOLID_TOTAL); + for (name, start, size, want_crc) in SOLID_MEMBERS { + let member = &out[start as usize..(start + size) as usize]; + assert_eq!(crc32(member), want_crc, "{name} differs from its data CRC"); + } +} + +/// Regression guard for the solid x86 position-base bug: without the +/// registered boundaries the E8 transform runs with solid-stream offsets +/// and corrupts the x86 member, while the position-independent Delta +/// members still decode correctly. +#[test] +fn solid_group_without_boundaries_corrupts_only_x86_member() { + let dec = Decoder::with_unpack_size_and_window(SOLID_TOTAL, SOLID_WINDOW); + let out = drive(dec, RAR5_SOLID_GROUP, SOLID_TOTAL).unwrap(); + for (name, start, size, want_crc) in SOLID_MEMBERS { + let got = crc32(&out[start as usize..(start + size) as usize]); + if name == "x86slice.bin" { + assert_ne!(got, want_crc, "x86 member should differ without boundaries"); + } else { + assert_eq!(got, want_crc, "{name} is position-independent"); + } + } +} + // ─── factory (only if the feature is enabled) ──────────────────────────── #[cfg(feature = "factory")]