+ add FPAQ0 order-0 arithmetic building block

Compression.Core/Entropy/Fpaq/Fpaq0BuildingBlock.cs

@@ -0,0 +1,122 @@
+using System.Buffers.Binary;
+using Compression.Core.Entropy.Arithmetic;
+using Compression.Registry;
+
+namespace Compression.Core.Entropy.Fpaq;
+
+/// <summary>
+/// Exposes FPAQ0-style adaptive order-0 arithmetic compression as a benchmarkable building block.
+/// Each byte is coded MSB-first with one binary probability model per prefix of the current byte.
+/// </summary>
+/// <remarks>
+/// The model starts every branch at one zero and one one, updates after each coded bit, and
+/// halves large counts to keep the arithmetic coder's probability calculation bounded. This is
+/// the compact order-0 modelling scheme described for Matt Mahoney's FPAQ family; the outer
+/// four-byte original-length field is CompressionWorkbench framing rather than an FPAQ archive.
+/// Reference: https://mattmahoney.net/dc/ — FPAQ section.
+/// </remarks>
+public sealed class Fpaq0BuildingBlock : IBuildingBlock {
+  private const int ContextCount = 256;
+  private const int RescaleAt = 32768;
+
+  /// <inheritdoc/>
+  public string Id => "BB_Fpaq0";
+
+  /// <inheritdoc/>
+  public string DisplayName => "FPAQ0";
+
+  /// <inheritdoc/>
+  public string Description => "Adaptive order-0 binary arithmetic compression";
+
+  /// <inheritdoc/>
+  public AlgorithmFamily Family => AlgorithmFamily.Entropy;
+
+  /// <inheritdoc/>
+  public byte[] Compress(ReadOnlySpan<byte> data) {
+    using var output = new MemoryStream();
+    Span<byte> lengthBytes = stackalloc byte[sizeof(int)];
+    BinaryPrimitives.WriteInt32LittleEndian(lengthBytes, data.Length);
+    output.Write(lengthBytes);
+
+    if (data.IsEmpty)
+      return output.ToArray();
+
+    var zeroCounts = CreateCounts();
+    var oneCounts = CreateCounts();
+    var encoder = new ArithmeticEncoder(output);
+
+    foreach (var value in data) {
+      var context = 1;
+      for (var shift = 7; shift >= 0; --shift) {
+        var bit = value >> shift & 1;
+        encoder.EncodeBit(bit, ProbabilityOfZero(zeroCounts[context], oneCounts[context]));
+        UpdateModel(zeroCounts, oneCounts, context, bit);
+        context = context << 1 | bit;
+      }
+    }
+
+    encoder.Finish();
+    return output.ToArray();
+  }
+
+  /// <inheritdoc/>
+  public byte[] Decompress(ReadOnlySpan<byte> data) {
+    if (data.Length < sizeof(int))
+      throw new InvalidDataException("FPAQ0 stream is missing its original-length header.");
+
+    var originalLength = BinaryPrimitives.ReadInt32LittleEndian(data);
+    if (originalLength < 0)
+      throw new InvalidDataException("FPAQ0 stream declares a negative original length.");
+    if (originalLength == 0) {
+      if (data.Length != sizeof(int))
+        throw new InvalidDataException("FPAQ0 empty stream contains trailing payload data.");
+      return [];
+    }
+    if (data.Length == sizeof(int))
+      throw new InvalidDataException("FPAQ0 stream has no arithmetic-coded payload.");
+
+    var result = new byte[originalLength];
+    var zeroCounts = CreateCounts();
+    var oneCounts = CreateCounts();
+    using var input = new MemoryStream(data[sizeof(int)..].ToArray(), writable: false);
+    var decoder = new ArithmeticDecoder(input);
+
+    for (var index = 0; index < result.Length; ++index) {
+      var context = 1;
+      var value = 0;
+      for (var bitIndex = 0; bitIndex < 8; ++bitIndex) {
+        var bit = decoder.DecodeBit(ProbabilityOfZero(zeroCounts[context], oneCounts[context]));
+        UpdateModel(zeroCounts, oneCounts, context, bit);
+        value = value << 1 | bit;
+        context = context << 1 | bit;
+      }
+      result[index] = (byte)value;
+    }
+
+    return result;
+  }
+
+  private static int[] CreateCounts() {
+    var result = new int[ContextCount];
+    Array.Fill(result, 1);
+    return result;
+  }
+
+  private static int ProbabilityOfZero(int zeroCount, int oneCount) {
+    var probability = (int)(((long)zeroCount << 16) / (zeroCount + oneCount));
+    return Math.Clamp(probability, 1, ushort.MaxValue);
+  }
+
+  private static void UpdateModel(int[] zeroCounts, int[] oneCounts, int context, int bit) {
+    if (bit == 0)
+      ++zeroCounts[context];
+    else
+      ++oneCounts[context];
+
+    if (zeroCounts[context] + oneCounts[context] < RescaleAt)
+      return;
+
+    zeroCounts[context] = zeroCounts[context] + 1 >> 1;
+    oneCounts[context] = oneCounts[context] + 1 >> 1;
+  }
+}

Compression.Core/README.md

@@ -4926,6 +4926,26 @@ Exponential Golomb encoder. Used in H.264/H.265 video codecs. Order-k exp-Golomb
 | `Encode` | `void Encode(int value)` | Encodes a non-negative value using exp-Golomb coding. |
 | `Flush` | `void Flush()` | Flushes any remaining bits in the buffer. |
 
+### Namespace `Compression.Core.Entropy.Fpaq`
+
+[`Fpaq0BuildingBlock`](#fpaq0buildingblock)
+
+#### `Fpaq0BuildingBlock`
+
+Exposes FPAQ0-style adaptive order-0 arithmetic compression as a benchmarkable building block. Each byte is coded MSB-first with one binary probability model per prefix of the current byte.
+
+Implements `IBuildingBlock`.
+
+| Member | Signature | Summary |
+| --- | --- | --- |
+| `Fpaq0BuildingBlock` | `Fpaq0BuildingBlock()` |  |
+| `Description` | `string Description { get; }` |  |
+| `DisplayName` | `string DisplayName { get; }` |  |
+| `Family` | `AlgorithmFamily Family { get; }` |  |
+| `Id` | `string Id { get; }` |  |
+| `Compress` | `byte[] Compress(ReadOnlySpan<byte> data)` |  |
+| `Decompress` | `byte[] Decompress(ReadOnlySpan<byte> data)` |  |
+
 ### Namespace `Compression.Core.Entropy.Fse`
 
 [`FseDecoder`](#fsedecoder) · [`FseEncoder`](#fseencoder) · [`FseTable`](#fsetable) · [`HuffmanFse`](#huffmanfse)

Compression.Tests/BuildingBlocks/Fpaq0BuildingBlockTests.cs

@@ -0,0 +1,105 @@
+using System.Buffers.Binary;
+using System.Text;
+using Compression.Core.Entropy.Fpaq;
+using Compression.Registry;
+
+namespace Compression.Tests.BuildingBlocks;
+
+[TestFixture]
+public sealed class Fpaq0BuildingBlockTests {
+  private static readonly Fpaq0BuildingBlock Bb = new();
+
+  [Test, Category("HappyPath"), Category("RoundTrip")]
+  public void Empty_RoundTrips() {
+    var compressed = Bb.Compress([]);
+    var roundTrip = Bb.Decompress(compressed);
+
+    Assert.Multiple(() => {
+      Assert.That(compressed, Is.EqualTo(new byte[] { 0, 0, 0, 0 }).AsCollection);
+      Assert.That(roundTrip, Is.Empty);
+    });
+  }
+
+  [Test, Category("HappyPath"), Category("RoundTrip")]
+  public void RepeatedByte_RoundTripsAndCompresses() {
+    var data = new byte[20_000];
+    Array.Fill(data, (byte)'a');
+
+    var compressed = Bb.Compress(data);
+    var roundTrip = Bb.Decompress(compressed);
+
+    Assert.Multiple(() => {
+      Assert.That(compressed.Length, Is.LessThan(data.Length / 100));
+      Assert.That(roundTrip, Is.EqualTo(data).AsCollection);
+    });
+  }
+
+  [Test, Category("HappyPath"), Category("RoundTrip")]
+  public void RepeatedPhrase_RoundTripsAndCompresses() {
+    const string phrase = "the quick brown fox jumps over the lazy dog. ";
+    var builder = new StringBuilder(20_000 + phrase.Length);
+    while (builder.Length < 20_000)
+      builder.Append(phrase);
+    var data = Encoding.ASCII.GetBytes(builder.ToString(0, 20_000));
+
+    var compressed = Bb.Compress(data);
+    var roundTrip = Bb.Decompress(compressed);
+
+    Assert.Multiple(() => {
+      Assert.That(compressed.Length, Is.LessThan(data.Length * 3 / 4));
+      Assert.That(roundTrip, Is.EqualTo(data).AsCollection);
+    });
+  }
+
+  [Test, Category("HappyPath"), Category("RoundTrip")]
+  public void IncompressibleRandom_RoundTrips() {
+    var random = new Random(0xF0A0);
+    var data = new byte[8192];
+    random.NextBytes(data);
+
+    var roundTrip = Bb.Decompress(Bb.Compress(data));
+
+    Assert.That(roundTrip, Is.EqualTo(data).AsCollection);
+  }
+
+  [Test, Category("EdgeCase"), Category("RoundTrip")]
+  public void AllByteValues_RoundTrip() {
+    var data = Enumerable.Range(0, 256).Select(value => (byte)value).ToArray();
+    var roundTrip = Bb.Decompress(Bb.Compress(data));
+    Assert.That(roundTrip, Is.EqualTo(data).AsCollection);
+  }
+
+  [Test, Category("EdgeCase")]
+  public void KnownVector_IsDeterministic() {
+    var data = Encoding.ASCII.GetBytes("abababababababab");
+    var compressed = Bb.Compress(data);
+
+    Assert.That(compressed, Is.EqualTo(new byte[] {
+      16, 0, 0, 0,
+      0x61, 0x6F, 0x3D, 0x33, 0xCC, 0x9A, 0x80,
+    }).AsCollection);
+    Assert.That(Bb.Decompress(compressed), Is.EqualTo(data).AsCollection);
+  }
+
+  [Test, Category("EdgeCase")]
+  public void MissingPayload_IsRejected() {
+    byte[] malformed = [1, 0, 0, 0];
+    Assert.That(() => Bb.Decompress(malformed), Throws.TypeOf<InvalidDataException>());
+  }
+
+  [Test, Category("EdgeCase")]
+  public void NegativeLength_IsRejected() {
+    var malformed = new byte[4];
+    BinaryPrimitives.WriteInt32LittleEndian(malformed, -1);
+    Assert.That(() => Bb.Decompress(malformed), Throws.TypeOf<InvalidDataException>());
+  }
+
+  [Test, Category("EdgeCase")]
+  public void Registry_Metadata_IsStable() {
+    Assert.Multiple(() => {
+      Assert.That(Bb.Id, Is.EqualTo("BB_Fpaq0"));
+      Assert.That(Bb.DisplayName, Is.EqualTo("FPAQ0"));
+      Assert.That(Bb.Family, Is.EqualTo(AlgorithmFamily.Entropy));
+    });
+  }
+}

Hawkynt.FileFormats.Archives/README.md

@@ -3222,7 +3222,7 @@ Implements `IFormatDescriptor`, `IStreamFormatOperations`.
 
 #### `CscStream`
 
-CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size \| uint24 csc_blocksize \| uint24 raw_blocksize \| uint32 actual_size
+CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size | uint24 csc_blocksize | uint24 raw_blocksize | uint32 actual_size
 
 | Member | Signature | Summary |
 | --- | --- | --- |
@@ -4820,7 +4820,7 @@ DEFLATE (RFC 1951) decoder for the PEtite dialect: block type 1 carries the dyna
 
 #### `PetiteUnpacker`
 
-Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000\|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments)
+Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments)
 
 | Member | Signature | Summary |
 | --- | --- | --- |
@@ -9003,8 +9003,8 @@ WORM writer for the NumPy NPY array serialization format (NEP 1). Emits a v1 fil
 | Member | Signature | Summary |
 | --- | --- | --- |
 | `DefaultDtype` | `const string DefaultDtype` | Default dtype string used when no explicit type is supplied. |
-| `Write` | `static void Write(Stream output, ReadOnlySpan<byte> payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. |
-| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. |
+| `Write` | `static void Write(Stream output, ReadOnlySpan<byte> payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. |
+| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. |
 
 #### `NpzFormatDescriptor`
 
@@ -11909,8 +11909,8 @@ Reader and writer for the Microsoft SZDD / COMPRESS.EXE file format. SZDD uses a
 | --- | --- | --- |
 | `CompressQBasic` | `static byte[] CompressQBasic(ReadOnlySpan<byte> data)` | Compresses `data` in the older "SZ " (QBasic) COMPRESS variant and returns the result. The body is the same LZSS stream as SZDD, wrapped in the 12-byte "SZ " header (8-byte magic + little-endian u32 uncompressed length). Round-trips through `Decompress`. |
 | `CompressQBasic` | `static void CompressQBasic(Stream input, Stream output)` | Stream overload of `CompressQBasic`. |
-| `Compress` | `static byte[] Compress(ReadOnlySpan<byte> data, char missingChar = _)` | Compresses `data` in SZDD format and returns the result as a new byte array. |
-| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = _)` | Compresses `input` in SZDD format and writes the result to `output`. |
+| `Compress` | `static byte[] Compress(ReadOnlySpan<byte> data, char missingChar = '_')` | Compresses `data` in SZDD format and returns the result as a new byte array. |
+| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = '_')` | Compresses `input` in SZDD format and writes the result to `output`. |
 | `Decompress` | `static byte[] Decompress(ReadOnlySpan<byte> data)` | Decompresses an SZDD-encoded byte array and returns the raw data. |
 | `Decompress` | `static void Decompress(Stream input, Stream output)` | Decompresses an SZDD-encoded stream and writes the raw data to `output`. |
 | `GetMissingChar` | `static char GetMissingChar(Stream input)` | Returns the "missing character" stored in the SZDD header — the last character of the original filename extension before it was replaced with `'_'`. |