diff --git a/Compression.Core/Entropy/BpeBuildingBlock.cs b/Compression.Core/Entropy/BpeBuildingBlock.cs index 80eb6c97d..e8a90412f 100644 --- a/Compression.Core/Entropy/BpeBuildingBlock.cs +++ b/Compression.Core/Entropy/BpeBuildingBlock.cs @@ -3,182 +3,406 @@ namespace Compression.Core.Entropy; +/// Controls how a byte-pair grammar is constructed. +public enum BpeConstructionStrategy { + /// Repeatedly replace the currently most frequent profitable pair. + Greedy, + + /// Explore every profitable merge sequence within each search block and keep the smallest result. + Exhaustive, +} + /// -/// Exposes Byte Pair Encoding (BPE) as a benchmarkable building block. -/// Iteratively replaces the most frequent consecutive byte pair with a new symbol. -/// Header: 2-byte LE dictionary size, 6 bytes per entry (code, val1, val2), -/// 4-byte LE data length (in values), 2 bytes per encoded value. +/// Exposes Philip Gage's byte-pair compression as a benchmarkable building block. +/// Repeated adjacent byte pairs are replaced by byte values that do not occur in the +/// current block; the replacement table is stored with the encoded bytes. /// +/// +/// +/// Greedy construction uses blocks of at most 65,535 bytes and repeatedly takes the +/// most frequent profitable pair. Exhaustive construction is combinatorial, so it uses +/// 64-byte search blocks and evaluates every profitable merge sequence in each block. +/// The wire format is identical for both strategies and the decoder is strategy-agnostic. +/// +/// +/// Wire format: 4-byte little-endian original length, followed by blocks. Each block +/// starts with its 2-byte original length and 2-byte stored length. Equal lengths mean +/// the block is raw. A compressed block contains one byte rule count, then +/// (code,left,right) triples in creation order, followed by the encoded byte +/// sequence. A replacement is profitable when its actual non-overlapping occurrence +/// count exceeds the three-byte rule cost. +/// +/// +/// Reference: Philip Gage, "A New Algorithm for Data Compression", Dr. Dobb's Journal, +/// February 1994. +/// +/// public sealed class BpeBuildingBlock : IBuildingBlock { /// public string Id => "BB_BPE"; /// public string DisplayName => "Byte Pair Encoding"; /// - public string Description => "Iterative most-frequent pair replacement"; + public string Description => "Philip Gage byte-pair compression using unused byte symbols"; /// - public AlgorithmFamily Family => AlgorithmFamily.Entropy; + public AlgorithmFamily Family => AlgorithmFamily.Dictionary; - private const int MaxIterations = 256; - private const int FirstCode = 256; + /// The grammar-construction strategy used by . + public BpeConstructionStrategy ConstructionStrategy { get; } + + private const int MaxBlockLength = ushort.MaxValue; + private const int ExhaustiveBlockLength = 64; + private const int AlphabetSize = 256; + private const int PairCount = AlphabetSize * AlphabetSize; + private const int RuleSize = 3; + + /// Creates a BPE building block using greedy grammar construction. + public BpeBuildingBlock() : this(BpeConstructionStrategy.Greedy) { } + + /// Creates a BPE building block using the requested grammar-construction strategy. + /// How pair substitutions are selected. + public BpeBuildingBlock(BpeConstructionStrategy constructionStrategy) { + if (!Enum.IsDefined(constructionStrategy)) + throw new ArgumentOutOfRangeException(nameof(constructionStrategy)); + this.ConstructionStrategy = constructionStrategy; + } /// public byte[] Compress(ReadOnlySpan data) { - using var ms = new MemoryStream(); - - // Work with an int array so codes > 255 are representable. - var dataArr = new int[data.Length]; - for (var i = 0; i < data.Length; i++) - dataArr[i] = data[i]; - var dataLen = data.Length; - - var dictionary = new List<(int code, int val1, int val2)>(); - var nextCode = FirstCode; - - var pairCounts = new Dictionary(Math.Min(dataLen, 4096)); - - for (var iter = 0; iter < MaxIterations && dataLen >= 2; iter++) { - // Find most frequent consecutive pair. - pairCounts.Clear(); - for (var i = 0; i < dataLen - 1; i++) { - var key = ((long)dataArr[i] << 32) | (uint)dataArr[i + 1]; - pairCounts.TryGetValue(key, out var count); - pairCounts[key] = count + 1; + using var output = new MemoryStream(); + Span integer = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(integer, data.Length); + output.Write(integer); + + var maximumBlockLength = this.ConstructionStrategy == BpeConstructionStrategy.Exhaustive + ? ExhaustiveBlockLength + : MaxBlockLength; + + for (var offset = 0; offset < data.Length;) { + var blockLength = Math.Min(maximumBlockLength, data.Length - offset); + WriteBlock(output, data.Slice(offset, blockLength), this.ConstructionStrategy); + offset += blockLength; + } + + return output.ToArray(); + } + + /// + public byte[] Decompress(ReadOnlySpan data) { + if (data.Length < sizeof(int)) + throw new InvalidDataException("BPE stream is missing its original-length header."); + + var originalLength = BinaryPrimitives.ReadInt32LittleEndian(data); + if (originalLength < 0) + throw new InvalidDataException("BPE stream declares a negative original length."); + if (originalLength == 0) { + if (data.Length != sizeof(int)) + throw new InvalidDataException("BPE empty stream contains trailing data."); + return []; + } + + var result = new byte[originalLength]; + var inputOffset = sizeof(int); + var outputOffset = 0; + + while (outputOffset < originalLength) { + if (inputOffset + 4 > data.Length) + throw new InvalidDataException("BPE stream ends inside a block header."); + + var blockLength = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(inputOffset, 2)); + var storedLength = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(inputOffset + 2, 2)); + inputOffset += 4; + + if (blockLength == 0) + throw new InvalidDataException("BPE stream contains a zero-length block."); + if (blockLength > originalLength - outputOffset) + throw new InvalidDataException("BPE block expands past the declared original length."); + if (storedLength == 0 || inputOffset + storedLength > data.Length) + throw new InvalidDataException("BPE stream ends inside a block payload."); + + var block = data.Slice(inputOffset, storedLength); + inputOffset += storedLength; + + if (storedLength == blockLength) { + block.CopyTo(result.AsSpan(outputOffset, blockLength)); + } else { + DecodeBlock(block, result.AsSpan(outputOffset, blockLength)); } - // Pick the most frequent pair, ties going to the one that occurs earliest. - // Scanning the data rather than the count table is what makes that rule - // explicit: the winner is decided by positions in the input, not by the - // order in which a hash table hands its entries back. - long bestKey = 0; - var bestCount = 0; - for (var i = 0; i < dataLen - 1; i++) { - var key = ((long)dataArr[i] << 32) | (uint)dataArr[i + 1]; - var count = pairCounts[key]; - if (count <= bestCount) + outputOffset += blockLength; + } + + if (inputOffset != data.Length) + throw new InvalidDataException("BPE stream contains trailing data."); + + return result; + } + + private static void WriteBlock(Stream output, ReadOnlySpan block, BpeConstructionStrategy strategy) { + var freeCodes = FindFreeCodes(block); + var encoded = strategy == BpeConstructionStrategy.Exhaustive + ? BuildExhaustive(block, freeCodes) + : BuildGreedy(block, freeCodes); + + var compressedLength = 1 + encoded.Rules.Length * RuleSize + encoded.Sequence.Length; + Span header = stackalloc byte[4]; + BinaryPrimitives.WriteUInt16LittleEndian(header, checked((ushort)block.Length)); + + if (compressedLength >= block.Length) { + BinaryPrimitives.WriteUInt16LittleEndian(header[2..], checked((ushort)block.Length)); + output.Write(header); + output.Write(block); + return; + } + + BinaryPrimitives.WriteUInt16LittleEndian(header[2..], checked((ushort)compressedLength)); + output.Write(header); + output.WriteByte(checked((byte)encoded.Rules.Length)); + foreach (var rule in encoded.Rules) { + output.WriteByte(rule.Code); + output.WriteByte(rule.Left); + output.WriteByte(rule.Right); + } + output.Write(encoded.Sequence); + } + + private static byte[] FindFreeCodes(ReadOnlySpan block) { + Span unavailable = stackalloc bool[AlphabetSize]; + foreach (var value in block) + unavailable[value] = true; + + var result = new byte[AlphabetSize]; + var count = 0; + for (var value = AlphabetSize - 1; value >= 0; --value) + if (!unavailable[value]) + result[count++] = (byte)value; + + return result.AsSpan(0, count).ToArray(); + } + + private static EncodedBlock BuildGreedy(ReadOnlySpan block, byte[] freeCodes) { + var sequence = block.ToArray(); + var length = sequence.Length; + var freeCount = freeCodes.Length; + var rules = new List(freeCount); + var counts = new int[PairCount]; + var earliest = new int[PairCount]; + var lastEnd = new int[PairCount]; + + while (freeCount > 0 && length >= 2) { + Array.Clear(counts); + Array.Fill(earliest, int.MaxValue); + Array.Fill(lastEnd, -1); + + for (var i = 0; i + 1 < length; ++i) { + var pair = sequence[i] << 8 | sequence[i + 1]; + if (lastEnd[pair] >= i) continue; + lastEnd[pair] = i + 1; + ++counts[pair]; + if (earliest[pair] == int.MaxValue) + earliest[pair] = i; + } + var bestPair = -1; + var bestCount = RuleSize; + var bestPosition = int.MaxValue; + for (var pair = 0; pair < PairCount; ++pair) { + var count = counts[pair]; + if (count < bestCount || count == bestCount && earliest[pair] >= bestPosition) + continue; + bestPair = pair; bestCount = count; - bestKey = key; + bestPosition = earliest[pair]; } - // Stop if the best pair doesn't save enough to justify the dictionary entry cost. - var netSavings = (long)bestCount * 2 - 6; - if (netSavings <= 0) + if (bestPair < 0 || bestCount <= RuleSize) break; - var b1 = (int)(bestKey >> 32); - var b2 = (int)(bestKey & 0xFFFFFFFF); + var left = (byte)(bestPair >> 8); + var right = (byte)bestPair; + var code = freeCodes[--freeCount]; + rules.Add(new Rule(code, left, right)); - // Replace all occurrences in-place. - var prevLen = dataLen; - var writePos = 0; - for (var i = 0; i < dataLen; i++) { - if (i < dataLen - 1 && dataArr[i] == b1 && dataArr[i + 1] == b2) { - dataArr[writePos++] = nextCode; - i++; // skip next + var write = 0; + for (var read = 0; read < length;) { + if (read + 1 < length && sequence[read] == left && sequence[read + 1] == right) { + sequence[write++] = code; + read += 2; } else { - dataArr[writePos++] = dataArr[i]; + sequence[write++] = sequence[read++]; } } - dataLen = writePos; + length = write; + } - dictionary.Add((nextCode, b1, b2)); - nextCode++; + return new EncodedBlock([.. rules], sequence.AsSpan(0, length).ToArray()); + } - // Stop if this iteration shrank the data by less than 0.5%. The product is - // taken in 64-bit: a single round on a large input can remove far more than - // int.MaxValue / 200 pairs, and the wrapped value then reads as "no progress" - // and aborts the merge loop after one round, costing compression ratio. - if ((long)(prevLen - dataLen) * 200 < prevLen) - break; + private static EncodedBlock BuildExhaustive(ReadOnlySpan block, byte[] freeCodes) { + var search = new ExhaustiveSearch(freeCodes); + var result = search.FindBest(block.ToArray(), freeCodes.Length); + return new EncodedBlock(result.Rules, result.Sequence); + } + + private static List FindProfitablePairs(ReadOnlySpan sequence) { + var pairStats = new Dictionary(Math.Max(0, sequence.Length - 1)); + + for (var i = 0; i + 1 < sequence.Length; ++i) { + var pair = sequence[i] << 8 | sequence[i + 1]; + if (!pairStats.TryGetValue(pair, out var stats)) + stats = new PairStats(0, i, -1); + + if (stats.LastEnd >= i) + continue; + + ++stats.Count; + stats.LastEnd = i + 1; + pairStats[pair] = stats; } - // Write header: 2-byte LE dictionary size. - Span header = stackalloc byte[2]; - BinaryPrimitives.WriteUInt16LittleEndian(header, (ushort)dictionary.Count); - ms.Write(header); - - // Dictionary entries: 2-byte LE code, 2-byte LE val1, 2-byte LE val2. - Span entry = stackalloc byte[6]; - foreach (var (code, val1, val2) in dictionary) { - BinaryPrimitives.WriteUInt16LittleEndian(entry, (ushort)code); - BinaryPrimitives.WriteUInt16LittleEndian(entry[2..], (ushort)val1); - BinaryPrimitives.WriteUInt16LittleEndian(entry[4..], (ushort)val2); - ms.Write(entry); + var result = new List(pairStats.Count); + foreach (var (pair, stats) in pairStats) { + // A code created by k substitutions occurs exactly k times. Any later rule + // containing that code can therefore occur at most k times as well. A pair + // used three times or fewer can never lead to a future rule that recovers + // its three-byte rule cost, so excluding it does not prune an optimal grammar. + if (stats.Count <= RuleSize) + continue; + result.Add(new PairCandidate((byte)(pair >> 8), (byte)pair, stats.Count, stats.Earliest)); } - // Data length (number of values) as 4-byte LE. - Span lenBuf = stackalloc byte[4]; - BinaryPrimitives.WriteInt32LittleEndian(lenBuf, dataLen); - ms.Write(lenBuf); + result.Sort(static (left, right) => { + var order = right.Count.CompareTo(left.Count); + if (order != 0) + return order; + order = left.Earliest.CompareTo(right.Earliest); + if (order != 0) + return order; + order = left.Left.CompareTo(right.Left); + return order != 0 ? order : left.Right.CompareTo(right.Right); + }); + return result; + } + + private static byte[] ReplacePair(ReadOnlySpan sequence, PairCandidate pair, byte code) { + var result = new byte[sequence.Length - pair.Count]; + var write = 0; - // Encoded data: 2 bytes per value. - Span valBuf = stackalloc byte[2]; - for (var i = 0; i < dataLen; i++) { - BinaryPrimitives.WriteUInt16LittleEndian(valBuf, (ushort)dataArr[i]); - ms.Write(valBuf); + for (var read = 0; read < sequence.Length;) { + if (read + 1 < sequence.Length && sequence[read] == pair.Left && sequence[read + 1] == pair.Right) { + result[write++] = code; + read += 2; + } else { + result[write++] = sequence[read++]; + } } - return ms.ToArray(); + return result; } - /// - public byte[] Decompress(ReadOnlySpan data) { - var offset = 0; - - var dictSize = BinaryPrimitives.ReadUInt16LittleEndian(data); - offset += 2; - - // Read dictionary in order. - var rules = new (ushort code, ushort val1, ushort val2)[dictSize]; - for (var i = 0; i < dictSize; i++) { - rules[i] = ( - BinaryPrimitives.ReadUInt16LittleEndian(data[offset..]), - BinaryPrimitives.ReadUInt16LittleEndian(data[(offset + 2)..]), - BinaryPrimitives.ReadUInt16LittleEndian(data[(offset + 4)..]) - ); - offset += 6; - } + private static void DecodeBlock(ReadOnlySpan block, Span destination) { + if (block.IsEmpty) + throw new InvalidDataException("BPE compressed block is empty."); + + var ruleCount = block[0]; + var rulesLength = 1 + ruleCount * RuleSize; + if (rulesLength >= block.Length) + throw new InvalidDataException("BPE compressed block has no encoded payload."); - // Read data length. - var dataLen = BinaryPrimitives.ReadInt32LittleEndian(data[offset..]); - offset += 4; + Span rules = stackalloc Rule[AlphabetSize]; + Span ruleIndex = stackalloc int[AlphabetSize]; + ruleIndex.Fill(-1); - // Read encoded data into array. - var arr = new int[dataLen + dictSize * 2]; // extra space for expansion - for (var i = 0; i < dataLen; i++) { - arr[i] = BinaryPrimitives.ReadUInt16LittleEndian(data[offset..]); - offset += 2; + var offset = 1; + for (var index = 0; index < ruleCount; ++index) { + var code = block[offset++]; + var left = block[offset++]; + var right = block[offset++]; + if (ruleIndex[code] >= 0) + throw new InvalidDataException("BPE compressed block defines a replacement code twice."); + rules[index] = new Rule(code, left, right); + ruleIndex[code] = index; } - var currentLen = dataLen; - - // Expand codes in reverse order (highest code first). - var buffer = new int[arr.Length * 2]; - for (var i = rules.Length - 1; i >= 0; i--) { - var (code, val1, val2) = rules[i]; - var maxLen = currentLen * 2; - if (buffer.Length < maxLen) - buffer = new int[maxLen]; - - var writePos = 0; - for (var j = 0; j < currentLen; j++) { - if (arr[j] == code) { - buffer[writePos++] = val1; - buffer[writePos++] = val2; - } else { - buffer[writePos++] = arr[j]; + + for (var index = 0; index < ruleCount; ++index) { + var rule = rules[index]; + var leftRule = ruleIndex[rule.Left]; + var rightRule = ruleIndex[rule.Right]; + if (leftRule >= index || rightRule >= index) + throw new InvalidDataException("BPE compressed block contains a forward or cyclic rule reference."); + } + + Span expansion = stackalloc byte[AlphabetSize]; + var written = 0; + while (offset < block.Length) { + var stackLength = 1; + expansion[0] = block[offset++]; + + while (stackLength > 0) { + var symbol = expansion[--stackLength]; + var index = ruleIndex[symbol]; + if (index < 0) { + if (written >= destination.Length) + throw new InvalidDataException("BPE block expands past its declared length."); + destination[written++] = symbol; + continue; } - } - (arr, buffer) = (buffer, arr); - currentLen = writePos; + if (stackLength + 2 > expansion.Length) + throw new InvalidDataException("BPE rule expansion is deeper than the byte alphabet permits."); + var rule = rules[index]; + expansion[stackLength++] = rule.Right; + expansion[stackLength++] = rule.Left; + } } - // All values should now be in 0-255 range. - var result = new byte[currentLen]; - for (var i = 0; i < currentLen; i++) - result[i] = (byte)arr[i]; - return result; + if (written != destination.Length) + throw new InvalidDataException("BPE block does not expand to its declared length."); } + + private sealed class ExhaustiveSearch(byte[] freeCodes) { + private readonly Dictionary<(int FreeCount, string Sequence), SearchResult> _memo = []; + + public SearchResult FindBest(byte[] sequence, int freeCount) { + var key = (freeCount, Convert.ToHexString(sequence)); + if (this._memo.TryGetValue(key, out var memoized)) + return memoized; + + var best = new SearchResult(sequence.Length, [], sequence); + if (freeCount == 0 || sequence.Length < 2) { + this._memo[key] = best; + return best; + } + + var candidates = FindProfitablePairs(sequence); + if (candidates.Count == 0) { + this._memo[key] = best; + return best; + } + + var code = freeCodes[freeCount - 1]; + foreach (var candidate in candidates) { + var replaced = ReplacePair(sequence, candidate, code); + var child = this.FindBest(replaced, freeCount - 1); + var cost = RuleSize + child.Cost; + if (cost >= best.Cost) + continue; + + var rules = new Rule[child.Rules.Length + 1]; + rules[0] = new Rule(code, candidate.Left, candidate.Right); + child.Rules.CopyTo(rules, 1); + best = new SearchResult(cost, rules, child.Sequence); + } + + this._memo[key] = best; + return best; + } + } + + private readonly record struct EncodedBlock(Rule[] Rules, byte[] Sequence); + private readonly record struct Rule(byte Code, byte Left, byte Right); + private readonly record struct PairCandidate(byte Left, byte Right, int Count, int Earliest); + private record struct PairStats(int Count, int Earliest, int LastEnd); + private sealed record SearchResult(int Cost, Rule[] Rules, byte[] Sequence); } diff --git a/Compression.Core/README.md b/Compression.Core/README.md index 8d55e91c4..2b0c9e389 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -4230,7 +4230,7 @@ On-disk RAID metadata format a member superblock was recognised as. ### Namespace `Compression.Core.Entropy` -[`ArithmeticBuildingBlock`](#arithmeticbuildingblock) · [`BpeBuildingBlock`](#bpebuildingblock) · [`DmcBuildingBlock`](#dmcbuildingblock) · [`EliasDeltaBuildingBlock`](#eliasdeltabuildingblock) · [`EliasGammaBuildingBlock`](#eliasgammabuildingblock) · [`FibonacciBuildingBlock`](#fibonaccibuildingblock) · [`FseBuildingBlock`](#fsebuildingblock) · [`GolombBuildingBlock`](#golombbuildingblock) · [`GolombFixedMBuildingBlock`](#golombfixedmbuildingblock) · [`GolombProfile`](#golombprofile) · [`LevenshteinBuildingBlock`](#levenshteinbuildingblock) · [`OmegaBuildingBlock`](#omegabuildingblock) · [`RangeCodingBuildingBlock`](#rangecodingbuildingblock) · [`ShannonFanoBuildingBlock`](#shannonfanobuildingblock) · [`TunstallBuildingBlock`](#tunstallbuildingblock) · [`UnaryBuildingBlock`](#unarybuildingblock) +[`ArithmeticBuildingBlock`](#arithmeticbuildingblock) · [`BpeBuildingBlock`](#bpebuildingblock) · [`BpeConstructionStrategy`](#bpeconstructionstrategy) · [`DmcBuildingBlock`](#dmcbuildingblock) · [`EliasDeltaBuildingBlock`](#eliasdeltabuildingblock) · [`EliasGammaBuildingBlock`](#eliasgammabuildingblock) · [`FibonacciBuildingBlock`](#fibonaccibuildingblock) · [`FseBuildingBlock`](#fsebuildingblock) · [`GolombBuildingBlock`](#golombbuildingblock) · [`GolombFixedMBuildingBlock`](#golombfixedmbuildingblock) · [`GolombProfile`](#golombprofile) · [`LevenshteinBuildingBlock`](#levenshteinbuildingblock) · [`OmegaBuildingBlock`](#omegabuildingblock) · [`RangeCodingBuildingBlock`](#rangecodingbuildingblock) · [`ShannonFanoBuildingBlock`](#shannonfanobuildingblock) · [`TunstallBuildingBlock`](#tunstallbuildingblock) · [`UnaryBuildingBlock`](#unarybuildingblock) #### `ArithmeticBuildingBlock` @@ -4250,13 +4250,15 @@ Implements `IBuildingBlock`. #### `BpeBuildingBlock` -Exposes Byte Pair Encoding (BPE) as a benchmarkable building block. Iteratively replaces the most frequent consecutive byte pair with a new symbol. Header: 2-byte LE dictionary size, 6 bytes per entry (code, val1, val2), 4-byte LE data length (in values), 2 bytes per encoded value. +Exposes Philip Gage's byte-pair compression as a benchmarkable building block. Repeated adjacent byte pairs are replaced by byte values that do not occur in the current block; the replacement table is stored with the encoded bytes. Implements `IBuildingBlock`. | Member | Signature | Summary | | --- | --- | --- | -| `BpeBuildingBlock` | `BpeBuildingBlock()` | | +| `BpeBuildingBlock` | `BpeBuildingBlock()` | Creates a BPE building block using greedy grammar construction. | +| `BpeBuildingBlock` | `BpeBuildingBlock(BpeConstructionStrategy constructionStrategy)` | Creates a BPE building block using the requested grammar-construction strategy. | +| `ConstructionStrategy` | `BpeConstructionStrategy ConstructionStrategy { get; }` | The grammar-construction strategy used by `Compress`. | | `Description` | `string Description { get; }` | | | `DisplayName` | `string DisplayName { get; }` | | | `Family` | `AlgorithmFamily Family { get; }` | | @@ -4264,6 +4266,15 @@ Implements `IBuildingBlock`. | `Compress` | `byte[] Compress(ReadOnlySpan data)` | | | `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | | +#### `BpeConstructionStrategy` + +Controls how a byte-pair grammar is constructed. + +| Value | Numeric | Summary | +| --- | --- | --- | +| `Greedy` | `0` | Repeatedly replace the currently most frequent profitable pair. | +| `Exhaustive` | `1` | Explore every profitable merge sequence within each search block and keep the smallest result. | + #### `DmcBuildingBlock` Exposes Dynamic Markov Compression as a benchmarkable building block. Bit-level finite-context modeling with state cloning: a finite automaton where each state predicts the next bit. States that become too frequent are cloned to create higher-order contexts, improving prediction. Uses an arithmetic coder for the output bitstream. diff --git a/Compression.Tests/BuildingBlocks/BpeBuildingBlockTests.cs b/Compression.Tests/BuildingBlocks/BpeBuildingBlockTests.cs new file mode 100644 index 000000000..d30819fff --- /dev/null +++ b/Compression.Tests/BuildingBlocks/BpeBuildingBlockTests.cs @@ -0,0 +1,158 @@ +using System.Buffers.Binary; +using System.Text; +using Compression.Core.Entropy; +using Compression.Registry; + +namespace Compression.Tests.BuildingBlocks; + +[TestFixture] +public sealed class BpeBuildingBlockTests { + private static readonly BpeBuildingBlock Bb = new(); + private static readonly BpeBuildingBlock Exhaustive = new(BpeConstructionStrategy.Exhaustive); + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Empty_RoundTrips() { + var compressed = Bb.Compress([]); + var roundTrip = Bb.Decompress(compressed); + + Assert.Multiple(() => { + Assert.That(compressed, Is.EqualTo(new byte[] { 0, 0, 0, 0 }).AsCollection); + Assert.That(roundTrip, Is.Empty); + }); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void RepeatedByte_RoundTripsAndCompresses() { + var data = new byte[20_000]; + Array.Fill(data, (byte)'a'); + + var compressed = Bb.Compress(data); + var roundTrip = Bb.Decompress(compressed); + + Assert.Multiple(() => { + Assert.That(compressed.Length, Is.LessThan(data.Length / 10)); + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + }); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void RepeatedPhrase_RoundTripsAndCompresses() { + const string phrase = "the quick brown fox jumps over the lazy dog. "; + var data = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(phrase, 512))); + + var compressed = Bb.Compress(data); + var roundTrip = Bb.Decompress(compressed); + + Assert.Multiple(() => { + Assert.That(compressed.Length, Is.LessThan(data.Length * 3 / 4)); + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + }); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void IncompressibleRandom_RoundTrips() { + var rng = new Random(0xB0E); + var data = new byte[8192]; + rng.NextBytes(data); + + var roundTrip = Bb.Decompress(Bb.Compress(data)); + + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + } + + [Test, Category("EdgeCase"), Category("RoundTrip")] + public void AllByteValues_FallsBackToRawBlock() { + var data = Enumerable.Range(0, 256).Select(value => (byte)value).ToArray(); + + var compressed = Bb.Compress(data); + var roundTrip = Bb.Decompress(compressed); + + Assert.Multiple(() => { + Assert.That(BinaryPrimitives.ReadInt32LittleEndian(compressed), Is.EqualTo(256)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(compressed.AsSpan(4, 2)), Is.EqualTo(256)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(compressed.AsSpan(6, 2)), Is.EqualTo(256)); + Assert.That(compressed.AsSpan(8).ToArray(), Is.EqualTo(data).AsCollection); + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + }); + } + + [Test, Category("EdgeCase"), Category("RoundTrip")] + public void MoreThanOneBlock_RoundTripsAcrossBoundary() { + var data = new byte[ushort.MaxValue + 4096]; + for (var i = 0; i < data.Length; ++i) + data[i] = (byte)(i * 31 + i / 17); + + var roundTrip = Bb.Decompress(Bb.Compress(data)); + + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + } + + [Test, Category("EdgeCase")] + public void KnownVector_UsesDeterministicPairOrder() { + var data = Encoding.ASCII.GetBytes("abababababababab"); + + var compressed = Bb.Compress(data); + + Assert.That(compressed, Is.EqualTo(new byte[] { + 16, 0, 0, 0, + 16, 0, 11, 0, + 2, + 0, (byte)'a', (byte)'b', + 1, 0, 0, + 1, 1, 1, 1, + }).AsCollection); + Assert.That(Bb.Decompress(compressed), Is.EqualTo(data).AsCollection); + } + + [Test, Category("EdgeCase"), Category("RoundTrip")] + public void ExhaustiveConstruction_BeatsGreedyOnNonGreedyGrammar() { + byte[] data = [0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1]; + + var greedy = Bb.Compress(data); + var exhaustive = Exhaustive.Compress(data); + + Assert.Multiple(() => { + Assert.That(greedy.Length, Is.EqualTo(21)); + Assert.That(exhaustive.Length, Is.EqualTo(20)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(greedy.AsSpan(6, 2)), Is.EqualTo(13)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(exhaustive.AsSpan(6, 2)), Is.EqualTo(12)); + Assert.That(Bb.Decompress(exhaustive), Is.EqualTo(data).AsCollection); + Assert.That(Exhaustive.Decompress(greedy), Is.EqualTo(data).AsCollection); + }); + } + + [Test, Category("EdgeCase"), Category("RoundTrip")] + public void ExhaustiveConstruction_RoundTripsAcrossSearchBlocks() { + var data = new byte[130]; + Array.Fill(data, (byte)'a'); + + var compressed = Exhaustive.Compress(data); + + Assert.That(Bb.Decompress(compressed), Is.EqualTo(data).AsCollection); + } + + [Test, Category("EdgeCase")] + public void MalformedForwardRule_IsRejected() { + byte[] malformed = { + 4, 0, 0, 0, + 4, 0, 8, 0, + 2, + 0, 1, (byte)'a', + 1, (byte)'b', (byte)'c', + 0, + }; + + Assert.That(() => Bb.Decompress(malformed), Throws.TypeOf()); + } + + [Test, Category("EdgeCase")] + public void Registry_Metadata_IsStable() { + Assert.Multiple(() => { + Assert.That(Bb.Id, Is.EqualTo("BB_BPE")); + Assert.That(Bb.DisplayName, Is.EqualTo("Byte Pair Encoding")); + Assert.That(Bb.Family, Is.EqualTo(AlgorithmFamily.Dictionary)); + Assert.That(Bb.ConstructionStrategy, Is.EqualTo(BpeConstructionStrategy.Greedy)); + Assert.That(Exhaustive.ConstructionStrategy, Is.EqualTo(BpeConstructionStrategy.Exhaustive)); + }); + } +} diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 6b7a34cac..656231075 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -3222,7 +3222,7 @@ Implements `IFormatDescriptor`, `IStreamFormatOperations`. #### `CscStream` -CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size \| uint24 csc_blocksize \| uint24 raw_blocksize \| uint32 actual_size +CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size | uint24 csc_blocksize | uint24 raw_blocksize | uint32 actual_size | Member | Signature | Summary | | --- | --- | --- | @@ -4820,7 +4820,7 @@ DEFLATE (RFC 1951) decoder for the PEtite dialect: block type 1 carries the dyna #### `PetiteUnpacker` -Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000\|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments) +Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments) | Member | Signature | Summary | | --- | --- | --- | @@ -9003,8 +9003,8 @@ WORM writer for the NumPy NPY array serialization format (NEP 1). Emits a v1 fil | Member | Signature | Summary | | --- | --- | --- | | `DefaultDtype` | `const string DefaultDtype` | Default dtype string used when no explicit type is supplied. | -| `Write` | `static void Write(Stream output, ReadOnlySpan payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. | -| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. | +| `Write` | `static void Write(Stream output, ReadOnlySpan payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. | +| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. | #### `NpzFormatDescriptor` @@ -11909,8 +11909,8 @@ Reader and writer for the Microsoft SZDD / COMPRESS.EXE file format. SZDD uses a | --- | --- | --- | | `CompressQBasic` | `static byte[] CompressQBasic(ReadOnlySpan data)` | Compresses `data` in the older "SZ " (QBasic) COMPRESS variant and returns the result. The body is the same LZSS stream as SZDD, wrapped in the 12-byte "SZ " header (8-byte magic + little-endian u32 uncompressed length). Round-trips through `Decompress`. | | `CompressQBasic` | `static void CompressQBasic(Stream input, Stream output)` | Stream overload of `CompressQBasic`. | -| `Compress` | `static byte[] Compress(ReadOnlySpan data, char missingChar = _)` | Compresses `data` in SZDD format and returns the result as a new byte array. | -| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = _)` | Compresses `input` in SZDD format and writes the result to `output`. | +| `Compress` | `static byte[] Compress(ReadOnlySpan data, char missingChar = '_')` | Compresses `data` in SZDD format and returns the result as a new byte array. | +| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = '_')` | Compresses `input` in SZDD format and writes the result to `output`. | | `Decompress` | `static byte[] Decompress(ReadOnlySpan data)` | Decompresses an SZDD-encoded byte array and returns the raw data. | | `Decompress` | `static void Decompress(Stream input, Stream output)` | Decompresses an SZDD-encoded stream and writes the raw data to `output`. | | `GetMissingChar` | `static char GetMissingChar(Stream input)` | Returns the "missing character" stored in the SZDD header — the last character of the original filename extension before it was replaced with `'_'`. | diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index af7ac0e76..b2bf80568 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -921,7 +921,7 @@ Reader for Intel HEX records (`:LLAAAATT[DD…]CC`), the long-standing flash-pro #### `SRecordReader` -Reader for Motorola S-Record files (`Stnn[aaaa\|aaaaaa\|aaaaaaaa]dd…cc`). Recognised types: S0 header, S1/S2/S3 data (16/24/32-bit address), S5/S6 record counts (informational), S7/S8/S9 termination (32/24/16-bit start addr). +Reader for Motorola S-Record files (`Stnn[aaaa|aaaaaa|aaaaaaaa]dd…cc`). Recognised types: S0 header, S1/S2/S3 data (16/24/32-bit address), S5/S6 record counts (informational), S7/S8/S9 termination (32/24/16-bit start addr). | Member | Signature | Summary | | --- | --- | --- | @@ -2820,7 +2820,7 @@ Random-access in-place modifier for BBC Micro Acorn DFS `.ssd` images. The DFS c | Member | Signature | Summary | | --- | --- | --- | -| `AddFile` | `static void AddFile(Stream image, string name, byte[] data, char directory = $, uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | Adds a file to an existing single-sided DFS image. Caller is responsible for ensuring the name does not already exist (use `RemoveFile` first for replace-by-name semantics). The file is placed in the lowest contiguous gap large enough to hold it. | +| `AddFile` | `static void AddFile(Stream image, string name, byte[] data, char directory = '$', uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | Adds a file to an existing single-sided DFS image. Caller is responsible for ensuring the name does not already exist (use `RemoveFile` first for replace-by-name semantics). The file is placed in the lowest contiguous gap large enough to hold it. | | `RemoveFile` | `static bool RemoveFile(Stream image, string name, bool wipeData = true)` | Removes a named file from the image. Returns true if found and removed. When `wipeData` is true, the data sectors are zeroed. | #### `BbcReader` @@ -2855,7 +2855,7 @@ Builds a fresh BBC Micro Acorn DFS `.ssd` single-sided disk image from scratch ( | `SectorSize` | `const int SectorSize` | | | `SectorsPerTrack` | `const int SectorsPerTrack` | | | `TotalSectors40` | `const int TotalSectors40` | | -| `AddFile` | `void AddFile(string name, byte[] data, char directory = $, uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | | +| `AddFile` | `void AddFile(string name, byte[] data, char directory = '$', uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | | | `Build` | `byte[] Build(string diskTitle = "WORMDISK", int bootOption = 0)` | Builds the complete 40-track SSD image (100 000 bytes). | ### Namespace `FileSystem.BcacheFs` @@ -3232,7 +3232,7 @@ From-scratch writer for the Commodore nibble container the `CbmNibbleReader` con | `AddFile` | `void AddFile(string name, byte[] data)` | Adds a file to the flat directory. Commodore names are PETSCII and at most 16 characters; longer names are truncated. The default file type is PRG. | | `Build` | `byte[] Build()` | Builds the G64 GCR nibble image holding all added files. | | `DecodeToD64` | `static byte[] DecodeToD64(NibbleImage image)` | Reconstructs a standard 174 848-byte D64 image from the GCR tracks of a nibble image previously parsed by `CbmNibbleReader`. Each track is rescanned for sync marks and its header/data blocks GCR-decoded back into the correct sector slots. | -| `SetDisk` | `void SetDisk(string name, char id1 = 0, char id2 = 0)` | Sets the on-disk volume name (PETSCII, ≤16 chars) and the 2-byte disk id. | +| `SetDisk` | `void SetDisk(string name, char id1 = '0', char id2 = '0')` | Sets the on-disk volume name (PETSCII, ≤16 chars) and the 2-byte disk id. | | `WriteTo` | `void WriteTo(Stream output)` | Writes the G64 image to `output`. | #### `G64FormatDescriptor` @@ -6152,7 +6152,7 @@ Implements `IDisposable`. #### `JfsWriter` -Writes a minimal IBM Journaled File System (JFS1) aggregate image with a single allocation group, one fileset, and an inline dtree root directory. Byte layout matches the on-disk structures in `linux/fs/jfs` and the `jfsutils` reference (mkfs.jfs / fsck.jfs); validated by exit-zero from `fsck.jfs -n -f -v`. All integer fields are little-endian. `pxd_t` is packed as `len_addr = (len & 0xFFFFFF) \| ((addr >> 32) << 24)`, `addr2 = addr & 0xFFFFFFFF`. Dtree slot names are UCS-2 (UTF-16 LE). Round-trips through `JfsReader`. Aggregate inode table (block 11..14, IXSIZE=16 KB) holds the AGGR_RESERVED_I (0), AGGREGATE_I (1, → AIM), BMAP_I (2, → block-allocation map), LOG_I (3), BADBLOCK_I (4) and FILESYSTEM_I (16, → fileset AIM) metadata inodes. The fileset inode table at blocks 29..32 holds FILESET_RSVD_I (0), FILESET_EXT_I (1), ROOT_I (2, dtroot inline), ACL_I (3) and user file inodes (4+). +Writes a minimal IBM Journaled File System (JFS1) aggregate image with a single allocation group, one fileset, and an inline dtree root directory. Byte layout matches the on-disk structures in `linux/fs/jfs` and the `jfsutils` reference (mkfs.jfs / fsck.jfs); validated by exit-zero from `fsck.jfs -n -f -v`. All integer fields are little-endian. `pxd_t` is packed as `len_addr = (len & 0xFFFFFF) | ((addr >> 32) << 24)`, `addr2 = addr & 0xFFFFFFFF`. Dtree slot names are UCS-2 (UTF-16 LE). Round-trips through `JfsReader`. Aggregate inode table (block 11..14, IXSIZE=16 KB) holds the AGGR_RESERVED_I (0), AGGREGATE_I (1, → AIM), BMAP_I (2, → block-allocation map), LOG_I (3), BADBLOCK_I (4) and FILESYSTEM_I (16, → fileset AIM) metadata inodes. The fileset inode table at blocks 29..32 holds FILESET_RSVD_I (0), FILESET_EXT_I (1), ROOT_I (2, dtroot inline), ACL_I (3) and user file inodes (4+). | Member | Signature | Summary | | --- | --- | --- | @@ -7535,7 +7535,7 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati #### `Ocfs2InPlaceModifier` -True in-place R/W modifier for OCFS2 (Oracle Cluster Filesystem 2) images produced by `Ocfs2Writer`. Performs O(touched bytes) random-access I/O against the image: only the global bitmap data block, the root directory dinode (inline dirents), the affected file dinode block, and the file's data blocks are read or written. No whole-image read or rewrite. Layout (matches `Ocfs2Writer`'s single-node geometry): 4 KB blocks = 4 KB clusters; one dinode per block.Superblock dinode at block 2; global bitmap dinode at block 3; bitmap data at block 4 (1 bit per cluster, LSB-first, bit=1 means used).Root directory dinode at block 5 (INODE01) with inline dirents in id2 after the 8-byte ocfs2_inline_data header (id2 + 8), each entry `inode(8) \| rec_len(2) \| name_len(1) \| file_type(1) \| name[]`.User files start at block 8: each gets one dinode block, plus contiguous data clusters whose run is held in a single extent record.Scope (MVP, single-node only): root-directory mutations only. Sub-directory mutation, DLM/heartbeat lockdown, multi-node cluster semantics, and root-directory B-tree splits (extent-backed root) are out of scope and throw `NotSupportedException` if encountered. +True in-place R/W modifier for OCFS2 (Oracle Cluster Filesystem 2) images produced by `Ocfs2Writer`. Performs O(touched bytes) random-access I/O against the image: only the global bitmap data block, the root directory dinode (inline dirents), the affected file dinode block, and the file's data blocks are read or written. No whole-image read or rewrite. Layout (matches `Ocfs2Writer`'s single-node geometry): 4 KB blocks = 4 KB clusters; one dinode per block.Superblock dinode at block 2; global bitmap dinode at block 3; bitmap data at block 4 (1 bit per cluster, LSB-first, bit=1 means used).Root directory dinode at block 5 (INODE01) with inline dirents in id2 after the 8-byte ocfs2_inline_data header (id2 + 8), each entry `inode(8) | rec_len(2) | name_len(1) | file_type(1) | name[]`.User files start at block 8: each gets one dinode block, plus contiguous data clusters whose run is held in a single extent record.Scope (MVP, single-node only): root-directory mutations only. Sub-directory mutation, DLM/heartbeat lockdown, multi-node cluster semantics, and root-directory B-tree splits (extent-backed root) are out of scope and throw `NotSupportedException` if encountered. | Member | Signature | Summary | | --- | --- | --- | @@ -10429,7 +10429,7 @@ Builds a fresh ZX Spectrum `.scl` TR-DOS archive from scratch (WORM). | --- | --- | --- | | `ZxSclWriter` | `ZxSclWriter()` | | | `MaxEntries` | `const int MaxEntries` | TR-DOS hard cap: headers are stored in a single 256-entry directory-like table. | -| `AddFile` | `void AddFile(string name, byte[] data, char fileType = C, ushort param1 = 32768, ushort param2 = 0)` | | +| `AddFile` | `void AddFile(string name, byte[] data, char fileType = 'C', ushort param1 = 32768, ushort param2 = 0)` | | | `Build` | `byte[] Build()` | |