diff --git a/Compression.Core/Dictionary/Nintendo/NintendoLzCodecs.cs b/Compression.Core/Dictionary/Nintendo/NintendoLzCodecs.cs new file mode 100644 index 000000000..72a1c0143 --- /dev/null +++ b/Compression.Core/Dictionary/Nintendo/NintendoLzCodecs.cs @@ -0,0 +1,281 @@ +using System.Buffers.Binary; + +namespace Compression.Core.Dictionary.Nintendo; + +/// Shared match finder and wire codecs for Nintendo Yaz0/Yay0 LZ compression. +internal static class NintendoLzCodecs { + private const int WindowSize = 4096; + private const int MinMatch = 3; + private const int MaxMatch = 273; + private const int HashSize = 1 << 14; + private const int HashMask = HashSize - 1; + private const int MaxChain = 4096; + + internal static byte[] CompressYaz0(ReadOnlySpan source) { + var data = source.ToArray(); + var tokens = Tokenize(data); + using var output = new MemoryStream(); + + Span header = stackalloc byte[16]; + "Yaz0"u8.CopyTo(header); + BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)data.Length)); + output.Write(header); + + for (var index = 0; index < tokens.Count;) { + var control = 0; + using var payload = new MemoryStream(); + for (var bit = 7; bit >= 0 && index < tokens.Count; --bit, ++index) { + var token = tokens[index]; + if (token.IsLiteral) { + control |= 1 << bit; + payload.WriteByte(token.Literal); + continue; + } + WriteYaz0Reference(payload, token.Distance, token.Length); + } + output.WriteByte((byte)control); + payload.WriteTo(output); + } + + return output.ToArray(); + } + + internal static byte[] DecompressYaz0(ReadOnlySpan data) { + if (data.Length < 16 || !data[..4].SequenceEqual("Yaz0"u8)) + throw new InvalidDataException("Not a valid Yaz0 stream."); + + var declared = BinaryPrimitives.ReadUInt32BigEndian(data[4..8]); + if (declared > int.MaxValue) + throw new InvalidDataException("Yaz0 output is too large for an in-memory building block."); + var output = new byte[(int)declared]; + var input = 16; + var written = 0; + + while (written < output.Length) { + if (input >= data.Length) + throw new InvalidDataException("Yaz0 stream ends before the declared output length."); + var control = data[input++]; + for (var bit = 7; bit >= 0 && written < output.Length; --bit) { + if ((control & 1 << bit) != 0) { + if (input >= data.Length) + throw new InvalidDataException("Yaz0 stream ends inside a literal."); + output[written++] = data[input++]; + continue; + } + + if (input + 2 > data.Length) + throw new InvalidDataException("Yaz0 stream ends inside a back-reference."); + var first = data[input++]; + var second = data[input++]; + var distance = ((first & 0x0F) << 8 | second) + 1; + var length = first >> 4; + if (length == 0) { + if (input >= data.Length) + throw new InvalidDataException("Yaz0 stream ends before a long-match length."); + length = data[input++] + 0x12; + } else { + length += 2; + } + CopyReference(output, ref written, distance, length, "Yaz0"); + } + } + + return output; + } + + internal static byte[] CompressYay0(ReadOnlySpan source) { + var data = source.ToArray(); + var tokens = Tokenize(data); + var maskWords = (tokens.Count + 31) / 32; + var masks = new uint[maskWords]; + using var links = new MemoryStream(); + using var chunks = new MemoryStream(); + + Span linkBytes = stackalloc byte[2]; + for (var index = 0; index < tokens.Count; ++index) { + var token = tokens[index]; + if (token.IsLiteral) { + masks[index / 32] |= 1u << (31 - index % 32); + chunks.WriteByte(token.Literal); + continue; + } + + var distance = token.Distance - 1; + var lengthNibble = token.Length <= 17 ? token.Length - 2 : 0; + var link = (ushort)(lengthNibble << 12 | distance); + BinaryPrimitives.WriteUInt16BigEndian(linkBytes, link); + links.Write(linkBytes); + if (lengthNibble == 0) + chunks.WriteByte((byte)(token.Length - 0x12)); + } + + var linkOffset = checked(16 + masks.Length * 4); + var chunkOffset = checked(linkOffset + (int)links.Length); + using var output = new MemoryStream(); + Span header = stackalloc byte[16]; + "Yay0"u8.CopyTo(header); + BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)data.Length)); + BinaryPrimitives.WriteUInt32BigEndian(header[8..], checked((uint)linkOffset)); + BinaryPrimitives.WriteUInt32BigEndian(header[12..], checked((uint)chunkOffset)); + output.Write(header); + + Span maskBytes = stackalloc byte[4]; + foreach (var mask in masks) { + BinaryPrimitives.WriteUInt32BigEndian(maskBytes, mask); + output.Write(maskBytes); + } + links.Position = 0; + links.CopyTo(output); + chunks.Position = 0; + chunks.CopyTo(output); + return output.ToArray(); + } + + internal static byte[] DecompressYay0(ReadOnlySpan data) { + if (data.Length < 16 || !data[..4].SequenceEqual("Yay0"u8)) + throw new InvalidDataException("Not a valid Yay0 stream."); + + var declared = BinaryPrimitives.ReadUInt32BigEndian(data[4..8]); + if (declared > int.MaxValue) + throw new InvalidDataException("Yay0 output is too large for an in-memory building block."); + var linkOffset = checked((int)BinaryPrimitives.ReadUInt32BigEndian(data[8..12])); + var chunkOffset = checked((int)BinaryPrimitives.ReadUInt32BigEndian(data[12..16])); + if (linkOffset < 16 || linkOffset > chunkOffset || chunkOffset > data.Length) + throw new InvalidDataException("Yay0 table offsets are outside the stream."); + + var output = new byte[(int)declared]; + var maskPosition = 16; + var linkPosition = linkOffset; + var chunkPosition = chunkOffset; + var written = 0; + uint mask = 0; + var maskBits = 0; + + while (written < output.Length) { + if (maskBits == 0) { + if (maskPosition + 4 > linkOffset) + throw new InvalidDataException("Yay0 mask table ends before the declared output length."); + mask = BinaryPrimitives.ReadUInt32BigEndian(data.Slice(maskPosition, 4)); + maskPosition += 4; + maskBits = 32; + } + + if ((mask & 0x80000000u) != 0) { + if (chunkPosition >= data.Length) + throw new InvalidDataException("Yay0 literal table ends inside the stream."); + output[written++] = data[chunkPosition++]; + } else { + if (linkPosition + 2 > chunkOffset) + throw new InvalidDataException("Yay0 link table ends inside a back-reference."); + var link = BinaryPrimitives.ReadUInt16BigEndian(data.Slice(linkPosition, 2)); + linkPosition += 2; + var distance = (link & 0x0FFF) + 1; + var length = link >> 12; + if (length == 0) { + if (chunkPosition >= data.Length) + throw new InvalidDataException("Yay0 chunk table ends before a long-match length."); + length = data[chunkPosition++] + 0x12; + } else { + length += 2; + } + CopyReference(output, ref written, distance, length, "Yay0"); + } + + mask <<= 1; + --maskBits; + } + + return output; + } + + private static void WriteYaz0Reference(Stream output, int distance, int length) { + var encodedDistance = distance - 1; + if (length <= 17) { + output.WriteByte((byte)((length - 2) << 4 | encodedDistance >> 8)); + output.WriteByte((byte)encodedDistance); + return; + } + output.WriteByte((byte)(encodedDistance >> 8)); + output.WriteByte((byte)encodedDistance); + output.WriteByte((byte)(length - 0x12)); + } + + private static void CopyReference(byte[] output, ref int written, int distance, int length, string format) { + var source = written - distance; + if (source < 0) + throw new InvalidDataException($"{format} back-reference points before the output buffer."); + if (length > output.Length - written) + throw new InvalidDataException($"{format} back-reference expands past the declared output length."); + for (var i = 0; i < length; ++i) + output[written++] = output[source + i]; + } + + private static List Tokenize(byte[] data) { + var result = new List(data.Length); + if (data.Length == 0) + return result; + + var head = new int[HashSize]; + var previous = new int[data.Length]; + Array.Fill(head, -1); + Array.Fill(previous, -1); + + for (var position = 0; position < data.Length;) { + var (distance, length) = FindMatch(data, position, head, previous); + if (length >= MinMatch) { + result.Add(Token.Reference(distance, length)); + for (var i = 0; i < length; ++i) + UpdateHash(data, position + i, head, previous); + position += length; + } else { + result.Add(Token.LiteralByte(data[position])); + UpdateHash(data, position, head, previous); + ++position; + } + } + return result; + } + + private static (int Distance, int Length) FindMatch(byte[] data, int position, int[] head, int[] previous) { + if (position + MinMatch > data.Length) + return (0, 0); + + var hash = Hash3(data, position); + var minimum = Math.Max(0, position - WindowSize); + var maximumLength = Math.Min(MaxMatch, data.Length - position); + var bestLength = 0; + var bestDistance = 0; + var candidate = head[hash]; + + for (var walked = 0; candidate >= minimum && candidate >= 0 && walked < MaxChain; ++walked) { + var length = 0; + while (length < maximumLength && data[candidate + length] == data[position + length]) + ++length; + if (length > bestLength) { + bestLength = length; + bestDistance = position - candidate; + if (length == maximumLength) + break; + } + candidate = previous[candidate]; + } + + return bestLength >= MinMatch ? (bestDistance, bestLength) : (0, 0); + } + + private static void UpdateHash(byte[] data, int position, int[] head, int[] previous) { + if (position + MinMatch > data.Length) + return; + var hash = Hash3(data, position); + previous[position] = head[hash]; + head[hash] = position; + } + + private static int Hash3(byte[] data, int position) + => ((data[position] << 6) ^ (data[position + 1] << 3) ^ data[position + 2]) & HashMask; + + private readonly record struct Token(bool IsLiteral, byte Literal, int Distance, int Length) { + internal static Token LiteralByte(byte value) => new(true, value, 0, 0); + internal static Token Reference(int distance, int length) => new(false, 0, distance, length); + } +} diff --git a/Compression.Core/Dictionary/Nintendo/Yay0BuildingBlock.cs b/Compression.Core/Dictionary/Nintendo/Yay0BuildingBlock.cs new file mode 100644 index 000000000..d205f9714 --- /dev/null +++ b/Compression.Core/Dictionary/Nintendo/Yay0BuildingBlock.cs @@ -0,0 +1,33 @@ +using Compression.Registry; + +namespace Compression.Core.Dictionary.Nintendo; + +/// +/// Exposes Nintendo Yay0 split-table LZ compression as a benchmarkable building block. +/// +/// +/// Yay0 uses the same 4 KiB LZ reference grammar as Yaz0 but stores 32-bit mask words, +/// 16-bit link records, and literal/long-length bytes in separate tables. Reference: +/// https://www.amnoid.de/gc/yay0.txt +/// +public sealed class Yay0BuildingBlock : IBuildingBlock { + /// + public string Id => "BB_Yay0"; + + /// + public string DisplayName => "Yay0"; + + /// + public string Description => "Nintendo Yay0 split-table LZ compression"; + + /// + public AlgorithmFamily Family => AlgorithmFamily.Dictionary; + + /// + public byte[] Compress(ReadOnlySpan data) + => NintendoLzCodecs.CompressYay0(data); + + /// + public byte[] Decompress(ReadOnlySpan data) + => NintendoLzCodecs.DecompressYay0(data); +} diff --git a/Compression.Core/Dictionary/Nintendo/Yaz0BuildingBlock.cs b/Compression.Core/Dictionary/Nintendo/Yaz0BuildingBlock.cs new file mode 100644 index 000000000..64ca9a3a2 --- /dev/null +++ b/Compression.Core/Dictionary/Nintendo/Yaz0BuildingBlock.cs @@ -0,0 +1,33 @@ +using Compression.Registry; + +namespace Compression.Core.Dictionary.Nintendo; + +/// +/// Exposes Nintendo Yaz0 grouped-flag LZ compression as a benchmarkable building block. +/// +/// +/// Yaz0 uses a 4 KiB sliding window, literals/back-references selected by MSB-first +/// flag bytes, and a 16-byte big-endian header. Reference: +/// https://www.amnoid.de/gc/yaz0.txt +/// +public sealed class Yaz0BuildingBlock : IBuildingBlock { + /// + public string Id => "BB_Yaz0"; + + /// + public string DisplayName => "Yaz0"; + + /// + public string Description => "Nintendo Yaz0 grouped-flag LZ compression"; + + /// + public AlgorithmFamily Family => AlgorithmFamily.Dictionary; + + /// + public byte[] Compress(ReadOnlySpan data) + => NintendoLzCodecs.CompressYaz0(data); + + /// + public byte[] Decompress(ReadOnlySpan data) + => NintendoLzCodecs.DecompressYaz0(data); +} diff --git a/Compression.Core/README.md b/Compression.Core/README.md index 8d55e91c4..ed6fcf8f4 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -2746,6 +2746,42 @@ MS LZH decompressor — reads back the bit stream produced by `MsLzhCompressor`. | `MsLzhDecompressor` | `MsLzhDecompressor()` | | | `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | Decompresses an MS LZH bit stream. | +### Namespace `Compression.Core.Dictionary.Nintendo` + +[`Yay0BuildingBlock`](#yay0buildingblock) · [`Yaz0BuildingBlock`](#yaz0buildingblock) + +#### `Yay0BuildingBlock` + +Exposes Nintendo Yay0 split-table LZ compression as a benchmarkable building block. + +Implements `IBuildingBlock`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Yay0BuildingBlock` | `Yay0BuildingBlock()` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `Compress` | `byte[] Compress(ReadOnlySpan data)` | | +| `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | | + +#### `Yaz0BuildingBlock` + +Exposes Nintendo Yaz0 grouped-flag LZ compression as a benchmarkable building block. + +Implements `IBuildingBlock`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Yaz0BuildingBlock` | `Yaz0BuildingBlock()` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `Compress` | `byte[] Compress(ReadOnlySpan data)` | | +| `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | | + ### Namespace `Compression.Core.Dictionary.Nrv2b` [`Nrv2bBuildingBlock`](#nrv2bbuildingblock) diff --git a/Compression.Tests/BuildingBlocks/NintendoLzBuildingBlockTests.cs b/Compression.Tests/BuildingBlocks/NintendoLzBuildingBlockTests.cs new file mode 100644 index 000000000..2adb3780a --- /dev/null +++ b/Compression.Tests/BuildingBlocks/NintendoLzBuildingBlockTests.cs @@ -0,0 +1,126 @@ +using System.Text; +using Compression.Core.Dictionary.Nintendo; +using Compression.Registry; + +namespace Compression.Tests.BuildingBlocks; + +[TestFixture] +public sealed class NintendoLzBuildingBlockTests { + private static readonly Yaz0BuildingBlock Yaz0 = new(); + private static readonly Yay0BuildingBlock Yay0 = new(); + + [TestCaseSource(nameof(Blocks))] + [Category("HappyPath"), Category("RoundTrip")] + public void Empty_RoundTrips(IBuildingBlock block) { + var compressed = block.Compress([]); + var roundTrip = block.Decompress(compressed); + Assert.That(roundTrip, Is.Empty); + } + + [Test, Category("EdgeCase")] + public void Yaz0_LiteralOnlyVector_IsStable() { + var compressed = Yaz0.Compress("ABC"u8); + Assert.That(compressed, Is.EqualTo(new byte[] { + (byte)'Y', (byte)'a', (byte)'z', (byte)'0', + 0, 0, 0, 3, + 0, 0, 0, 0, 0, 0, 0, 0, + 0xE0, (byte)'A', (byte)'B', (byte)'C', + }).AsCollection); + Assert.That(Yaz0.Decompress(compressed), Is.EqualTo("ABC"u8.ToArray()).AsCollection); + } + + [Test, Category("EdgeCase")] + public void Yay0_LiteralOnlyVector_IsStable() { + var compressed = Yay0.Compress("ABC"u8); + Assert.That(compressed, Is.EqualTo(new byte[] { + (byte)'Y', (byte)'a', (byte)'y', (byte)'0', + 0, 0, 0, 3, + 0, 0, 0, 20, + 0, 0, 0, 20, + 0xE0, 0, 0, 0, + (byte)'A', (byte)'B', (byte)'C', + }).AsCollection); + Assert.That(Yay0.Decompress(compressed), Is.EqualTo("ABC"u8.ToArray()).AsCollection); + } + + [TestCaseSource(nameof(Blocks))] + [Category("HappyPath"), Category("RoundTrip")] + public void RepeatedByte_RoundTripsAndCompresses(IBuildingBlock block) { + var data = new byte[20_000]; + Array.Fill(data, (byte)'A'); + var compressed = block.Compress(data); + var roundTrip = block.Decompress(compressed); + + Assert.Multiple(() => { + Assert.That(compressed.Length, Is.LessThan(data.Length / 10)); + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + }); + } + + [TestCaseSource(nameof(Blocks))] + [Category("HappyPath"), Category("RoundTrip")] + public void RepeatedPhrase_RoundTripsAndCompresses(IBuildingBlock block) { + const string phrase = "the quick brown fox jumps over the lazy dog. "; + var data = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat(phrase, 512))); + var compressed = block.Compress(data); + var roundTrip = block.Decompress(compressed); + + Assert.Multiple(() => { + Assert.That(compressed.Length, Is.LessThan(data.Length * 3 / 4)); + Assert.That(roundTrip, Is.EqualTo(data).AsCollection); + }); + } + + [TestCaseSource(nameof(Blocks))] + [Category("HappyPath"), Category("RoundTrip")] + public void IncompressibleRandom_RoundTrips(IBuildingBlock block) { + var random = new Random(0x59A0); + var data = new byte[8192]; + random.NextBytes(data); + Assert.That(block.Decompress(block.Compress(data)), Is.EqualTo(data).AsCollection); + } + + [TestCaseSource(nameof(Blocks))] + [Category("EdgeCase"), Category("RoundTrip")] + public void AllByteValues_RoundTrip(IBuildingBlock block) { + var data = Enumerable.Range(0, 256).Select(value => (byte)value).ToArray(); + Assert.That(block.Decompress(block.Compress(data)), Is.EqualTo(data).AsCollection); + } + + [Test, Category("EdgeCase")] + public void Yaz0_InvalidBackwardReference_IsRejected() { + byte[] malformed = { + (byte)'Y', (byte)'a', (byte)'z', (byte)'0', + 0, 0, 0, 3, + 0, 0, 0, 0, 0, 0, 0, 0, + 0x00, 0x10, 0x00, + }; + Assert.That(() => Yaz0.Decompress(malformed), Throws.TypeOf()); + } + + [Test, Category("EdgeCase")] + public void Yay0_InvalidOffsets_AreRejected() { + byte[] malformed = { + (byte)'Y', (byte)'a', (byte)'y', (byte)'0', + 0, 0, 0, 1, + 0, 0, 0, 12, + 0, 0, 0, 16, + }; + Assert.That(() => Yay0.Decompress(malformed), Throws.TypeOf()); + } + + [Test, Category("EdgeCase")] + public void Registry_Metadata_IsStable() { + Assert.Multiple(() => { + Assert.That(Yaz0.Id, Is.EqualTo("BB_Yaz0")); + Assert.That(Yaz0.Family, Is.EqualTo(AlgorithmFamily.Dictionary)); + Assert.That(Yay0.Id, Is.EqualTo("BB_Yay0")); + Assert.That(Yay0.Family, Is.EqualTo(AlgorithmFamily.Dictionary)); + }); + } + + private static IEnumerable Blocks() { + yield return Yaz0; + yield return Yay0; + } +} diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 6b7a34cac..656231075 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -3222,7 +3222,7 @@ Implements `IFormatDescriptor`, `IStreamFormatOperations`. #### `CscStream` -CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size \| uint24 csc_blocksize \| uint24 raw_blocksize \| uint32 actual_size +CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size | uint24 csc_blocksize | uint24 raw_blocksize | uint32 actual_size | Member | Signature | Summary | | --- | --- | --- | @@ -4820,7 +4820,7 @@ DEFLATE (RFC 1951) decoder for the PEtite dialect: block type 1 carries the dyna #### `PetiteUnpacker` -Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000\|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments) +Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments) | Member | Signature | Summary | | --- | --- | --- | @@ -9003,8 +9003,8 @@ WORM writer for the NumPy NPY array serialization format (NEP 1). Emits a v1 fil | Member | Signature | Summary | | --- | --- | --- | | `DefaultDtype` | `const string DefaultDtype` | Default dtype string used when no explicit type is supplied. | -| `Write` | `static void Write(Stream output, ReadOnlySpan payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. | -| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. | +| `Write` | `static void Write(Stream output, ReadOnlySpan payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. | +| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. | #### `NpzFormatDescriptor` @@ -11909,8 +11909,8 @@ Reader and writer for the Microsoft SZDD / COMPRESS.EXE file format. SZDD uses a | --- | --- | --- | | `CompressQBasic` | `static byte[] CompressQBasic(ReadOnlySpan data)` | Compresses `data` in the older "SZ " (QBasic) COMPRESS variant and returns the result. The body is the same LZSS stream as SZDD, wrapped in the 12-byte "SZ " header (8-byte magic + little-endian u32 uncompressed length). Round-trips through `Decompress`. | | `CompressQBasic` | `static void CompressQBasic(Stream input, Stream output)` | Stream overload of `CompressQBasic`. | -| `Compress` | `static byte[] Compress(ReadOnlySpan data, char missingChar = _)` | Compresses `data` in SZDD format and returns the result as a new byte array. | -| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = _)` | Compresses `input` in SZDD format and writes the result to `output`. | +| `Compress` | `static byte[] Compress(ReadOnlySpan data, char missingChar = '_')` | Compresses `data` in SZDD format and returns the result as a new byte array. | +| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = '_')` | Compresses `input` in SZDD format and writes the result to `output`. | | `Decompress` | `static byte[] Decompress(ReadOnlySpan data)` | Decompresses an SZDD-encoded byte array and returns the raw data. | | `Decompress` | `static void Decompress(Stream input, Stream output)` | Decompresses an SZDD-encoded stream and writes the raw data to `output`. | | `GetMissingChar` | `static char GetMissingChar(Stream input)` | Returns the "missing character" stored in the SZDD header — the last character of the original filename extension before it was replaced with `'_'`. | diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index af7ac0e76..b2bf80568 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -921,7 +921,7 @@ Reader for Intel HEX records (`:LLAAAATT[DD…]CC`), the long-standing flash-pro #### `SRecordReader` -Reader for Motorola S-Record files (`Stnn[aaaa\|aaaaaa\|aaaaaaaa]dd…cc`). Recognised types: S0 header, S1/S2/S3 data (16/24/32-bit address), S5/S6 record counts (informational), S7/S8/S9 termination (32/24/16-bit start addr). +Reader for Motorola S-Record files (`Stnn[aaaa|aaaaaa|aaaaaaaa]dd…cc`). Recognised types: S0 header, S1/S2/S3 data (16/24/32-bit address), S5/S6 record counts (informational), S7/S8/S9 termination (32/24/16-bit start addr). | Member | Signature | Summary | | --- | --- | --- | @@ -2820,7 +2820,7 @@ Random-access in-place modifier for BBC Micro Acorn DFS `.ssd` images. The DFS c | Member | Signature | Summary | | --- | --- | --- | -| `AddFile` | `static void AddFile(Stream image, string name, byte[] data, char directory = $, uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | Adds a file to an existing single-sided DFS image. Caller is responsible for ensuring the name does not already exist (use `RemoveFile` first for replace-by-name semantics). The file is placed in the lowest contiguous gap large enough to hold it. | +| `AddFile` | `static void AddFile(Stream image, string name, byte[] data, char directory = '$', uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | Adds a file to an existing single-sided DFS image. Caller is responsible for ensuring the name does not already exist (use `RemoveFile` first for replace-by-name semantics). The file is placed in the lowest contiguous gap large enough to hold it. | | `RemoveFile` | `static bool RemoveFile(Stream image, string name, bool wipeData = true)` | Removes a named file from the image. Returns true if found and removed. When `wipeData` is true, the data sectors are zeroed. | #### `BbcReader` @@ -2855,7 +2855,7 @@ Builds a fresh BBC Micro Acorn DFS `.ssd` single-sided disk image from scratch ( | `SectorSize` | `const int SectorSize` | | | `SectorsPerTrack` | `const int SectorsPerTrack` | | | `TotalSectors40` | `const int TotalSectors40` | | -| `AddFile` | `void AddFile(string name, byte[] data, char directory = $, uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | | +| `AddFile` | `void AddFile(string name, byte[] data, char directory = '$', uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | | | `Build` | `byte[] Build(string diskTitle = "WORMDISK", int bootOption = 0)` | Builds the complete 40-track SSD image (100 000 bytes). | ### Namespace `FileSystem.BcacheFs` @@ -3232,7 +3232,7 @@ From-scratch writer for the Commodore nibble container the `CbmNibbleReader` con | `AddFile` | `void AddFile(string name, byte[] data)` | Adds a file to the flat directory. Commodore names are PETSCII and at most 16 characters; longer names are truncated. The default file type is PRG. | | `Build` | `byte[] Build()` | Builds the G64 GCR nibble image holding all added files. | | `DecodeToD64` | `static byte[] DecodeToD64(NibbleImage image)` | Reconstructs a standard 174 848-byte D64 image from the GCR tracks of a nibble image previously parsed by `CbmNibbleReader`. Each track is rescanned for sync marks and its header/data blocks GCR-decoded back into the correct sector slots. | -| `SetDisk` | `void SetDisk(string name, char id1 = 0, char id2 = 0)` | Sets the on-disk volume name (PETSCII, ≤16 chars) and the 2-byte disk id. | +| `SetDisk` | `void SetDisk(string name, char id1 = '0', char id2 = '0')` | Sets the on-disk volume name (PETSCII, ≤16 chars) and the 2-byte disk id. | | `WriteTo` | `void WriteTo(Stream output)` | Writes the G64 image to `output`. | #### `G64FormatDescriptor` @@ -6152,7 +6152,7 @@ Implements `IDisposable`. #### `JfsWriter` -Writes a minimal IBM Journaled File System (JFS1) aggregate image with a single allocation group, one fileset, and an inline dtree root directory. Byte layout matches the on-disk structures in `linux/fs/jfs` and the `jfsutils` reference (mkfs.jfs / fsck.jfs); validated by exit-zero from `fsck.jfs -n -f -v`. All integer fields are little-endian. `pxd_t` is packed as `len_addr = (len & 0xFFFFFF) \| ((addr >> 32) << 24)`, `addr2 = addr & 0xFFFFFFFF`. Dtree slot names are UCS-2 (UTF-16 LE). Round-trips through `JfsReader`. Aggregate inode table (block 11..14, IXSIZE=16 KB) holds the AGGR_RESERVED_I (0), AGGREGATE_I (1, → AIM), BMAP_I (2, → block-allocation map), LOG_I (3), BADBLOCK_I (4) and FILESYSTEM_I (16, → fileset AIM) metadata inodes. The fileset inode table at blocks 29..32 holds FILESET_RSVD_I (0), FILESET_EXT_I (1), ROOT_I (2, dtroot inline), ACL_I (3) and user file inodes (4+). +Writes a minimal IBM Journaled File System (JFS1) aggregate image with a single allocation group, one fileset, and an inline dtree root directory. Byte layout matches the on-disk structures in `linux/fs/jfs` and the `jfsutils` reference (mkfs.jfs / fsck.jfs); validated by exit-zero from `fsck.jfs -n -f -v`. All integer fields are little-endian. `pxd_t` is packed as `len_addr = (len & 0xFFFFFF) | ((addr >> 32) << 24)`, `addr2 = addr & 0xFFFFFFFF`. Dtree slot names are UCS-2 (UTF-16 LE). Round-trips through `JfsReader`. Aggregate inode table (block 11..14, IXSIZE=16 KB) holds the AGGR_RESERVED_I (0), AGGREGATE_I (1, → AIM), BMAP_I (2, → block-allocation map), LOG_I (3), BADBLOCK_I (4) and FILESYSTEM_I (16, → fileset AIM) metadata inodes. The fileset inode table at blocks 29..32 holds FILESET_RSVD_I (0), FILESET_EXT_I (1), ROOT_I (2, dtroot inline), ACL_I (3) and user file inodes (4+). | Member | Signature | Summary | | --- | --- | --- | @@ -7535,7 +7535,7 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati #### `Ocfs2InPlaceModifier` -True in-place R/W modifier for OCFS2 (Oracle Cluster Filesystem 2) images produced by `Ocfs2Writer`. Performs O(touched bytes) random-access I/O against the image: only the global bitmap data block, the root directory dinode (inline dirents), the affected file dinode block, and the file's data blocks are read or written. No whole-image read or rewrite. Layout (matches `Ocfs2Writer`'s single-node geometry): 4 KB blocks = 4 KB clusters; one dinode per block.Superblock dinode at block 2; global bitmap dinode at block 3; bitmap data at block 4 (1 bit per cluster, LSB-first, bit=1 means used).Root directory dinode at block 5 (INODE01) with inline dirents in id2 after the 8-byte ocfs2_inline_data header (id2 + 8), each entry `inode(8) \| rec_len(2) \| name_len(1) \| file_type(1) \| name[]`.User files start at block 8: each gets one dinode block, plus contiguous data clusters whose run is held in a single extent record.Scope (MVP, single-node only): root-directory mutations only. Sub-directory mutation, DLM/heartbeat lockdown, multi-node cluster semantics, and root-directory B-tree splits (extent-backed root) are out of scope and throw `NotSupportedException` if encountered. +True in-place R/W modifier for OCFS2 (Oracle Cluster Filesystem 2) images produced by `Ocfs2Writer`. Performs O(touched bytes) random-access I/O against the image: only the global bitmap data block, the root directory dinode (inline dirents), the affected file dinode block, and the file's data blocks are read or written. No whole-image read or rewrite. Layout (matches `Ocfs2Writer`'s single-node geometry): 4 KB blocks = 4 KB clusters; one dinode per block.Superblock dinode at block 2; global bitmap dinode at block 3; bitmap data at block 4 (1 bit per cluster, LSB-first, bit=1 means used).Root directory dinode at block 5 (INODE01) with inline dirents in id2 after the 8-byte ocfs2_inline_data header (id2 + 8), each entry `inode(8) | rec_len(2) | name_len(1) | file_type(1) | name[]`.User files start at block 8: each gets one dinode block, plus contiguous data clusters whose run is held in a single extent record.Scope (MVP, single-node only): root-directory mutations only. Sub-directory mutation, DLM/heartbeat lockdown, multi-node cluster semantics, and root-directory B-tree splits (extent-backed root) are out of scope and throw `NotSupportedException` if encountered. | Member | Signature | Summary | | --- | --- | --- | @@ -10429,7 +10429,7 @@ Builds a fresh ZX Spectrum `.scl` TR-DOS archive from scratch (WORM). | --- | --- | --- | | `ZxSclWriter` | `ZxSclWriter()` | | | `MaxEntries` | `const int MaxEntries` | TR-DOS hard cap: headers are stored in a single 256-entry directory-like table. | -| `AddFile` | `void AddFile(string name, byte[] data, char fileType = C, ushort param1 = 32768, ushort param2 = 0)` | | +| `AddFile` | `void AddFile(string name, byte[] data, char fileType = 'C', ushort param1 = 32768, ushort param2 = 0)` | | | `Build` | `byte[] Build()` | |