diff --git a/Compression.Core/Dictionary/Lzw/LzcCodec.cs b/Compression.Core/Dictionary/Lzw/LzcCodec.cs new file mode 100644 index 000000000..58be036ae --- /dev/null +++ b/Compression.Core/Dictionary/Lzw/LzcCodec.cs @@ -0,0 +1,344 @@ +using System.Buffers.Binary; +using Compression.Registry; + +namespace Compression.Core.Dictionary.Lzw; + +/// +/// UNIX compress / LZC codec for native .Z streams. +/// +/// +/// LZC uses LSB-first variable-width LZW codes, but unlike a continuous bit stream it packs +/// codes in groups of eight. A code-width change or block CLEAR flushes the entire current +/// group before coding resumes at the new width. The stream carries the traditional +/// 1F 9D header and has no embedded uncompressed-length or checksum field. +/// +public static class LzcCodec { + private const byte Magic1 = 0x1F; + private const byte Magic2 = 0x9D; + private const byte BlockModeFlag = 0x80; + private const byte ReservedFlagsMask = 0x60; + private const byte MaxBitsMask = 0x1F; + private const int InitialBits = 9; + private const int ClearCode = 256; + + /// Compresses data to a native UNIX compress (.Z) stream. + /// Uncompressed bytes. + /// Maximum LZW code width, from 9 through 16. + /// Whether code 256 is reserved as the block CLEAR code. + /// + /// The encoder emits a standards-compatible stream without heuristic dictionary clears. + /// When the dictionary fills it remains fixed at the maximum width, which is valid for both + /// block and non-block streams. The decoder accepts CLEAR codes produced by adaptive encoders. + /// + public static byte[] Compress(ReadOnlySpan data, int maxBits = 16, bool blockMode = true) { + ValidateMaxBits(maxBits); + + using var output = new MemoryStream(); + output.WriteByte(Magic1); + output.WriteByte(Magic2); + output.WriteByte((byte)(maxBits | (blockMode ? BlockModeFlag : 0))); + + if (data.IsEmpty) + return output.ToArray(); + + var firstFreeCode = blockMode ? ClearCode + 1 : ClearCode; + var maxCodeCount = 1 << maxBits; + var dictionary = new Dictionary<(int Prefix, byte Suffix), int>(); + var nextEncoderCode = firstFreeCode; + + var writer = new CodeWriter(output, InitialBits); + var decoderNextCode = firstFreeCode; + var width = InitialBits; + var widthMaxCode = (1 << width) - 1; + var hasPrevious = false; + + void Emit(int code) { + writer.Write(code); + + // The encoder learns a phrase when it emits its previous phrase, while the decoder + // can only learn it after seeing the following code. Track decoder state separately + // so width transitions occur on the same code boundary at both ends. + if (hasPrevious && decoderNextCode < maxCodeCount) + ++decoderNextCode; + hasPrevious = true; + + if (width >= maxBits || decoderNextCode <= widthMaxCode) + return; + + ++width; + widthMaxCode = (1 << width) - 1; + writer.Align(width); + } + + int currentCode = data[0]; + foreach (var nextByte in data[1..]) { + var key = (currentCode, nextByte); + if (dictionary.TryGetValue(key, out var existingCode)) { + currentCode = existingCode; + continue; + } + + Emit(currentCode); + if (nextEncoderCode < maxCodeCount) + dictionary[key] = nextEncoderCode++; + currentCode = nextByte; + } + + Emit(currentCode); + writer.Finish(); + return output.ToArray(); + } + + /// Decompresses a complete native UNIX compress (.Z) stream. + public static byte[] Decompress(ReadOnlySpan data) + => DecompressCore(data, null, null); + + /// Decompresses a native .Z stream and requires an exact expanded length. + public static byte[] Decompress(ReadOnlySpan data, int expandedLength) { + ArgumentOutOfRangeException.ThrowIfNegative(expandedLength); + return DecompressCore(data, expandedLength, null); + } + + /// + /// Decompresses a native .Z stream and requires both an exact expanded length and + /// the expected maximum code width from an enclosing format. + /// + public static byte[] Decompress(ReadOnlySpan data, int expandedLength, int expectedMaxBits) { + ArgumentOutOfRangeException.ThrowIfNegative(expandedLength); + ValidateMaxBits(expectedMaxBits); + return DecompressCore(data, expandedLength, expectedMaxBits); + } + + private static byte[] DecompressCore(ReadOnlySpan data, int? expandedLength, int? expectedMaxBits) { + if (data.Length < 3) + throw new InvalidDataException("LZC stream is shorter than its three-byte header."); + if (data[0] != Magic1 || data[1] != Magic2) + throw new InvalidDataException("LZC stream has invalid 1F 9D magic bytes."); + + var flags = data[2]; + if ((flags & ReservedFlagsMask) != 0) + throw new InvalidDataException($"LZC stream uses reserved header flags 0x{flags & ReservedFlagsMask:X2}."); + + var maxBits = flags & MaxBitsMask; + if (maxBits is < InitialBits or > 16) + throw new InvalidDataException($"LZC stream declares invalid maximum code width {maxBits}."); + if (expectedMaxBits is { } expected && maxBits != expected) + throw new InvalidDataException($"LZC stream declares {maxBits}-bit codes, but the enclosing format requires {expected}-bit LZC."); + + var blockMode = (flags & BlockModeFlag) != 0; + var firstFreeCode = blockMode ? ClearCode + 1 : ClearCode; + var maxCodeCount = 1 << maxBits; + var prefix = new int[maxCodeCount]; + var suffix = new byte[maxCodeCount]; + var reverse = new byte[maxCodeCount]; + using var output = expandedLength is { } length ? new MemoryStream(length) : new MemoryStream(); + + var reader = new CodeReader(data[3..], InitialBits); + var nextCode = firstFreeCode; + var width = InitialBits; + var widthMaxCode = (1 << width) - 1; + var previousCode = -1; + + while (true) { + if (width < maxBits && nextCode > widthMaxCode) { + ++width; + widthMaxCode = (1 << width) - 1; + reader.Align(width); + } + + if (!reader.TryRead(out var code)) + break; + + if (blockMode && code == ClearCode) { + nextCode = firstFreeCode; + width = InitialBits; + widthMaxCode = (1 << width) - 1; + previousCode = -1; + reader.Align(width); + continue; + } + + if (previousCode < 0) { + if ((uint)code > byte.MaxValue) + throw new InvalidDataException($"LZC stream starts a dictionary block with non-literal code {code}."); + EnsureOutputFits(output, expandedLength, 1); + output.WriteByte((byte)code); + previousCode = code; + continue; + } + + var isKwKwK = code == nextCode && nextCode < maxCodeCount; + if (code > nextCode || (code == nextCode && !isKwKwK)) + throw new InvalidDataException($"LZC stream references future dictionary code {code} (next {nextCode})."); + + var phraseCode = isKwKwK ? previousCode : code; + var phraseLength = DecodePhrase(phraseCode, nextCode, prefix, suffix, reverse, out var firstByte); + var writeLength = checked(phraseLength + (isKwKwK ? 1 : 0)); + EnsureOutputFits(output, expandedLength, writeLength); + for (var index = phraseLength - 1; index >= 0; --index) + output.WriteByte(reverse[index]); + if (isKwKwK) + output.WriteByte(firstByte); + + if (nextCode < maxCodeCount) { + prefix[nextCode] = previousCode; + suffix[nextCode] = firstByte; + ++nextCode; + } + + previousCode = code; + } + + if (expandedLength is { } required && output.Length != required) + throw new InvalidDataException($"LZC stream expanded to {output.Length} bytes instead of the required {required}."); + return output.ToArray(); + } + + private static int DecodePhrase(int code, int nextCode, int[] prefix, byte[] suffix, + Span reverse, out byte firstByte) { + var count = 0; + var current = code; + while (current >= 256) { + if (current >= nextCode) + throw new InvalidDataException($"LZC dictionary chain references undefined code {current}."); + if (count >= reverse.Length) + throw new InvalidDataException("LZC dictionary phrase exceeds the maximum representable length."); + reverse[count++] = suffix[current]; + current = prefix[current]; + } + + if ((uint)current > byte.MaxValue) + throw new InvalidDataException($"LZC dictionary chain terminates at invalid literal {current}."); + if (count >= reverse.Length) + throw new InvalidDataException("LZC dictionary phrase exceeds the maximum representable length."); + + firstByte = (byte)current; + reverse[count++] = firstByte; + return count; + } + + private static void EnsureOutputFits(MemoryStream output, int? expandedLength, int additionalBytes) { + if (expandedLength is { } limit && output.Length + additionalBytes > limit) + throw new InvalidDataException($"LZC stream expands beyond the required {limit} bytes."); + } + + private static void ValidateMaxBits(int maxBits) { + if (maxBits is < InitialBits or > 16) + throw new ArgumentOutOfRangeException(nameof(maxBits), maxBits, "LZC maximum code width must be in the range 9..16."); + } + + private sealed class CodeWriter(Stream output, int width) { + private readonly Stream _output = output; + private int _width = width; + private UInt128 _bits; + private int _codeCount; + + public void Write(int code) { + this._bits |= (UInt128)(uint)code << (this._codeCount * this._width); + if (++this._codeCount == 8) + this.Flush(fullGroup: true); + } + + public void Align(int newWidth) { + if (this._codeCount != 0) + this.Flush(fullGroup: true); + this._width = newWidth; + } + + public void Finish() { + if (this._codeCount != 0) + this.Flush(fullGroup: false); + } + + private void Flush(bool fullGroup) { + var byteCount = fullGroup ? this._width : (this._codeCount * this._width + 7) >> 3; + for (var index = 0; index < byteCount; ++index) + this._output.WriteByte((byte)(this._bits >> (index * 8))); + this._bits = 0; + this._codeCount = 0; + } + } + + private ref struct CodeReader { + private readonly ReadOnlySpan _source; + private int _sourceOffset; + private int _width; + private UInt128 _bits; + private int _codesRemaining; + + public CodeReader(ReadOnlySpan source, int width) { + this._source = source; + this._sourceOffset = 0; + this._width = width; + this._bits = 0; + this._codesRemaining = 0; + } + + public void Align(int newWidth) { + this._bits = 0; + this._codesRemaining = 0; + this._width = newWidth; + } + + public bool TryRead(out int code) { + if (this._codesRemaining == 0) { + if (this._sourceOffset >= this._source.Length) { + code = 0; + return false; + } + + var byteCount = Math.Min(this._width, this._source.Length - this._sourceOffset); + var codeCount = byteCount * 8 / this._width; + if (codeCount == 0) + throw new InvalidDataException("LZC stream ends with a truncated code."); + + this._bits = 0; + for (var index = 0; index < byteCount; ++index) + this._bits |= (UInt128)this._source[this._sourceOffset + index] << (index * 8); + this._sourceOffset += byteCount; + this._codesRemaining = codeCount; + } + + var mask = ((UInt128)1 << this._width) - 1; + code = (int)(this._bits & mask); + this._bits >>= this._width; + --this._codesRemaining; + return true; + } + } +} + +/// Benchmarkable UNIX compress / LZC building block. +/// +/// Native .Z streams do not carry their expanded length, so this building-block envelope +/// prefixes a little-endian 32-bit length before a native 16-bit block-mode stream. +/// +public sealed class LzcBuildingBlock : IBuildingBlock { + /// + public string Id => "BB_Lzc"; + /// + public string DisplayName => "LZC (UNIX compress)"; + /// + public string Description => "UNIX compress LZW with 9-16 bit codes and eight-code width-transition packing"; + /// + public AlgorithmFamily Family => AlgorithmFamily.Dictionary; + + /// + public byte[] Compress(ReadOnlySpan data) { + var native = LzcCodec.Compress(data); + var result = new byte[checked(4 + native.Length)]; + BinaryPrimitives.WriteInt32LittleEndian(result, data.Length); + native.CopyTo(result.AsSpan(4)); + return result; + } + + /// + public byte[] Decompress(ReadOnlySpan data) { + if (data.Length < 4) + throw new InvalidDataException("LZC building-block envelope is truncated."); + var length = BinaryPrimitives.ReadInt32LittleEndian(data); + if (length < 0) + throw new InvalidDataException("LZC building-block envelope has a negative expanded length."); + return LzcCodec.Decompress(data[4..], length, 16); + } +} diff --git a/Compression.Core/README.md b/Compression.Core/README.md index c1659410a..4ec70f61b 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -204,7 +204,7 @@ Use the concrete version you intend to consume; this document does not predict a -Every public and protected member of all 540 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). +Every public and protected member of all 542 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 2b5ee7bb1..7a0d05a30 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -2100,7 +2100,34 @@ Decompresses data produced by `LzvnCompressor`. ### Namespace `Compression.Core.Dictionary.Lzw` -[`LzwBuildingBlock`](#lzwbuildingblock) · [`LzwCompressionLevel`](#lzwcompressionlevel) · [`LzwDecoder`](#lzwdecoder) · [`LzwEncoder`](#lzwencoder) · [`NuLzwBuildingBlock`](#nulzwbuildingblock) · [`NuLzwCodec`](#nulzwcodec) · [`NuLzwVariant`](#nulzwvariant) +[`LzcBuildingBlock`](#lzcbuildingblock) · [`LzcCodec`](#lzccodec) · [`LzwBuildingBlock`](#lzwbuildingblock) · [`LzwCompressionLevel`](#lzwcompressionlevel) · [`LzwDecoder`](#lzwdecoder) · [`LzwEncoder`](#lzwencoder) · [`NuLzwBuildingBlock`](#nulzwbuildingblock) · [`NuLzwCodec`](#nulzwcodec) · [`NuLzwVariant`](#nulzwvariant) + +#### `LzcBuildingBlock` + +Benchmarkable UNIX `compress` / LZC building block. + +Implements `IBuildingBlock`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `LzcBuildingBlock` | `LzcBuildingBlock()` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `Compress` | `byte[] Compress(ReadOnlySpan data)` | | +| `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | | + +#### `LzcCodec` + +UNIX `compress` / LZC codec for native `.Z` streams. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Compress` | `static byte[] Compress(ReadOnlySpan data, int maxBits = 16, bool blockMode = true)` | Compresses data to a native UNIX `compress` (`.Z`) stream. | +| `Decompress` | `static byte[] Decompress(ReadOnlySpan data)` | Decompresses a complete native UNIX `compress` (`.Z`) stream. | +| `Decompress` | `static byte[] Decompress(ReadOnlySpan data, int expandedLength)` | Decompresses a native `.Z` stream and requires an exact expanded length. | +| `Decompress` | `static byte[] Decompress(ReadOnlySpan data, int expandedLength, int expectedMaxBits)` | Decompresses a native `.Z` stream and requires both an exact expanded length and the expected maximum code width from an enclosing format. | #### `LzwBuildingBlock` diff --git a/Compression.Tests/BuildingBlocks/LzcTests.cs b/Compression.Tests/BuildingBlocks/LzcTests.cs new file mode 100644 index 000000000..b278a8c2e --- /dev/null +++ b/Compression.Tests/BuildingBlocks/LzcTests.cs @@ -0,0 +1,104 @@ +using System.Buffers.Binary; +using Compression.Core.Dictionary.Lzw; + +namespace Compression.Tests.BuildingBlocks; + +[TestFixture] +public sealed class LzcTests { + private static readonly byte[] Tobe = "TOBEORNOTTOBEORTOBEORNOT"u8.ToArray(); + + [TestCase(12, "1F9D8C549E0829F2448A932754020E2CA890A04184")] + [TestCase(16, "1F9D90549E0829F2448A932754020E2CA890A04184")] + public void GzipValidatedReferenceVector_MatchesNativeCompressBytes(int maxBits, string expectedHex) { + var packed = LzcCodec.Compress(Tobe, maxBits); + + Assert.That(packed, Is.EqualTo(Convert.FromHexString(expectedHex))); + Assert.That(LzcCodec.Decompress(packed, Tobe.Length, maxBits), Is.EqualTo(Tobe)); + } + + [Test] + public void Decoder_ClearCodeDiscardsRemainderOfEightCodeGroup() { + // Codes A, B, CLEAR fill only part of a 9-byte/8-code group. The remaining bytes are + // alignment padding; C and D start a fresh 9-bit group. GNU gzip accepts this .Z vector. + var packed = Convert.FromHexString("1F9D8C418400040000000000438800"); + + Assert.That(LzcCodec.Decompress(packed), Is.EqualTo("ABCD"u8.ToArray())); + } + + [TestCase(12)] + [TestCase(16)] + public void RoundTrip_CrossesCodeWidthsAndDictionaryLimit(int maxBits) { + var data = new byte[100_000]; + var state = 0x12345678u; + for (var index = 0; index < data.Length; ++index) { + state = state * 1664525u + 1013904223u; + data[index] = (byte)(state >> 24); + } + + var packed = LzcCodec.Compress(data, maxBits); + + Assert.That(LzcCodec.Decompress(packed, data.Length, maxBits), Is.EqualTo(data)); + } + + [TestCase(12)] + [TestCase(16)] + public void RoundTrip_NonBlockModeCrossesCodeWidths(int maxBits) { + var data = Enumerable.Range(0, 12_000).Select(i => (byte)((i * 73 + i / 17) & 0xFF)).ToArray(); + + var packed = LzcCodec.Compress(data, maxBits, blockMode: false); + + Assert.That(packed[2] & 0x80, Is.Zero); + Assert.That(LzcCodec.Decompress(packed, data.Length, maxBits), Is.EqualTo(data)); + } + + [Test] + public void EmptyStream_HasNativeHeaderAndRoundTrips() { + var packed = LzcCodec.Compress([]); + + Assert.That(packed, Is.EqualTo(new byte[] { 0x1F, 0x9D, 0x90 })); + Assert.That(LzcCodec.Decompress(packed), Is.Empty); + } + + [Test] + public void FutureDictionaryCodeIsRejected() { + var packed = Convert.FromHexString("1F9D8C415802"); // literal A, then undefined code 300 + + Assert.Throws(() => LzcCodec.Decompress(packed)); + } + + [Test] + public void TruncatedFinalCodeIsRejected() { + var packed = LzcCodec.Compress(Tobe, 12); + Array.Resize(ref packed, packed.Length - 1); + + Assert.Throws(() => LzcCodec.Decompress(packed, Tobe.Length, 12)); + } + + [Test] + public void HeaderReservedBitsAreRejected() { + var packed = LzcCodec.Compress(Tobe, 12); + packed[2] |= 0x20; + + Assert.Throws(() => LzcCodec.Decompress(packed)); + } + + [Test] + public void EnclosingFormatMaxBitsMismatchIsRejected() { + var packed = LzcCodec.Compress(Tobe, 16); + + Assert.Throws(() => LzcCodec.Decompress(packed, Tobe.Length, 12)); + } + + [Test] + public void BuildingBlock_EnvelopeCarriesExpandedLength() { + var block = new LzcBuildingBlock(); + var data = Enumerable.Range(0, 20_000).Select(i => (byte)((i * 29) & 0x7F)).ToArray(); + + var packed = block.Compress(data); + + Assert.That(block.Id, Is.EqualTo("BB_Lzc")); + Assert.That(BinaryPrimitives.ReadInt32LittleEndian(packed), Is.EqualTo(data.Length)); + Assert.That(packed.AsSpan(4, 3).ToArray(), Is.EqualTo(new byte[] { 0x1F, 0x9D, 0x90 })); + Assert.That(block.Decompress(packed), Is.EqualTo(data)); + } +} diff --git a/Compression.Tests/Compress/CompressStreamTests.cs b/Compression.Tests/Compress/CompressStreamTests.cs index 8a5a30aab..0f67241e6 100644 --- a/Compression.Tests/Compress/CompressStreamTests.cs +++ b/Compression.Tests/Compress/CompressStreamTests.cs @@ -83,6 +83,14 @@ public void Header_FlagsContainMaxBitsAndBlockMode() { Assert.That(flags & 0x80, Is.EqualTo(0x80)); // block mode flag } + [Category("ThemVsUs")] + [Test] + public void Decompress_GzipValidatedClearAlignmentVector() { + var compressed = Convert.FromHexString("1F9D8C418400040000000000438800"); + + Assert.That(DecompressData(compressed), Is.EqualTo("ABCD"u8.ToArray())); + } + [Category("HappyPath")] [Category("RoundTrip")] [Test] @@ -93,6 +101,22 @@ public void RoundTrip_NoBlockMode() { Assert.That(result, Is.EqualTo(data)); } + [Category("HappyPath")] + [Category("RoundTrip")] + [Test] + public void RoundTrip_CrossesMultipleCodeWidths() { + var data = new byte[20_000]; + var state = 0xCAFEBABEu; + for (var index = 0; index < data.Length; ++index) { + state = state * 1103515245u + 12345u; + data[index] = (byte)(state >> 24); + } + + var compressed = CompressData(data, maxBits: 12); + + Assert.That(DecompressData(compressed), Is.EqualTo(data)); + } + [Category("HappyPath")] [Test] public void RepetitiveData_CompressesWell() { diff --git a/Compression.Tests/NuFx/NuFxTests.cs b/Compression.Tests/NuFx/NuFxTests.cs index 95f3ca2b1..8095575b0 100644 --- a/Compression.Tests/NuFx/NuFxTests.cs +++ b/Compression.Tests/NuFx/NuFxTests.cs @@ -18,7 +18,7 @@ public void Descriptor_AdvertisesTrueReadWriteAndSupportedMethods() { Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); Assert.That(descriptor.Methods.Select(m => m.Name), - Is.EquivalentTo(new[] { "stored", "squeeze", "nulzw1", "nulzw2", "auto" })); + Is.EquivalentTo(new[] { "stored", "squeeze", "nulzw1", "nulzw2", "lzc12", "lzc16", "auto" })); Assert.That(descriptor.Extensions, Does.Contain(".shk")); Assert.That(descriptor.Extensions, Does.Contain(".sdk")); } @@ -27,6 +27,8 @@ public void Descriptor_AdvertisesTrueReadWriteAndSupportedMethods() { [TestCase("squeeze")] [TestCase("nulzw1")] [TestCase("nulzw2")] + [TestCase("lzc12")] + [TestCase("lzc16")] [TestCase("auto")] public void Create_RoundTripsEveryWritableCompressionMethod(string method) { var descriptor = new NuFxFormatDescriptor(); @@ -51,6 +53,45 @@ public void Create_RoundTripsEveryWritableCompressionMethod(string method) { Assert.That(integrity.ValidEntries, Is.EqualTo(2)); } + [TestCase("lzc12", 4, 0x8C)] + [TestCase("lzc16", 5, 0x90)] + public void Create_LzcThreadCarriesNativeCompressHeader(string method, int expectedFormat, byte expectedFlags) { + var descriptor = new NuFxFormatDescriptor(); + using var archive = new MemoryStream(); + descriptor.Create(archive, [ArchiveInputInfo.InMemory("TOBE", "TOBEORNOTTOBEORTOBEORNOT"u8)], + new FormatCreateOptions { MethodName = method }); + + var bytes = archive.ToArray(); + const int recordStart = 48; + var attribCount = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(recordStart + 6, 2)); + var deprecatedNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(recordStart + attribCount - 2, 2)); + var threadHeadersStart = checked(recordStart + attribCount + deprecatedNameLength); + var firstThread = bytes.AsSpan(threadHeadersStart, 16); + var secondThread = bytes.AsSpan(threadHeadersStart + 16, 16); + var filenameStorageLength = BinaryPrimitives.ReadUInt32LittleEndian(firstThread.Slice(12, 4)); + var dataStart = checked(threadHeadersStart + 32 + (int)filenameStorageLength); + + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(secondThread.Slice(2, 2)), Is.EqualTo((ushort)expectedFormat)); + Assert.That(bytes.AsSpan(dataStart, 3).ToArray(), Is.EqualTo(new byte[] { 0x1F, 0x9D, expectedFlags })); + } + + [TestCase("lzc12", "LZC-12")] + [TestCase("lzc16", "LZC-16")] + public void DirectReplace_PreservesLzcCompressionMethod(string method, string listedMethod) { + var descriptor = new NuFxFormatDescriptor(); + using var archive = new MemoryStream(); + descriptor.Create(archive, [ArchiveInputInfo.InMemory("ONE", SampleB)], + new FormatCreateOptions { MethodName = method }); + + var replacement = Enumerable.Range(0, 15_000).Select(i => (byte)((i * 19 + i / 7) & 0xFF)).ToArray(); + descriptor.Add(archive, [ArchiveInputInfo.InMemory("ONE", replacement)]); + + archive.Position = 0; + Assert.That(descriptor.List(archive, null).Single().Method, Is.EqualTo(listedMethod)); + archive.Position = 0; + Assert.That(descriptor.ExtractEntryToMemory(archive, "ONE", null), Is.EqualTo(replacement)); + } + [Test] public void Create_DiskImageModeProducesSdkStyleDiskThread() { var descriptor = new NuFxFormatDescriptor(); diff --git a/FileFormats/FileFormat.Compress/CompressStream.cs b/FileFormats/FileFormat.Compress/CompressStream.cs index a6e93771b..3555503fb 100644 --- a/FileFormats/FileFormat.Compress/CompressStream.cs +++ b/FileFormats/FileFormat.Compress/CompressStream.cs @@ -1,4 +1,3 @@ -using Compression.Core.BitIO; using Compression.Core.Dictionary.Lzw; using Compression.Core.Streams; @@ -6,7 +5,8 @@ namespace FileFormat.Compress; /// /// Stream for reading and writing Unix compress (.Z) format data. -/// Uses LZW compression with variable-width codes (9-16 bits) in LSB-first order. +/// Uses LZC/LZW compression with variable-width codes (9-16 bits) and the format's +/// required eight-code packing/alignment rules. /// public sealed class CompressStream : CompressionStream { private readonly int _maxBits; @@ -27,7 +27,7 @@ public sealed class CompressStream : CompressionStream { /// The underlying stream. /// Whether to compress or decompress. /// Maximum LZW code width (9-16). Defaults to 16. - /// Whether to use block mode (clear codes). Defaults to true. + /// Whether to reserve the block CLEAR code. Defaults to true. /// Whether to leave the inner stream open. public CompressStream(Stream stream, CompressionStreamMode mode, int maxBits = CompressConstants.DefaultMaxBits, @@ -72,57 +72,14 @@ protected override void CompressBlock(byte[] buffer, int offset, int count) { /// protected override void FinishCompression() { - var data = this._compressBuffer!.ToArray(); - - // Write header - InnerStream.WriteByte(CompressConstants.Magic1); - InnerStream.WriteByte(CompressConstants.Magic2); - - var flags = (byte)(this._maxBits & CompressConstants.MaxBitsMask); - if (this._blockMode) - flags |= CompressConstants.BlockModeFlag; - InnerStream.WriteByte(flags); - - // LZW compress - // Unix compress uses: minBits=9, clear code at 256 (when blockMode), no stop code, LSB bit order - var encoder = new LzwEncoder( - InnerStream, - minBits: CompressConstants.MinBits, - maxBits: this._maxBits, - useClearCode: this._blockMode, - useStopCode: false, - bitOrder: BitOrder.LsbFirst); - encoder.Encode(data); + var compressed = LzcCodec.Compress(this._compressBuffer!.ToArray(), this._maxBits, this._blockMode); + InnerStream.Write(compressed); } private void ReadAndDecompress() { - // Read header - var b1 = InnerStream.ReadByte(); - var b2 = InnerStream.ReadByte(); - var flags = InnerStream.ReadByte(); - - if (b1 < 0 || b2 < 0 || flags < 0) - throw new InvalidDataException("Truncated compress header."); - - if (b1 != CompressConstants.Magic1 || b2 != CompressConstants.Magic2) - throw new InvalidDataException("Invalid compress magic bytes."); - - var maxBits = flags & CompressConstants.MaxBitsMask; - var blockMode = (flags & CompressConstants.BlockModeFlag) != 0; - - if (maxBits < CompressConstants.MinBits || maxBits > CompressConstants.DefaultMaxBits) - throw new InvalidDataException($"Invalid compress max bits: {maxBits}"); - - // LZW decompress - var decoder = new LzwDecoder( - InnerStream, - minBits: CompressConstants.MinBits, - maxBits: maxBits, - useClearCode: blockMode, - useStopCode: false, - bitOrder: BitOrder.LsbFirst); - - this._decompressedData = decoder.Decode(); + using var input = new MemoryStream(); + InnerStream.CopyTo(input); + this._decompressedData = LzcCodec.Decompress(input.ToArray()); this._decompressPos = 0; } } diff --git a/FileFormats/FileFormat.Squeeze/NuFx/NuFxFormatDescriptor.cs b/FileFormats/FileFormat.Squeeze/NuFx/NuFxFormatDescriptor.cs index 5f5617511..ee0abd237 100644 --- a/FileFormats/FileFormat.Squeeze/NuFx/NuFxFormatDescriptor.cs +++ b/FileFormats/FileFormat.Squeeze/NuFx/NuFxFormatDescriptor.cs @@ -10,7 +10,7 @@ namespace FileFormat.NuFx; /// /// NuFX / ShrinkIt archive descriptor for Apple II and Apple IIgs archives. -/// Supports plain SHK/SDK archives, native Stored/Squeeze/NuLZW1/NuLZW2 creation, +/// Supports plain SHK/SDK archives, native Stored/Squeeze/NuLZW1/NuLZW2/LZC-12/LZC-16 creation, /// record-preserving direct add/replace/remove, and slack-compacting rebuilds. /// public sealed class NuFxFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, @@ -32,6 +32,8 @@ public sealed class NuFxFormatDescriptor : IFormatDescriptor, IArchiveFormatOper public IReadOnlyList Methods => [ new("nulzw2", "ShrinkIt LZW/2"), new("nulzw1", "ShrinkIt LZW/1"), + new("lzc16", "UNIX compress LZC-16"), + new("lzc12", "UNIX compress LZC-12"), new("squeeze", "Squeeze"), new("stored", "Stored"), new("auto", "Auto (smallest)"), @@ -39,7 +41,7 @@ public sealed class NuFxFormatDescriptor : IFormatDescriptor, IArchiveFormatOper public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; public string Description => - "Apple II/IIgs NuFX (ShrinkIt) archive — SHK/SDK read/write with Stored, Squeeze, LZW/1 and LZW/2 threads."; + "Apple II/IIgs NuFX (ShrinkIt) archive — SHK/SDK read/write with Stored, Squeeze, LZW/1, LZW/2, LZC-12 and LZC-16 threads."; public IReadOnlyList OptionsSchema { get; } = [ new("Mode", "Archive mode", FormatOptionKind.Enum, "Files", ["Files", "DiskImage"], @@ -302,7 +304,7 @@ public ValidationResult ValidateIntegrity(Stream stream) { var validEntries = 0; foreach (var record in parsed.Records) { var format = record.DataThread?.Format ?? (ushort)0; - if (format > 3) { + if (format > 5) { issues.Add(new ValidationIssue(ValidationLevel.Integrity, IssueSeverity.Warning, "NUFX_UNCHECKED_METHOD", $"'{record.Name}' uses compression format {format}, which is structurally preserved but not decoded by this implementation.", record.StartOffset)); @@ -370,6 +372,8 @@ private static string NormalizeMethod(string? method) { "squeeze" or "sq" => "squeeze", "nulzw1" or "lzw1" => "nulzw1", "nulzw2" or "lzw2" => "nulzw2", + "lzc12" or "lzc-12" => "lzc12", + "lzc16" or "lzc-16" => "lzc16", "auto" => "auto", _ => throw new NotSupportedException($"NuFX creation method '{method}' is not supported."), }; @@ -543,6 +547,8 @@ internal static byte[] ExtractRecord(Stream archive, NuFxRecord record) { 1 => DecompressSqueeze(compressed), 2 => NuLzwCodec.Decompress(compressed, NuLzwVariant.Lzw1, logicalLength), 3 => NuLzwCodec.Decompress(compressed, NuLzwVariant.Lzw2, logicalLength), + 4 => LzcCodec.Decompress(compressed, logicalLength, 12), + 5 => LzcCodec.Decompress(compressed, logicalLength, 16), _ => throw new NotSupportedException($"NuFX thread compression format {thread.Format} is not supported for extraction."), }; @@ -573,6 +579,8 @@ internal static byte[] ReplaceDataForkPreservingRecord(Stream archive, NuFxRecor 1 => "squeeze", 2 => "nulzw1", 3 => "nulzw2", + 4 => "lzc12", + 5 => "lzc16", _ => "stored", }; var selected = CompressBest(newData, method); @@ -897,6 +905,10 @@ private static (ushort Format, byte[] Bytes) CompressBest(byte[] data, string me return (2, NuLzwCodec.Compress(data, NuLzwVariant.Lzw1)); if (method == "nulzw2") return (3, NuLzwCodec.Compress(data, NuLzwVariant.Lzw2)); + if (method == "lzc12") + return (4, LzcCodec.Compress(data, 12)); + if (method == "lzc16") + return (5, LzcCodec.Compress(data, 16)); if (method != "auto") throw new NotSupportedException($"Unsupported NuFX method '{method}'."); @@ -905,6 +917,8 @@ private static (ushort Format, byte[] Bytes) CompressBest(byte[] data, string me (1, CompressSqueeze(data)), (2, NuLzwCodec.Compress(data, NuLzwVariant.Lzw1)), (3, NuLzwCodec.Compress(data, NuLzwVariant.Lzw2)), + (4, LzcCodec.Compress(data, 12)), + (5, LzcCodec.Compress(data, 16)), }; return candidates.OrderBy(c => c.Bytes.Length).ThenBy(c => c.Format).First(); }