diff --git a/Compression.Lib/Compression.Lib.csproj b/Compression.Lib/Compression.Lib.csproj index 8e2da84b2..06fc3b4c2 100644 --- a/Compression.Lib/Compression.Lib.csproj +++ b/Compression.Lib/Compression.Lib.csproj @@ -464,6 +464,7 @@ + diff --git a/Compression.Tests/BinaryII/BinaryIITests.cs b/Compression.Tests/BinaryII/BinaryIITests.cs new file mode 100644 index 000000000..25f19ac37 --- /dev/null +++ b/Compression.Tests/BinaryII/BinaryIITests.cs @@ -0,0 +1,193 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using FileFormat.BinaryII; + +namespace Compression.Tests.BinaryII; + +[TestFixture] +public sealed class BinaryIITests { + [Test] + public void Descriptor_AdvertisesRealReadWriteAndSqueeze() { + var d = new BinaryIIFormatDescriptor(); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); + Assert.That(d.Methods.Select(m => m.Name), Is.EquivalentTo(new[] { "stored", "squeeze", "auto" })); + Assert.That(d.Extensions, Does.Contain(".bny")); + Assert.That(d.Extensions, Does.Contain(".bqy")); + } + + [Test] + public void Stored_CreateRoundTripsFilesAndDirectories() { + var d = new BinaryIIFormatDescriptor(); + var inputs = new ArchiveInputInfo[] { + new("DIR", "dir", true), + ArchiveInputInfo.InMemory("dir/hello.txt", "hello"u8), + ArchiveInputInfo.InMemory("root.bin", new byte[] { 0, 1, 2, 3, 4 }), + }; + using var archive = new MemoryStream(); + + d.Create(archive, inputs, new FormatCreateOptions { MethodName = "stored" }); + + var bytes = archive.ToArray(); + Assert.That(bytes.Length % 128, Is.Zero); + Assert.That(bytes.AsSpan(0, 3).ToArray(), Is.EqualTo(new byte[] { 0x0A, 0x47, 0x4C })); + archive.Position = 0; + var listed = d.List(archive, null); + Assert.That(listed.Select(e => e.Name), Is.EqualTo(new[] { "DIR", "DIR/HELLO.TXT", "ROOT.BIN" })); + Assert.That(listed.Single(e => e.Name == "DIR").IsDirectory, Is.True); + + archive.Position = 0; + Assert.That(d.ExtractEntryToMemory(archive, "DIR/HELLO.TXT", null), Is.EqualTo("hello"u8.ToArray())); + archive.Position = 0; + Assert.That(d.ExtractEntryToMemory(archive, "ROOT.BIN", null), Is.EqualTo(new byte[] { 0, 1, 2, 3, 4 })); + } + + [Test] + public void Squeeze_CreateSetsFlagAndRoundTrips() { + var d = new BinaryIIFormatDescriptor(); + var payload = Enumerable.Repeat((byte)'A', 4096).Concat(Enumerable.Repeat((byte)'B', 1024)).ToArray(); + using var archive = new MemoryStream(); + + d.Create(archive, [ArchiveInputInfo.InMemory("sample.dat", payload)], new FormatCreateOptions { MethodName = "squeeze" }); + + var raw = archive.ToArray(); + Assert.That((raw[0x7D] & 0x80) != 0, Is.True); + Assert.That(raw[128], Is.EqualTo(0x76)); + Assert.That(raw[129], Is.EqualTo(0xFF)); + + archive.Position = 0; + var listed = d.List(archive, null); + Assert.That(listed.Single().Method, Is.EqualTo("Squeeze")); + archive.Position = 0; + Assert.That(d.ExtractEntryToMemory(archive, "SAMPLE.DAT", null), Is.EqualTo(payload)); + } + + [Test] + public void QqSuffix_TriggersHistoricalSqueezeFallbackWithoutFlag() { + var d = new BinaryIIFormatDescriptor(); + var payload = Enumerable.Repeat((byte)0x5A, 1024).ToArray(); + using var archive = new MemoryStream(); + d.Create(archive, [ArchiveInputInfo.InMemory("thing.qq", payload)], new FormatCreateOptions { MethodName = "squeeze" }); + + var raw = archive.ToArray(); + raw[0x7D] &= 0x7F; + using var compat = new MemoryStream(raw, writable: false); + + Assert.That(d.ExtractEntryToMemory(compat, "THING.QQ", null), Is.EqualTo(payload)); + } + + [Test] + public void DirectAddReplaceRemovePatchesCountdownAndPreservesOtherPayloads() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ + ArchiveInputInfo.InMemory("one.bin", Enumerable.Repeat((byte)1, 129).ToArray()), + ArchiveInputInfo.InMemory("two.bin", Enumerable.Repeat((byte)2, 260).ToArray()), + ], new FormatCreateOptions { MethodName = "stored" }); + + var originalTwo = d.ExtractEntryToMemory(Reset(archive), "TWO.BIN", null); + d.Add(archive, [ArchiveInputInfo.InMemory("three.bin", Enumerable.Repeat((byte)3, 17).ToArray())]); + + var afterAdd = archive.ToArray(); + Assert.That(afterAdd[0x7F], Is.EqualTo(2)); + var secondHeader = 128 + 256; + Assert.That(afterAdd[secondHeader + 0x7F], Is.EqualTo(1)); + + d.Add(archive, [ArchiveInputInfo.InMemory("one.bin", Enumerable.Repeat((byte)9, 700).ToArray())]); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "ONE.BIN", null), Is.EqualTo(Enumerable.Repeat((byte)9, 700).ToArray())); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "TWO.BIN", null), Is.EqualTo(originalTwo)); + + d.Remove(archive, ["two.bin"]); + var names = d.List(Reset(archive), null).Select(e => e.Name).ToArray(); + Assert.That(names, Is.EqualTo(new[] { "ONE.BIN", "THREE.BIN" })); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "THREE.BIN", null), Is.EqualTo(Enumerable.Repeat((byte)3, 17).ToArray())); + var afterRemove = archive.ToArray(); + Assert.That(afterRemove[0x7F], Is.EqualTo(1)); + } + + [Test] + public void AddSynthesizesMissingParentDirectoryInPlace() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ArchiveInputInfo.InMemory("root.bin", "root"u8)], new FormatCreateOptions()); + + d.Add(archive, [ArchiveInputInfo.InMemory("sub/new.bin", "new"u8)]); + + var entries = d.List(Reset(archive), null); + Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "ROOT.BIN", "SUB", "SUB/NEW.BIN" })); + Assert.That(entries[1].IsDirectory, Is.True); + } + + [Test] + public void RemoveDirectoryRemovesItsDescendants() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ + ArchiveInputInfo.InMemory("keep.bin", "keep"u8), + ArchiveInputInfo.InMemory("tree/a.bin", "a"u8), + ArchiveInputInfo.InMemory("tree/b.bin", "b"u8), + ], new FormatCreateOptions()); + + d.Remove(archive, ["tree"]); + + Assert.That(d.List(Reset(archive), null).Select(e => e.Name), Is.EqualTo(new[] { "KEEP.BIN" })); + } + + [Test] + public void AutoCompressionUsesBlockRoundedSize() { + var d = new BinaryIIFormatDescriptor(); + using var compressible = new MemoryStream(); + d.Create(compressible, [ArchiveInputInfo.InMemory("repeat.bin", Enumerable.Repeat((byte)'A', 4096).ToArray())], + new FormatCreateOptions { MethodName = "auto" }); + Assert.That((compressible.ToArray()[0x7D] & 0x80) != 0, Is.True); + + using var tiny = new MemoryStream(); + d.Create(tiny, [ArchiveInputInfo.InMemory("tiny.bin", new byte[] { 1, 2, 3 })], + new FormatCreateOptions { MethodName = "auto" }); + Assert.That((tiny.ToArray()[0x7D] & 0x80) != 0, Is.False); + } + + [Test] + public void NamesAreNormalizedToProDosPartialPathRules() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ + ArchiveInputInfo.InMemory("123 very-long-directory-name/hello world.txt", "x"u8), + ArchiveInputInfo.InMemory("123 very-long-directory-name/hello world.txt", "y"u8), + ], new FormatCreateOptions()); + + var names = d.List(Reset(archive), null).Select(e => e.Name).ToArray(); + Assert.That(names[0], Does.StartWith("X123.VERY.LONG")); + Assert.That(names[^2], Is.Not.EqualTo(names[^1]).IgnoreCase); + Assert.That(names.All(n => n.Length <= 64), Is.True); + } + + [Test] + public void EncryptionIsRejected() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + Assert.Throws(() => + d.Create(archive, [ArchiveInputInfo.InMemory("x.bin", "x"u8)], new FormatCreateOptions { Password = "secret" })); + } + + [Test] + public void DefragmentTrimsTrailingGarbageButKeepsPayload() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ArchiveInputInfo.InMemory("x.bin", Enumerable.Repeat((byte)0xA5, 200).ToArray())], new FormatCreateOptions()); + var canonicalLength = archive.Length; + archive.Position = archive.Length; + archive.Write(new byte[321]); + + d.Defragment(archive); + + Assert.That(archive.Length, Is.EqualTo(canonicalLength)); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "X.BIN", null), Is.EqualTo(Enumerable.Repeat((byte)0xA5, 200).ToArray())); + } + + private static MemoryStream Reset(MemoryStream stream) { + stream.Position = 0; + return stream; + } +} diff --git a/CompressionWorkbench.slnx b/CompressionWorkbench.slnx index 6f558c5d6..10bb06389 100644 --- a/CompressionWorkbench.slnx +++ b/CompressionWorkbench.slnx @@ -461,6 +461,7 @@ + diff --git a/FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs b/FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs new file mode 100644 index 000000000..2f2d802cc --- /dev/null +++ b/FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs @@ -0,0 +1,509 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using FileFormat.Squeeze; + +namespace FileFormat.BinaryII; + +internal static class BinaryIIConstants { + public const int HeaderSize = 128; + public const int Alignment = 128; + public const int MaxRecords = 256; + public const int MaxNameLength = 64; + + public const byte ProDosAccessDefault = 0xE3; + public const byte ProDosFileTypeBinary = 0x06; + public const byte ProDosFileTypeDirectory = 0x0F; + public const byte ProDosStorageSeedling = 0x01; + public const byte ProDosStorageSapling = 0x02; + public const byte ProDosStorageTree = 0x03; + public const byte ProDosStorageDirectory = 0x0D; + + public const byte DataFlagSparse = 0x01; + public const byte DataFlagEncrypted = 0x40; + public const byte DataFlagCompressed = 0x80; + + public static int RoundUp128(int value) => checked((value + 127) & ~127); +} + +internal sealed record BinaryIIRecord( + string Name, + bool IsDirectory, + bool IsPhantom, + bool IsCompressed, + bool IsEncrypted, + bool IsSparse, + byte FileType, + byte StorageType, + byte DataFlags, + int StoredLength, + long HeaderOffset, + long DataOffset, + int PhysicalLength, + byte FilesToFollow +); + +internal enum BinaryIICompressionMode { + Stored, + Squeeze, + Auto, +} + +internal sealed record BinaryIIWriteRecord( + string Name, + bool IsDirectory, + byte[] Data, + bool Compress +); + +internal sealed class BinaryIIReader { + private readonly byte[] _data; + private readonly List _records = []; + + public IReadOnlyList PhysicalRecords => this._records; + public IEnumerable Entries => this._records.Where(r => !r.IsPhantom); + + public BinaryIIReader(Stream input) { + ArgumentNullException.ThrowIfNull(input); + if (input.CanSeek) input.Position = 0; + using var ms = new MemoryStream(); + input.CopyTo(ms); + this._data = ms.ToArray(); + this.Parse(); + } + + public byte[] Extract(BinaryIIRecord entry) { + ArgumentNullException.ThrowIfNull(entry); + if (entry.IsDirectory) + return []; + if (entry.IsEncrypted) + throw new NotSupportedException($"Binary II entry '{entry.Name}' is encrypted; the format never standardized an encryption method."); + if (entry.IsSparse) + throw new NotSupportedException($"Binary II entry '{entry.Name}' is marked sparse; the Binary II specification does not define sparse reconstruction semantics."); + + var stored = this._data.AsSpan((int)entry.DataOffset, entry.StoredLength).ToArray(); + if (!entry.IsCompressed) + return stored; + + using var src = new MemoryStream(stored, writable: false); + using var dst = new MemoryStream(); + SqueezeStream.Decompress(src, dst); + return dst.ToArray(); + } + + private void Parse() { + if (this._data.Length == 0) + return; + if (this._data.Length < BinaryIIConstants.HeaderSize) + throw new InvalidDataException("Binary II: archive is shorter than one 128-byte record header."); + + var offset = 0; + for (var recordIndex = 0; recordIndex < BinaryIIConstants.MaxRecords; recordIndex++) { + if (offset + BinaryIIConstants.HeaderSize > this._data.Length) + throw new InvalidDataException("Binary II: truncated record header."); + + var h = this._data.AsSpan(offset, BinaryIIConstants.HeaderSize); + if (h[0] != 0x0A || h[1] != 0x47 || h[2] != 0x4C || h[0x12] != 0x02) + throw new InvalidDataException($"Binary II: invalid record signature at offset 0x{offset:X}."); + + var nameLength = h[0x17]; + if (nameLength > BinaryIIConstants.MaxNameLength) + throw new InvalidDataException($"Binary II: record at 0x{offset:X} has invalid filename length {nameLength}."); + var name = Encoding.ASCII.GetString(h.Slice(0x18, nameLength)).Replace('\\', '/'); + + var lowEof = (uint)(h[0x14] | (h[0x15] << 8) | (h[0x16] << 16)); + var eof = lowEof | ((uint)h[0x74] << 24); + if (eof > int.MaxValue) + throw new InvalidDataException($"Binary II: entry '{name}' is too large for this in-memory reader."); + var storedLength = (int)eof; + var paddedLength = BinaryIIConstants.RoundUp128(storedLength); + var dataOffset = checked(offset + BinaryIIConstants.HeaderSize); + var physicalLength = checked(BinaryIIConstants.HeaderSize + paddedLength); + if ((long)dataOffset + storedLength > this._data.LongLength) + throw new InvalidDataException($"Binary II: entry '{name}' extends beyond end of archive."); + + var fileType = h[0x04]; + var storageType = h[0x07]; + var flags = h[0x7D]; + var phantom = h[0x7C] != 0; + var directory = fileType == BinaryIIConstants.ProDosFileTypeDirectory || storageType == BinaryIIConstants.ProDosStorageDirectory; + var compressed = (flags & BinaryIIConstants.DataFlagCompressed) != 0 + || name.EndsWith(".QQ", StringComparison.OrdinalIgnoreCase); + var encrypted = (flags & BinaryIIConstants.DataFlagEncrypted) != 0; + var sparse = (flags & BinaryIIConstants.DataFlagSparse) != 0; + var follows = h[0x7F]; + + this._records.Add(new BinaryIIRecord( + name, + directory, + phantom, + compressed, + encrypted, + sparse, + fileType, + storageType, + flags, + storedLength, + offset, + dataOffset, + physicalLength, + follows + )); + + var nextOffset = checked(offset + physicalLength); + if (nextOffset + BinaryIIConstants.HeaderSize > this._data.Length) + return; + var next = this._data.AsSpan(nextOffset, BinaryIIConstants.HeaderSize); + if (next[0] != 0x0A || next[1] != 0x47 || next[2] != 0x4C || next[0x12] != 0x02) + return; + offset = nextOffset; + } + + throw new InvalidDataException("Binary II: archive exceeds the 256-record limit imposed by the files-to-follow byte."); + } +} + +internal static class BinaryIIWriter { + public static byte[] Build(IReadOnlyList inputs, BinaryIICompressionMode mode) { + ArgumentNullException.ThrowIfNull(inputs); + var records = PrepareRecords(inputs, mode); + using var ms = new MemoryStream(); + for (var i = 0; i < records.Count; i++) { + var bytes = CreatePhysicalRecord(records[i], records.Count - i - 1); + ms.Write(bytes); + } + return ms.ToArray(); + } + + public static List PrepareRecords( + IReadOnlyList inputs, + BinaryIICompressionMode mode + ) { + var result = new List(); + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var input in inputs.OrderBy(i => i.ArchiveName, StringComparer.OrdinalIgnoreCase)) { + var normalized = NormalizePath(input.ArchiveName); + var parts = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0) + continue; + + var parent = ""; + for (var i = 0; i < parts.Length - (input.IsDirectory ? 0 : 1); i++) { + parent = parent.Length == 0 ? parts[i] : parent + "/" + parts[i]; + if (names.Add(parent)) + result.Add(new BinaryIIWriteRecord(parent, true, [], false)); + } + + if (input.IsDirectory) { + if (names.Add(normalized)) + result.Add(new BinaryIIWriteRecord(normalized, true, [], false)); + continue; + } + + var unique = MakeUnique(normalized, names); + names.Add(unique); + var data = input.ReadContent(); + var compress = ShouldCompress(unique, data, mode); + result.Add(new BinaryIIWriteRecord(unique, false, data, compress)); + } + + if (result.Count > BinaryIIConstants.MaxRecords) + throw new InvalidDataException($"Binary II supports at most {BinaryIIConstants.MaxRecords} physical records."); + return result; + } + + public static byte[] CreatePhysicalRecord(BinaryIIWriteRecord record, int filesToFollow) { + ArgumentNullException.ThrowIfNull(record); + if (filesToFollow is < 0 or > 255) + throw new ArgumentOutOfRangeException(nameof(filesToFollow)); + + var nameBytes = Encoding.ASCII.GetBytes(record.Name); + if (nameBytes.Length is < 1 or > BinaryIIConstants.MaxNameLength) + throw new InvalidDataException($"Binary II filename '{record.Name}' is outside the 1..64 byte range."); + + byte[] stored; + var compressed = false; + if (!record.IsDirectory && record.Compress) { + using var src = new MemoryStream(record.Data, writable: false); + using var dst = new MemoryStream(); + SqueezeStream.Compress(src, dst, Path.GetFileName(record.Name)); + stored = dst.ToArray(); + compressed = true; + } else { + stored = record.IsDirectory ? [] : record.Data; + } + + var padded = BinaryIIConstants.RoundUp128(stored.Length); + var output = new byte[BinaryIIConstants.HeaderSize + padded]; + var h = output.AsSpan(0, BinaryIIConstants.HeaderSize); + + h[0] = 0x0A; + h[1] = 0x47; + h[2] = 0x4C; + h[0x03] = BinaryIIConstants.ProDosAccessDefault; + h[0x04] = record.IsDirectory ? BinaryIIConstants.ProDosFileTypeDirectory : BinaryIIConstants.ProDosFileTypeBinary; + BinaryPrimitives.WriteUInt16LittleEndian(h[0x05..], 0); + h[0x07] = record.IsDirectory ? BinaryIIConstants.ProDosStorageDirectory : StorageTypeForLength(record.Data.Length); + BinaryPrimitives.WriteUInt16LittleEndian(h[0x08..], 0); + h[0x12] = 0x02; + + WriteUInt24LittleEndian(h[0x14..], stored.Length); + h[0x17] = (byte)nameBytes.Length; + nameBytes.AsSpan().CopyTo(h[0x18..]); + h[0x74] = (byte)((uint)stored.Length >> 24); + h[0x79] = 0x00; + h[0x7C] = 0x00; + h[0x7D] = compressed ? BinaryIIConstants.DataFlagCompressed : (byte)0x00; + h[0x7E] = 0x01; + h[0x7F] = (byte)filesToFollow; + + if (stored.Length > 0) + stored.AsSpan().CopyTo(output.AsSpan(BinaryIIConstants.HeaderSize)); + return output; + } + + public static string NormalizePath(string path) { + if (string.IsNullOrWhiteSpace(path)) + return "X"; + + var rawParts = path.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries); + var cooked = new List(rawParts.Length); + foreach (var raw in rawParts) { + if (raw is "." or "..") + continue; + var upper = raw.ToUpperInvariant(); + var sb = new StringBuilder(upper.Length + 1); + foreach (var ch in upper) { + if (ch is >= 'A' and <= 'Z' || ch is >= '0' and <= '9' || ch == '.') + sb.Append(ch); + else + sb.Append('.'); + } + + if (sb.Length == 0 || sb[0] is < 'A' or > 'Z') + sb.Insert(0, 'X'); + if (sb.Length > 15) + sb.Length = 15; + cooked.Add(sb.ToString()); + } + + if (cooked.Count == 0) + return "X"; + + var joined = string.Join('/', cooked); + if (joined.Length <= BinaryIIConstants.MaxNameLength) + return joined; + + joined = joined[..BinaryIIConstants.MaxNameLength].TrimEnd('/'); + return joined.Length == 0 ? "X" : joined; + } + + private static string MakeUnique(string normalized, HashSet existing) { + if (!existing.Contains(normalized)) + return normalized; + + var slash = normalized.LastIndexOf('/'); + var parent = slash >= 0 ? normalized[..(slash + 1)] : ""; + var leaf = slash >= 0 ? normalized[(slash + 1)..] : normalized; + for (var n = 1; n < 10000; n++) { + var suffix = "." + n.ToString(System.Globalization.CultureInfo.InvariantCulture); + var maxLeaf = Math.Min(15, BinaryIIConstants.MaxNameLength - parent.Length); + var prefixLength = Math.Max(1, maxLeaf - suffix.Length); + var candidateLeaf = leaf[..Math.Min(leaf.Length, prefixLength)] + suffix; + var candidate = parent + candidateLeaf; + if (!existing.Contains(candidate)) + return candidate; + } + + throw new InvalidDataException($"Binary II could not disambiguate duplicate path '{normalized}'."); + } + + private static bool ShouldCompress(string name, byte[] data, BinaryIICompressionMode mode) { + if (mode == BinaryIICompressionMode.Stored || data.Length == 0) + return false; + if (mode == BinaryIICompressionMode.Squeeze) + return true; + + using var src = new MemoryStream(data, writable: false); + using var dst = new MemoryStream(); + SqueezeStream.Compress(src, dst, Path.GetFileName(name)); + return BinaryIIConstants.RoundUp128(checked((int)dst.Length)) < BinaryIIConstants.RoundUp128(data.Length); + } + + private static byte StorageTypeForLength(int length) + => length <= 512 ? BinaryIIConstants.ProDosStorageSeedling + : length <= 128 * 1024 ? BinaryIIConstants.ProDosStorageSapling + : BinaryIIConstants.ProDosStorageTree; + + private static void WriteUInt24LittleEndian(Span destination, int value) { + var u = (uint)value; + destination[0] = (byte)u; + destination[1] = (byte)(u >> 8); + destination[2] = (byte)(u >> 16); + } +} + +internal static class BinaryIIInPlaceModifier { + private const int CopyBufferSize = 64 * 1024; + + public static void Add(Stream archive, IReadOnlyList inputs) { + ValidateWritable(archive); + ArgumentNullException.ThrowIfNull(inputs); + + var requested = BinaryIIWriter.PrepareRecords(inputs, BinaryIICompressionMode.Stored); + foreach (var record in requested) { + var reader = new BinaryIIReader(archive); + var existing = reader.PhysicalRecords.FirstOrDefault( + r => !r.IsPhantom && string.Equals(r.Name, record.Name, StringComparison.OrdinalIgnoreCase)); + + if (existing is not null) { + if (existing.IsDirectory && record.IsDirectory) + continue; + if (existing.IsDirectory != record.IsDirectory) + throw new InvalidOperationException($"Binary II cannot replace '{record.Name}' with a different entry kind while descendants may exist."); + ReplaceRecord(archive, existing, BinaryIIWriter.CreatePhysicalRecord(record, existing.FilesToFollow)); + } else { + var physicalCount = reader.PhysicalRecords.Count; + if (physicalCount >= BinaryIIConstants.MaxRecords) + throw new InvalidDataException("Binary II archive already contains the maximum 256 records."); + + var logicalEnd = reader.PhysicalRecords.Count == 0 + ? 0L + : reader.PhysicalRecords[^1].HeaderOffset + reader.PhysicalRecords[^1].PhysicalLength; + if (archive.Length != logicalEnd) + archive.SetLength(logicalEnd); + archive.Position = logicalEnd; + archive.Write(BinaryIIWriter.CreatePhysicalRecord(record, 0)); + } + } + + PatchFilesToFollow(archive); + } + + public static void Remove(Stream archive, string[] entryNames) { + ValidateWritable(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Length == 0) + return; + + var normalized = entryNames + .Where(n => !string.IsNullOrWhiteSpace(n)) + .Select(BinaryIIWriter.NormalizePath) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var reader = new BinaryIIReader(archive); + var removals = reader.PhysicalRecords + .Where(r => !r.IsPhantom && normalized.Any(n => + string.Equals(r.Name, n, StringComparison.OrdinalIgnoreCase) + || r.Name.StartsWith(n + "/", StringComparison.OrdinalIgnoreCase))) + .OrderByDescending(r => r.HeaderOffset) + .ToList(); + + foreach (var record in removals) + RemoveRange(archive, record.HeaderOffset, record.PhysicalLength); + + PatchFilesToFollow(archive); + } + + public static void Defragment(Stream archive) { + ValidateWritable(archive); + var reader = new BinaryIIReader(archive); + if (reader.PhysicalRecords.Count == 0) { + archive.SetLength(0); + return; + } + + foreach (var record in reader.PhysicalRecords) { + var payloadEnd = record.DataOffset + record.StoredLength; + var recordEnd = record.HeaderOffset + record.PhysicalLength; + if (payloadEnd < recordEnd) { + archive.Position = payloadEnd; + WriteZeros(archive, checked((int)(recordEnd - payloadEnd))); + } + } + + var end = reader.PhysicalRecords[^1].HeaderOffset + reader.PhysicalRecords[^1].PhysicalLength; + archive.SetLength(end); + PatchFilesToFollow(archive); + } + + private static void ReplaceRecord(Stream archive, BinaryIIRecord oldRecord, byte[] replacement) { + var delta = replacement.LongLength - oldRecord.PhysicalLength; + var tailStart = oldRecord.HeaderOffset + oldRecord.PhysicalLength; + ShiftTail(archive, tailStart, delta); + archive.Position = oldRecord.HeaderOffset; + archive.Write(replacement); + } + + private static void RemoveRange(Stream archive, long offset, long length) + => ShiftTail(archive, offset + length, -length); + + private static void ShiftTail(Stream archive, long tailStart, long delta) { + if (delta == 0) + return; + + var oldLength = archive.Length; + if (tailStart < 0 || tailStart > oldLength) + throw new ArgumentOutOfRangeException(nameof(tailStart)); + + var buffer = new byte[CopyBufferSize]; + if (delta > 0) { + archive.SetLength(checked(oldLength + delta)); + var remaining = oldLength - tailStart; + while (remaining > 0) { + var chunk = (int)Math.Min(buffer.Length, remaining); + var readPos = tailStart + remaining - chunk; + archive.Position = readPos; + archive.ReadExactly(buffer.AsSpan(0, chunk)); + archive.Position = readPos + delta; + archive.Write(buffer, 0, chunk); + remaining -= chunk; + } + } else { + var shift = -delta; + var readPos = tailStart; + var writePos = tailStart - shift; + while (readPos < oldLength) { + var chunk = (int)Math.Min(buffer.Length, oldLength - readPos); + archive.Position = readPos; + archive.ReadExactly(buffer.AsSpan(0, chunk)); + archive.Position = writePos; + archive.Write(buffer, 0, chunk); + readPos += chunk; + writePos += chunk; + } + archive.SetLength(checked(oldLength - shift)); + } + } + + private static void PatchFilesToFollow(Stream archive) { + var reader = new BinaryIIReader(archive); + if (reader.PhysicalRecords.Count > BinaryIIConstants.MaxRecords) + throw new InvalidDataException("Binary II archive contains more than 256 physical records."); + + for (var i = 0; i < reader.PhysicalRecords.Count; i++) { + archive.Position = reader.PhysicalRecords[i].HeaderOffset + 0x7F; + archive.WriteByte((byte)(reader.PhysicalRecords.Count - i - 1)); + } + if (archive.CanSeek) + archive.Position = 0; + } + + private static void ValidateWritable(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Binary II direct modification requires a readable, writable, seekable stream.", nameof(archive)); + } + + private static void WriteZeros(Stream output, int count) { + Span zeros = stackalloc byte[128]; + zeros.Clear(); + while (count > 0) { + var n = Math.Min(count, zeros.Length); + output.Write(zeros[..n]); + count -= n; + } + } +} diff --git a/FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs b/FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs new file mode 100644 index 000000000..9e7b3a1e3 --- /dev/null +++ b/FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs @@ -0,0 +1,164 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.BinaryII; + +/// +/// Apple II Binary II / BLU archive (.bny/.bqy). +/// +/// +/// Binary II is a deliberately simple record stream: every member is a 128-byte +/// metadata header followed by its payload rounded to a 128-byte boundary. +/// Version-1 headers preserve ProDOS/GS/OS metadata and mark Squeeze-compressed +/// payloads with data flag 0x80. Historical BLU extractors also infer Squeeze +/// from a .QQ member suffix, which this reader accepts for compatibility. +/// +public sealed class BinaryIIFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveCreatable, + IArchiveModifiable, + IArchiveDefragmentable, + IArchiveLayoutMap { + + public string Id => "BinaryII"; + public string DisplayName => "Apple II Binary II"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | + FormatCapabilities.CanExtract | + FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | + FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | + FormatCapabilities.SupportsDirectories; + public string DefaultExtension => ".bny"; + public IReadOnlyList Extensions => [".bny", ".bqy"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => + [new([0x0A, 0x47, 0x4C], Confidence: 0.99)]; + public IReadOnlyList Methods => [ + new("stored", "Stored"), + new("squeeze", "Squeeze"), + new("auto", "Auto", true), + ]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => + "Apple II Binary II / BLU record archive with stored or Squeeze-compressed members and direct 128-byte-record mutation"; + + public List List(Stream stream, string? password) { + ArgumentNullException.ThrowIfNull(stream); + var reader = new BinaryIIReader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + Index: index, + Name: entry.Name, + OriginalSize: entry.IsDirectory ? 0 : entry.IsCompressed ? -1 : entry.StoredLength, + CompressedSize: entry.StoredLength, + Method: entry.IsCompressed ? "Squeeze" : "Stored", + IsDirectory: entry.IsDirectory, + IsEncrypted: entry.IsEncrypted, + LastModified: null, + Kind: entry.IsPhantom ? "phantom" : null + )).ToList(); + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(outputDir); + var reader = new BinaryIIReader(stream); + foreach (var entry in reader.Entries) { + if (files is not null && !MatchesFilter(entry.Name, files)) + continue; + if (entry.IsDirectory) { + CreateSafeDirectory(outputDir, entry.Name); + continue; + } + WriteFile(outputDir, entry.Name, reader.Extract(entry)); + } + } + + public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryName); + var reader = new BinaryIIReader(archive); + var entry = reader.Entries.FirstOrDefault(e => + string.Equals(e.Name, entryName, StringComparison.OrdinalIgnoreCase) + || string.Equals(Path.GetFileName(e.Name), entryName, StringComparison.OrdinalIgnoreCase)); + if (entry is null) + return []; + return reader.Extract(entry); + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (!output.CanWrite) + throw new ArgumentException("Output stream is not writable.", nameof(output)); + if (!string.IsNullOrEmpty(options.Password) || options.EncryptFilenames || !string.IsNullOrEmpty(options.EncryptionMethod)) + throw new NotSupportedException("Binary II does not define an interoperable encryption method."); + + var method = (options.MethodName ?? "stored").Trim().ToLowerInvariant(); + var mode = method switch { + "" or "stored" or "store" => BinaryIICompressionMode.Stored, + "squeeze" or "sq" => BinaryIICompressionMode.Squeeze, + "auto" => BinaryIICompressionMode.Auto, + _ => throw new NotSupportedException($"Binary II compression method '{options.MethodName}' is not supported."), + }; + + var bytes = BinaryIIWriter.Build(inputs, mode); + if (output.CanSeek) { + output.Position = 0; + output.SetLength(0); + } + output.Write(bytes); + } + + /// + /// Adds new entries or replaces same-name entries by editing the existing + /// 128-byte record stream in place. Existing payloads are not decoded/re-encoded. + /// + public void Add(Stream archive, IReadOnlyList inputs) + => BinaryIIInPlaceModifier.Add(archive, inputs); + + /// + /// Removes named entries (and descendants of named directory entries) by + /// shifting the following record tail left and truncating the stream. + /// + public void Remove(Stream archive, string[] entryNames) + => BinaryIIInPlaceModifier.Remove(archive, entryNames); + + public void Defragment(Stream archive) + => BinaryIIInPlaceModifier.Defragment(archive); + + public void Defragment(Stream archive, DefragOptions options) + => BinaryIIInPlaceModifier.Defragment(archive); + + public IEnumerable EnumerateLayout(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + var reader = new BinaryIIReader(archive); + foreach (var entry in reader.PhysicalRecords) { + yield return new DefragBlockInfo(entry.HeaderOffset, BinaryIIConstants.HeaderSize, DefragBlockKind.MetadataReserved, entry.Name + " header"); + if (entry.StoredLength > 0) + yield return new DefragBlockInfo(entry.DataOffset, entry.StoredLength, DefragBlockKind.Used, entry.Name); + var padding = entry.PhysicalLength - BinaryIIConstants.HeaderSize - entry.StoredLength; + if (padding > 0) + yield return new DefragBlockInfo(entry.DataOffset + entry.StoredLength, padding, DefragBlockKind.Free, entry.Name + " padding"); + } + } + + private static void CreateSafeDirectory(string baseDir, string entryName) { + var parts = entryName.Replace('\\', '/') + .Split('/', StringSplitOptions.RemoveEmptyEntries) + .Where(p => p is not "." and not "..") + .ToArray(); + if (parts.Length == 0) + return; + var all = new string[parts.Length + 1]; + all[0] = baseDir; + Array.Copy(parts, 0, all, 1, parts.Length); + Directory.CreateDirectory(Path.Combine(all)); + } +} diff --git a/FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj b/FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj new file mode 100644 index 000000000..d5fec84dc --- /dev/null +++ b/FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj @@ -0,0 +1,9 @@ + + + FileFormat.BinaryII + + + + + + diff --git a/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj b/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj index 78cc0e257..5f0da11fc 100644 --- a/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj +++ b/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj @@ -68,6 +68,7 @@ + diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 6df766d07..29f31f861 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -615,7 +615,7 @@ This package is built against the repository's shared Core version. Consume a mu -Every public and protected member of all 968 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). +Every public and protected member of all 969 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 713901e72..60c18c857 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -3229,6 +3229,41 @@ Encodes files into BinHex 4.0 (.hqx) text format. | `BinHexWriter` | `BinHexWriter()` | | | `Write` | `static void Write(Stream output, string fileName, byte[] dataFork, byte[] resourceFork = null, string fileType = "TEXT", string fileCreator = "ttxt")` | Writes a BinHex 4.0 encoded file to the output stream. | +### Namespace `FileFormat.BinaryII` + +[`BinaryIIFormatDescriptor`](#binaryiiformatdescriptor) + +#### `BinaryIIFormatDescriptor` + +Apple II Binary II / BLU archive (.bny/.bqy). + +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `BinaryIIFormatDescriptor` | `BinaryIIFormatDescriptor()` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds new entries or replaces same-name entries by editing the existing 128-byte record stream in place. Existing payloads are not decoded/re-encoded. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | +| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes named entries (and descendants of named directory entries) by shifting the following record tail left and truncating the stream. | + ### Namespace `FileFormat.Bkf` [`BkfEntry`](#bkfentry) · [`BkfFormatDescriptor`](#bkfformatdescriptor) · [`BkfInPlaceModifier`](#bkfinplacemodifier) · [`BkfReader`](#bkfreader) · [`BkfWriter`](#bkfwriter) · [`BkfWriter.Item`](#bkfwriteritem)