From dc9281ffa94b943cced6ac20ae8af06a712eda5f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Sat, 29 Aug 2026 21:02:39 +0200 Subject: [PATCH] + read and write Apple II Binary II archives Binary II wraps each file in a 128-byte header carrying the ProDOS name, file and auxiliary type, dates and length, with the payload padded to the next block. Squeezed members are unpacked through the existing codec. The two projects that needed a reference to it got a Directory.Build.props each holding nothing but that reference. A Directory.Build.props does not inherit the one above it unless it says so, so both projects lost every repository-wide property, starting with TargetFramework. The reference goes in the project files, where their other formats are listed. --- Compression.Lib/Compression.Lib.csproj | 1 + Compression.Tests/BinaryII/BinaryIITests.cs | 193 +++++++ CompressionWorkbench.slnx | 1 + .../FileFormat.BinaryII/BinaryIIArchive.cs | 509 ++++++++++++++++++ .../BinaryIIFormatDescriptor.cs | 164 ++++++ .../FileFormat.BinaryII.csproj | 9 + .../Hawkynt.FileFormats.Archives.csproj | 1 + Hawkynt.FileFormats.Archives/README.md | 2 +- Hawkynt.FileFormats.Archives/REFERENCE.md | 35 ++ 9 files changed, 914 insertions(+), 1 deletion(-) create mode 100644 Compression.Tests/BinaryII/BinaryIITests.cs create mode 100644 FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs create mode 100644 FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs create mode 100644 FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj diff --git a/Compression.Lib/Compression.Lib.csproj b/Compression.Lib/Compression.Lib.csproj index 8e2da84b2..06fc3b4c2 100644 --- a/Compression.Lib/Compression.Lib.csproj +++ b/Compression.Lib/Compression.Lib.csproj @@ -464,6 +464,7 @@ + diff --git a/Compression.Tests/BinaryII/BinaryIITests.cs b/Compression.Tests/BinaryII/BinaryIITests.cs new file mode 100644 index 000000000..25f19ac37 --- /dev/null +++ b/Compression.Tests/BinaryII/BinaryIITests.cs @@ -0,0 +1,193 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using FileFormat.BinaryII; + +namespace Compression.Tests.BinaryII; + +[TestFixture] +public sealed class BinaryIITests { + [Test] + public void Descriptor_AdvertisesRealReadWriteAndSqueeze() { + var d = new BinaryIIFormatDescriptor(); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); + Assert.That(d.Methods.Select(m => m.Name), Is.EquivalentTo(new[] { "stored", "squeeze", "auto" })); + Assert.That(d.Extensions, Does.Contain(".bny")); + Assert.That(d.Extensions, Does.Contain(".bqy")); + } + + [Test] + public void Stored_CreateRoundTripsFilesAndDirectories() { + var d = new BinaryIIFormatDescriptor(); + var inputs = new ArchiveInputInfo[] { + new("DIR", "dir", true), + ArchiveInputInfo.InMemory("dir/hello.txt", "hello"u8), + ArchiveInputInfo.InMemory("root.bin", new byte[] { 0, 1, 2, 3, 4 }), + }; + using var archive = new MemoryStream(); + + d.Create(archive, inputs, new FormatCreateOptions { MethodName = "stored" }); + + var bytes = archive.ToArray(); + Assert.That(bytes.Length % 128, Is.Zero); + Assert.That(bytes.AsSpan(0, 3).ToArray(), Is.EqualTo(new byte[] { 0x0A, 0x47, 0x4C })); + archive.Position = 0; + var listed = d.List(archive, null); + Assert.That(listed.Select(e => e.Name), Is.EqualTo(new[] { "DIR", "DIR/HELLO.TXT", "ROOT.BIN" })); + Assert.That(listed.Single(e => e.Name == "DIR").IsDirectory, Is.True); + + archive.Position = 0; + Assert.That(d.ExtractEntryToMemory(archive, "DIR/HELLO.TXT", null), Is.EqualTo("hello"u8.ToArray())); + archive.Position = 0; + Assert.That(d.ExtractEntryToMemory(archive, "ROOT.BIN", null), Is.EqualTo(new byte[] { 0, 1, 2, 3, 4 })); + } + + [Test] + public void Squeeze_CreateSetsFlagAndRoundTrips() { + var d = new BinaryIIFormatDescriptor(); + var payload = Enumerable.Repeat((byte)'A', 4096).Concat(Enumerable.Repeat((byte)'B', 1024)).ToArray(); + using var archive = new MemoryStream(); + + d.Create(archive, [ArchiveInputInfo.InMemory("sample.dat", payload)], new FormatCreateOptions { MethodName = "squeeze" }); + + var raw = archive.ToArray(); + Assert.That((raw[0x7D] & 0x80) != 0, Is.True); + Assert.That(raw[128], Is.EqualTo(0x76)); + Assert.That(raw[129], Is.EqualTo(0xFF)); + + archive.Position = 0; + var listed = d.List(archive, null); + Assert.That(listed.Single().Method, Is.EqualTo("Squeeze")); + archive.Position = 0; + Assert.That(d.ExtractEntryToMemory(archive, "SAMPLE.DAT", null), Is.EqualTo(payload)); + } + + [Test] + public void QqSuffix_TriggersHistoricalSqueezeFallbackWithoutFlag() { + var d = new BinaryIIFormatDescriptor(); + var payload = Enumerable.Repeat((byte)0x5A, 1024).ToArray(); + using var archive = new MemoryStream(); + d.Create(archive, [ArchiveInputInfo.InMemory("thing.qq", payload)], new FormatCreateOptions { MethodName = "squeeze" }); + + var raw = archive.ToArray(); + raw[0x7D] &= 0x7F; + using var compat = new MemoryStream(raw, writable: false); + + Assert.That(d.ExtractEntryToMemory(compat, "THING.QQ", null), Is.EqualTo(payload)); + } + + [Test] + public void DirectAddReplaceRemovePatchesCountdownAndPreservesOtherPayloads() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ + ArchiveInputInfo.InMemory("one.bin", Enumerable.Repeat((byte)1, 129).ToArray()), + ArchiveInputInfo.InMemory("two.bin", Enumerable.Repeat((byte)2, 260).ToArray()), + ], new FormatCreateOptions { MethodName = "stored" }); + + var originalTwo = d.ExtractEntryToMemory(Reset(archive), "TWO.BIN", null); + d.Add(archive, [ArchiveInputInfo.InMemory("three.bin", Enumerable.Repeat((byte)3, 17).ToArray())]); + + var afterAdd = archive.ToArray(); + Assert.That(afterAdd[0x7F], Is.EqualTo(2)); + var secondHeader = 128 + 256; + Assert.That(afterAdd[secondHeader + 0x7F], Is.EqualTo(1)); + + d.Add(archive, [ArchiveInputInfo.InMemory("one.bin", Enumerable.Repeat((byte)9, 700).ToArray())]); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "ONE.BIN", null), Is.EqualTo(Enumerable.Repeat((byte)9, 700).ToArray())); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "TWO.BIN", null), Is.EqualTo(originalTwo)); + + d.Remove(archive, ["two.bin"]); + var names = d.List(Reset(archive), null).Select(e => e.Name).ToArray(); + Assert.That(names, Is.EqualTo(new[] { "ONE.BIN", "THREE.BIN" })); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "THREE.BIN", null), Is.EqualTo(Enumerable.Repeat((byte)3, 17).ToArray())); + var afterRemove = archive.ToArray(); + Assert.That(afterRemove[0x7F], Is.EqualTo(1)); + } + + [Test] + public void AddSynthesizesMissingParentDirectoryInPlace() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ArchiveInputInfo.InMemory("root.bin", "root"u8)], new FormatCreateOptions()); + + d.Add(archive, [ArchiveInputInfo.InMemory("sub/new.bin", "new"u8)]); + + var entries = d.List(Reset(archive), null); + Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "ROOT.BIN", "SUB", "SUB/NEW.BIN" })); + Assert.That(entries[1].IsDirectory, Is.True); + } + + [Test] + public void RemoveDirectoryRemovesItsDescendants() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ + ArchiveInputInfo.InMemory("keep.bin", "keep"u8), + ArchiveInputInfo.InMemory("tree/a.bin", "a"u8), + ArchiveInputInfo.InMemory("tree/b.bin", "b"u8), + ], new FormatCreateOptions()); + + d.Remove(archive, ["tree"]); + + Assert.That(d.List(Reset(archive), null).Select(e => e.Name), Is.EqualTo(new[] { "KEEP.BIN" })); + } + + [Test] + public void AutoCompressionUsesBlockRoundedSize() { + var d = new BinaryIIFormatDescriptor(); + using var compressible = new MemoryStream(); + d.Create(compressible, [ArchiveInputInfo.InMemory("repeat.bin", Enumerable.Repeat((byte)'A', 4096).ToArray())], + new FormatCreateOptions { MethodName = "auto" }); + Assert.That((compressible.ToArray()[0x7D] & 0x80) != 0, Is.True); + + using var tiny = new MemoryStream(); + d.Create(tiny, [ArchiveInputInfo.InMemory("tiny.bin", new byte[] { 1, 2, 3 })], + new FormatCreateOptions { MethodName = "auto" }); + Assert.That((tiny.ToArray()[0x7D] & 0x80) != 0, Is.False); + } + + [Test] + public void NamesAreNormalizedToProDosPartialPathRules() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ + ArchiveInputInfo.InMemory("123 very-long-directory-name/hello world.txt", "x"u8), + ArchiveInputInfo.InMemory("123 very-long-directory-name/hello world.txt", "y"u8), + ], new FormatCreateOptions()); + + var names = d.List(Reset(archive), null).Select(e => e.Name).ToArray(); + Assert.That(names[0], Does.StartWith("X123.VERY.LONG")); + Assert.That(names[^2], Is.Not.EqualTo(names[^1]).IgnoreCase); + Assert.That(names.All(n => n.Length <= 64), Is.True); + } + + [Test] + public void EncryptionIsRejected() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + Assert.Throws(() => + d.Create(archive, [ArchiveInputInfo.InMemory("x.bin", "x"u8)], new FormatCreateOptions { Password = "secret" })); + } + + [Test] + public void DefragmentTrimsTrailingGarbageButKeepsPayload() { + var d = new BinaryIIFormatDescriptor(); + using var archive = new MemoryStream(); + d.Create(archive, [ArchiveInputInfo.InMemory("x.bin", Enumerable.Repeat((byte)0xA5, 200).ToArray())], new FormatCreateOptions()); + var canonicalLength = archive.Length; + archive.Position = archive.Length; + archive.Write(new byte[321]); + + d.Defragment(archive); + + Assert.That(archive.Length, Is.EqualTo(canonicalLength)); + Assert.That(d.ExtractEntryToMemory(Reset(archive), "X.BIN", null), Is.EqualTo(Enumerable.Repeat((byte)0xA5, 200).ToArray())); + } + + private static MemoryStream Reset(MemoryStream stream) { + stream.Position = 0; + return stream; + } +} diff --git a/CompressionWorkbench.slnx b/CompressionWorkbench.slnx index 6f558c5d6..10bb06389 100644 --- a/CompressionWorkbench.slnx +++ b/CompressionWorkbench.slnx @@ -461,6 +461,7 @@ + diff --git a/FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs b/FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs new file mode 100644 index 000000000..2f2d802cc --- /dev/null +++ b/FileFormats/FileFormat.BinaryII/BinaryIIArchive.cs @@ -0,0 +1,509 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using FileFormat.Squeeze; + +namespace FileFormat.BinaryII; + +internal static class BinaryIIConstants { + public const int HeaderSize = 128; + public const int Alignment = 128; + public const int MaxRecords = 256; + public const int MaxNameLength = 64; + + public const byte ProDosAccessDefault = 0xE3; + public const byte ProDosFileTypeBinary = 0x06; + public const byte ProDosFileTypeDirectory = 0x0F; + public const byte ProDosStorageSeedling = 0x01; + public const byte ProDosStorageSapling = 0x02; + public const byte ProDosStorageTree = 0x03; + public const byte ProDosStorageDirectory = 0x0D; + + public const byte DataFlagSparse = 0x01; + public const byte DataFlagEncrypted = 0x40; + public const byte DataFlagCompressed = 0x80; + + public static int RoundUp128(int value) => checked((value + 127) & ~127); +} + +internal sealed record BinaryIIRecord( + string Name, + bool IsDirectory, + bool IsPhantom, + bool IsCompressed, + bool IsEncrypted, + bool IsSparse, + byte FileType, + byte StorageType, + byte DataFlags, + int StoredLength, + long HeaderOffset, + long DataOffset, + int PhysicalLength, + byte FilesToFollow +); + +internal enum BinaryIICompressionMode { + Stored, + Squeeze, + Auto, +} + +internal sealed record BinaryIIWriteRecord( + string Name, + bool IsDirectory, + byte[] Data, + bool Compress +); + +internal sealed class BinaryIIReader { + private readonly byte[] _data; + private readonly List _records = []; + + public IReadOnlyList PhysicalRecords => this._records; + public IEnumerable Entries => this._records.Where(r => !r.IsPhantom); + + public BinaryIIReader(Stream input) { + ArgumentNullException.ThrowIfNull(input); + if (input.CanSeek) input.Position = 0; + using var ms = new MemoryStream(); + input.CopyTo(ms); + this._data = ms.ToArray(); + this.Parse(); + } + + public byte[] Extract(BinaryIIRecord entry) { + ArgumentNullException.ThrowIfNull(entry); + if (entry.IsDirectory) + return []; + if (entry.IsEncrypted) + throw new NotSupportedException($"Binary II entry '{entry.Name}' is encrypted; the format never standardized an encryption method."); + if (entry.IsSparse) + throw new NotSupportedException($"Binary II entry '{entry.Name}' is marked sparse; the Binary II specification does not define sparse reconstruction semantics."); + + var stored = this._data.AsSpan((int)entry.DataOffset, entry.StoredLength).ToArray(); + if (!entry.IsCompressed) + return stored; + + using var src = new MemoryStream(stored, writable: false); + using var dst = new MemoryStream(); + SqueezeStream.Decompress(src, dst); + return dst.ToArray(); + } + + private void Parse() { + if (this._data.Length == 0) + return; + if (this._data.Length < BinaryIIConstants.HeaderSize) + throw new InvalidDataException("Binary II: archive is shorter than one 128-byte record header."); + + var offset = 0; + for (var recordIndex = 0; recordIndex < BinaryIIConstants.MaxRecords; recordIndex++) { + if (offset + BinaryIIConstants.HeaderSize > this._data.Length) + throw new InvalidDataException("Binary II: truncated record header."); + + var h = this._data.AsSpan(offset, BinaryIIConstants.HeaderSize); + if (h[0] != 0x0A || h[1] != 0x47 || h[2] != 0x4C || h[0x12] != 0x02) + throw new InvalidDataException($"Binary II: invalid record signature at offset 0x{offset:X}."); + + var nameLength = h[0x17]; + if (nameLength > BinaryIIConstants.MaxNameLength) + throw new InvalidDataException($"Binary II: record at 0x{offset:X} has invalid filename length {nameLength}."); + var name = Encoding.ASCII.GetString(h.Slice(0x18, nameLength)).Replace('\\', '/'); + + var lowEof = (uint)(h[0x14] | (h[0x15] << 8) | (h[0x16] << 16)); + var eof = lowEof | ((uint)h[0x74] << 24); + if (eof > int.MaxValue) + throw new InvalidDataException($"Binary II: entry '{name}' is too large for this in-memory reader."); + var storedLength = (int)eof; + var paddedLength = BinaryIIConstants.RoundUp128(storedLength); + var dataOffset = checked(offset + BinaryIIConstants.HeaderSize); + var physicalLength = checked(BinaryIIConstants.HeaderSize + paddedLength); + if ((long)dataOffset + storedLength > this._data.LongLength) + throw new InvalidDataException($"Binary II: entry '{name}' extends beyond end of archive."); + + var fileType = h[0x04]; + var storageType = h[0x07]; + var flags = h[0x7D]; + var phantom = h[0x7C] != 0; + var directory = fileType == BinaryIIConstants.ProDosFileTypeDirectory || storageType == BinaryIIConstants.ProDosStorageDirectory; + var compressed = (flags & BinaryIIConstants.DataFlagCompressed) != 0 + || name.EndsWith(".QQ", StringComparison.OrdinalIgnoreCase); + var encrypted = (flags & BinaryIIConstants.DataFlagEncrypted) != 0; + var sparse = (flags & BinaryIIConstants.DataFlagSparse) != 0; + var follows = h[0x7F]; + + this._records.Add(new BinaryIIRecord( + name, + directory, + phantom, + compressed, + encrypted, + sparse, + fileType, + storageType, + flags, + storedLength, + offset, + dataOffset, + physicalLength, + follows + )); + + var nextOffset = checked(offset + physicalLength); + if (nextOffset + BinaryIIConstants.HeaderSize > this._data.Length) + return; + var next = this._data.AsSpan(nextOffset, BinaryIIConstants.HeaderSize); + if (next[0] != 0x0A || next[1] != 0x47 || next[2] != 0x4C || next[0x12] != 0x02) + return; + offset = nextOffset; + } + + throw new InvalidDataException("Binary II: archive exceeds the 256-record limit imposed by the files-to-follow byte."); + } +} + +internal static class BinaryIIWriter { + public static byte[] Build(IReadOnlyList inputs, BinaryIICompressionMode mode) { + ArgumentNullException.ThrowIfNull(inputs); + var records = PrepareRecords(inputs, mode); + using var ms = new MemoryStream(); + for (var i = 0; i < records.Count; i++) { + var bytes = CreatePhysicalRecord(records[i], records.Count - i - 1); + ms.Write(bytes); + } + return ms.ToArray(); + } + + public static List PrepareRecords( + IReadOnlyList inputs, + BinaryIICompressionMode mode + ) { + var result = new List(); + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var input in inputs.OrderBy(i => i.ArchiveName, StringComparer.OrdinalIgnoreCase)) { + var normalized = NormalizePath(input.ArchiveName); + var parts = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0) + continue; + + var parent = ""; + for (var i = 0; i < parts.Length - (input.IsDirectory ? 0 : 1); i++) { + parent = parent.Length == 0 ? parts[i] : parent + "/" + parts[i]; + if (names.Add(parent)) + result.Add(new BinaryIIWriteRecord(parent, true, [], false)); + } + + if (input.IsDirectory) { + if (names.Add(normalized)) + result.Add(new BinaryIIWriteRecord(normalized, true, [], false)); + continue; + } + + var unique = MakeUnique(normalized, names); + names.Add(unique); + var data = input.ReadContent(); + var compress = ShouldCompress(unique, data, mode); + result.Add(new BinaryIIWriteRecord(unique, false, data, compress)); + } + + if (result.Count > BinaryIIConstants.MaxRecords) + throw new InvalidDataException($"Binary II supports at most {BinaryIIConstants.MaxRecords} physical records."); + return result; + } + + public static byte[] CreatePhysicalRecord(BinaryIIWriteRecord record, int filesToFollow) { + ArgumentNullException.ThrowIfNull(record); + if (filesToFollow is < 0 or > 255) + throw new ArgumentOutOfRangeException(nameof(filesToFollow)); + + var nameBytes = Encoding.ASCII.GetBytes(record.Name); + if (nameBytes.Length is < 1 or > BinaryIIConstants.MaxNameLength) + throw new InvalidDataException($"Binary II filename '{record.Name}' is outside the 1..64 byte range."); + + byte[] stored; + var compressed = false; + if (!record.IsDirectory && record.Compress) { + using var src = new MemoryStream(record.Data, writable: false); + using var dst = new MemoryStream(); + SqueezeStream.Compress(src, dst, Path.GetFileName(record.Name)); + stored = dst.ToArray(); + compressed = true; + } else { + stored = record.IsDirectory ? [] : record.Data; + } + + var padded = BinaryIIConstants.RoundUp128(stored.Length); + var output = new byte[BinaryIIConstants.HeaderSize + padded]; + var h = output.AsSpan(0, BinaryIIConstants.HeaderSize); + + h[0] = 0x0A; + h[1] = 0x47; + h[2] = 0x4C; + h[0x03] = BinaryIIConstants.ProDosAccessDefault; + h[0x04] = record.IsDirectory ? BinaryIIConstants.ProDosFileTypeDirectory : BinaryIIConstants.ProDosFileTypeBinary; + BinaryPrimitives.WriteUInt16LittleEndian(h[0x05..], 0); + h[0x07] = record.IsDirectory ? BinaryIIConstants.ProDosStorageDirectory : StorageTypeForLength(record.Data.Length); + BinaryPrimitives.WriteUInt16LittleEndian(h[0x08..], 0); + h[0x12] = 0x02; + + WriteUInt24LittleEndian(h[0x14..], stored.Length); + h[0x17] = (byte)nameBytes.Length; + nameBytes.AsSpan().CopyTo(h[0x18..]); + h[0x74] = (byte)((uint)stored.Length >> 24); + h[0x79] = 0x00; + h[0x7C] = 0x00; + h[0x7D] = compressed ? BinaryIIConstants.DataFlagCompressed : (byte)0x00; + h[0x7E] = 0x01; + h[0x7F] = (byte)filesToFollow; + + if (stored.Length > 0) + stored.AsSpan().CopyTo(output.AsSpan(BinaryIIConstants.HeaderSize)); + return output; + } + + public static string NormalizePath(string path) { + if (string.IsNullOrWhiteSpace(path)) + return "X"; + + var rawParts = path.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries); + var cooked = new List(rawParts.Length); + foreach (var raw in rawParts) { + if (raw is "." or "..") + continue; + var upper = raw.ToUpperInvariant(); + var sb = new StringBuilder(upper.Length + 1); + foreach (var ch in upper) { + if (ch is >= 'A' and <= 'Z' || ch is >= '0' and <= '9' || ch == '.') + sb.Append(ch); + else + sb.Append('.'); + } + + if (sb.Length == 0 || sb[0] is < 'A' or > 'Z') + sb.Insert(0, 'X'); + if (sb.Length > 15) + sb.Length = 15; + cooked.Add(sb.ToString()); + } + + if (cooked.Count == 0) + return "X"; + + var joined = string.Join('/', cooked); + if (joined.Length <= BinaryIIConstants.MaxNameLength) + return joined; + + joined = joined[..BinaryIIConstants.MaxNameLength].TrimEnd('/'); + return joined.Length == 0 ? "X" : joined; + } + + private static string MakeUnique(string normalized, HashSet existing) { + if (!existing.Contains(normalized)) + return normalized; + + var slash = normalized.LastIndexOf('/'); + var parent = slash >= 0 ? normalized[..(slash + 1)] : ""; + var leaf = slash >= 0 ? normalized[(slash + 1)..] : normalized; + for (var n = 1; n < 10000; n++) { + var suffix = "." + n.ToString(System.Globalization.CultureInfo.InvariantCulture); + var maxLeaf = Math.Min(15, BinaryIIConstants.MaxNameLength - parent.Length); + var prefixLength = Math.Max(1, maxLeaf - suffix.Length); + var candidateLeaf = leaf[..Math.Min(leaf.Length, prefixLength)] + suffix; + var candidate = parent + candidateLeaf; + if (!existing.Contains(candidate)) + return candidate; + } + + throw new InvalidDataException($"Binary II could not disambiguate duplicate path '{normalized}'."); + } + + private static bool ShouldCompress(string name, byte[] data, BinaryIICompressionMode mode) { + if (mode == BinaryIICompressionMode.Stored || data.Length == 0) + return false; + if (mode == BinaryIICompressionMode.Squeeze) + return true; + + using var src = new MemoryStream(data, writable: false); + using var dst = new MemoryStream(); + SqueezeStream.Compress(src, dst, Path.GetFileName(name)); + return BinaryIIConstants.RoundUp128(checked((int)dst.Length)) < BinaryIIConstants.RoundUp128(data.Length); + } + + private static byte StorageTypeForLength(int length) + => length <= 512 ? BinaryIIConstants.ProDosStorageSeedling + : length <= 128 * 1024 ? BinaryIIConstants.ProDosStorageSapling + : BinaryIIConstants.ProDosStorageTree; + + private static void WriteUInt24LittleEndian(Span destination, int value) { + var u = (uint)value; + destination[0] = (byte)u; + destination[1] = (byte)(u >> 8); + destination[2] = (byte)(u >> 16); + } +} + +internal static class BinaryIIInPlaceModifier { + private const int CopyBufferSize = 64 * 1024; + + public static void Add(Stream archive, IReadOnlyList inputs) { + ValidateWritable(archive); + ArgumentNullException.ThrowIfNull(inputs); + + var requested = BinaryIIWriter.PrepareRecords(inputs, BinaryIICompressionMode.Stored); + foreach (var record in requested) { + var reader = new BinaryIIReader(archive); + var existing = reader.PhysicalRecords.FirstOrDefault( + r => !r.IsPhantom && string.Equals(r.Name, record.Name, StringComparison.OrdinalIgnoreCase)); + + if (existing is not null) { + if (existing.IsDirectory && record.IsDirectory) + continue; + if (existing.IsDirectory != record.IsDirectory) + throw new InvalidOperationException($"Binary II cannot replace '{record.Name}' with a different entry kind while descendants may exist."); + ReplaceRecord(archive, existing, BinaryIIWriter.CreatePhysicalRecord(record, existing.FilesToFollow)); + } else { + var physicalCount = reader.PhysicalRecords.Count; + if (physicalCount >= BinaryIIConstants.MaxRecords) + throw new InvalidDataException("Binary II archive already contains the maximum 256 records."); + + var logicalEnd = reader.PhysicalRecords.Count == 0 + ? 0L + : reader.PhysicalRecords[^1].HeaderOffset + reader.PhysicalRecords[^1].PhysicalLength; + if (archive.Length != logicalEnd) + archive.SetLength(logicalEnd); + archive.Position = logicalEnd; + archive.Write(BinaryIIWriter.CreatePhysicalRecord(record, 0)); + } + } + + PatchFilesToFollow(archive); + } + + public static void Remove(Stream archive, string[] entryNames) { + ValidateWritable(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Length == 0) + return; + + var normalized = entryNames + .Where(n => !string.IsNullOrWhiteSpace(n)) + .Select(BinaryIIWriter.NormalizePath) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var reader = new BinaryIIReader(archive); + var removals = reader.PhysicalRecords + .Where(r => !r.IsPhantom && normalized.Any(n => + string.Equals(r.Name, n, StringComparison.OrdinalIgnoreCase) + || r.Name.StartsWith(n + "/", StringComparison.OrdinalIgnoreCase))) + .OrderByDescending(r => r.HeaderOffset) + .ToList(); + + foreach (var record in removals) + RemoveRange(archive, record.HeaderOffset, record.PhysicalLength); + + PatchFilesToFollow(archive); + } + + public static void Defragment(Stream archive) { + ValidateWritable(archive); + var reader = new BinaryIIReader(archive); + if (reader.PhysicalRecords.Count == 0) { + archive.SetLength(0); + return; + } + + foreach (var record in reader.PhysicalRecords) { + var payloadEnd = record.DataOffset + record.StoredLength; + var recordEnd = record.HeaderOffset + record.PhysicalLength; + if (payloadEnd < recordEnd) { + archive.Position = payloadEnd; + WriteZeros(archive, checked((int)(recordEnd - payloadEnd))); + } + } + + var end = reader.PhysicalRecords[^1].HeaderOffset + reader.PhysicalRecords[^1].PhysicalLength; + archive.SetLength(end); + PatchFilesToFollow(archive); + } + + private static void ReplaceRecord(Stream archive, BinaryIIRecord oldRecord, byte[] replacement) { + var delta = replacement.LongLength - oldRecord.PhysicalLength; + var tailStart = oldRecord.HeaderOffset + oldRecord.PhysicalLength; + ShiftTail(archive, tailStart, delta); + archive.Position = oldRecord.HeaderOffset; + archive.Write(replacement); + } + + private static void RemoveRange(Stream archive, long offset, long length) + => ShiftTail(archive, offset + length, -length); + + private static void ShiftTail(Stream archive, long tailStart, long delta) { + if (delta == 0) + return; + + var oldLength = archive.Length; + if (tailStart < 0 || tailStart > oldLength) + throw new ArgumentOutOfRangeException(nameof(tailStart)); + + var buffer = new byte[CopyBufferSize]; + if (delta > 0) { + archive.SetLength(checked(oldLength + delta)); + var remaining = oldLength - tailStart; + while (remaining > 0) { + var chunk = (int)Math.Min(buffer.Length, remaining); + var readPos = tailStart + remaining - chunk; + archive.Position = readPos; + archive.ReadExactly(buffer.AsSpan(0, chunk)); + archive.Position = readPos + delta; + archive.Write(buffer, 0, chunk); + remaining -= chunk; + } + } else { + var shift = -delta; + var readPos = tailStart; + var writePos = tailStart - shift; + while (readPos < oldLength) { + var chunk = (int)Math.Min(buffer.Length, oldLength - readPos); + archive.Position = readPos; + archive.ReadExactly(buffer.AsSpan(0, chunk)); + archive.Position = writePos; + archive.Write(buffer, 0, chunk); + readPos += chunk; + writePos += chunk; + } + archive.SetLength(checked(oldLength - shift)); + } + } + + private static void PatchFilesToFollow(Stream archive) { + var reader = new BinaryIIReader(archive); + if (reader.PhysicalRecords.Count > BinaryIIConstants.MaxRecords) + throw new InvalidDataException("Binary II archive contains more than 256 physical records."); + + for (var i = 0; i < reader.PhysicalRecords.Count; i++) { + archive.Position = reader.PhysicalRecords[i].HeaderOffset + 0x7F; + archive.WriteByte((byte)(reader.PhysicalRecords.Count - i - 1)); + } + if (archive.CanSeek) + archive.Position = 0; + } + + private static void ValidateWritable(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Binary II direct modification requires a readable, writable, seekable stream.", nameof(archive)); + } + + private static void WriteZeros(Stream output, int count) { + Span zeros = stackalloc byte[128]; + zeros.Clear(); + while (count > 0) { + var n = Math.Min(count, zeros.Length); + output.Write(zeros[..n]); + count -= n; + } + } +} diff --git a/FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs b/FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs new file mode 100644 index 000000000..9e7b3a1e3 --- /dev/null +++ b/FileFormats/FileFormat.BinaryII/BinaryIIFormatDescriptor.cs @@ -0,0 +1,164 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.BinaryII; + +/// +/// Apple II Binary II / BLU archive (.bny/.bqy). +/// +/// +/// Binary II is a deliberately simple record stream: every member is a 128-byte +/// metadata header followed by its payload rounded to a 128-byte boundary. +/// Version-1 headers preserve ProDOS/GS/OS metadata and mark Squeeze-compressed +/// payloads with data flag 0x80. Historical BLU extractors also infer Squeeze +/// from a .QQ member suffix, which this reader accepts for compatibility. +/// +public sealed class BinaryIIFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveCreatable, + IArchiveModifiable, + IArchiveDefragmentable, + IArchiveLayoutMap { + + public string Id => "BinaryII"; + public string DisplayName => "Apple II Binary II"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | + FormatCapabilities.CanExtract | + FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | + FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | + FormatCapabilities.SupportsDirectories; + public string DefaultExtension => ".bny"; + public IReadOnlyList Extensions => [".bny", ".bqy"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => + [new([0x0A, 0x47, 0x4C], Confidence: 0.99)]; + public IReadOnlyList Methods => [ + new("stored", "Stored"), + new("squeeze", "Squeeze"), + new("auto", "Auto", true), + ]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => + "Apple II Binary II / BLU record archive with stored or Squeeze-compressed members and direct 128-byte-record mutation"; + + public List List(Stream stream, string? password) { + ArgumentNullException.ThrowIfNull(stream); + var reader = new BinaryIIReader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + Index: index, + Name: entry.Name, + OriginalSize: entry.IsDirectory ? 0 : entry.IsCompressed ? -1 : entry.StoredLength, + CompressedSize: entry.StoredLength, + Method: entry.IsCompressed ? "Squeeze" : "Stored", + IsDirectory: entry.IsDirectory, + IsEncrypted: entry.IsEncrypted, + LastModified: null, + Kind: entry.IsPhantom ? "phantom" : null + )).ToList(); + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(outputDir); + var reader = new BinaryIIReader(stream); + foreach (var entry in reader.Entries) { + if (files is not null && !MatchesFilter(entry.Name, files)) + continue; + if (entry.IsDirectory) { + CreateSafeDirectory(outputDir, entry.Name); + continue; + } + WriteFile(outputDir, entry.Name, reader.Extract(entry)); + } + } + + public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryName); + var reader = new BinaryIIReader(archive); + var entry = reader.Entries.FirstOrDefault(e => + string.Equals(e.Name, entryName, StringComparison.OrdinalIgnoreCase) + || string.Equals(Path.GetFileName(e.Name), entryName, StringComparison.OrdinalIgnoreCase)); + if (entry is null) + return []; + return reader.Extract(entry); + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (!output.CanWrite) + throw new ArgumentException("Output stream is not writable.", nameof(output)); + if (!string.IsNullOrEmpty(options.Password) || options.EncryptFilenames || !string.IsNullOrEmpty(options.EncryptionMethod)) + throw new NotSupportedException("Binary II does not define an interoperable encryption method."); + + var method = (options.MethodName ?? "stored").Trim().ToLowerInvariant(); + var mode = method switch { + "" or "stored" or "store" => BinaryIICompressionMode.Stored, + "squeeze" or "sq" => BinaryIICompressionMode.Squeeze, + "auto" => BinaryIICompressionMode.Auto, + _ => throw new NotSupportedException($"Binary II compression method '{options.MethodName}' is not supported."), + }; + + var bytes = BinaryIIWriter.Build(inputs, mode); + if (output.CanSeek) { + output.Position = 0; + output.SetLength(0); + } + output.Write(bytes); + } + + /// + /// Adds new entries or replaces same-name entries by editing the existing + /// 128-byte record stream in place. Existing payloads are not decoded/re-encoded. + /// + public void Add(Stream archive, IReadOnlyList inputs) + => BinaryIIInPlaceModifier.Add(archive, inputs); + + /// + /// Removes named entries (and descendants of named directory entries) by + /// shifting the following record tail left and truncating the stream. + /// + public void Remove(Stream archive, string[] entryNames) + => BinaryIIInPlaceModifier.Remove(archive, entryNames); + + public void Defragment(Stream archive) + => BinaryIIInPlaceModifier.Defragment(archive); + + public void Defragment(Stream archive, DefragOptions options) + => BinaryIIInPlaceModifier.Defragment(archive); + + public IEnumerable EnumerateLayout(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + var reader = new BinaryIIReader(archive); + foreach (var entry in reader.PhysicalRecords) { + yield return new DefragBlockInfo(entry.HeaderOffset, BinaryIIConstants.HeaderSize, DefragBlockKind.MetadataReserved, entry.Name + " header"); + if (entry.StoredLength > 0) + yield return new DefragBlockInfo(entry.DataOffset, entry.StoredLength, DefragBlockKind.Used, entry.Name); + var padding = entry.PhysicalLength - BinaryIIConstants.HeaderSize - entry.StoredLength; + if (padding > 0) + yield return new DefragBlockInfo(entry.DataOffset + entry.StoredLength, padding, DefragBlockKind.Free, entry.Name + " padding"); + } + } + + private static void CreateSafeDirectory(string baseDir, string entryName) { + var parts = entryName.Replace('\\', '/') + .Split('/', StringSplitOptions.RemoveEmptyEntries) + .Where(p => p is not "." and not "..") + .ToArray(); + if (parts.Length == 0) + return; + var all = new string[parts.Length + 1]; + all[0] = baseDir; + Array.Copy(parts, 0, all, 1, parts.Length); + Directory.CreateDirectory(Path.Combine(all)); + } +} diff --git a/FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj b/FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj new file mode 100644 index 000000000..d5fec84dc --- /dev/null +++ b/FileFormats/FileFormat.BinaryII/FileFormat.BinaryII.csproj @@ -0,0 +1,9 @@ + + + FileFormat.BinaryII + + + + + + diff --git a/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj b/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj index 78cc0e257..5f0da11fc 100644 --- a/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj +++ b/Hawkynt.FileFormats.Archives/Hawkynt.FileFormats.Archives.csproj @@ -68,6 +68,7 @@ + diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 6df766d07..29f31f861 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -615,7 +615,7 @@ This package is built against the repository's shared Core version. Consume a mu -Every public and protected member of all 968 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). +Every public and protected member of all 969 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 713901e72..60c18c857 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -3229,6 +3229,41 @@ Encodes files into BinHex 4.0 (.hqx) text format. | `BinHexWriter` | `BinHexWriter()` | | | `Write` | `static void Write(Stream output, string fileName, byte[] dataFork, byte[] resourceFork = null, string fileType = "TEXT", string fileCreator = "ttxt")` | Writes a BinHex 4.0 encoded file to the output stream. | +### Namespace `FileFormat.BinaryII` + +[`BinaryIIFormatDescriptor`](#binaryiiformatdescriptor) + +#### `BinaryIIFormatDescriptor` + +Apple II Binary II / BLU archive (.bny/.bqy). + +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `BinaryIIFormatDescriptor` | `BinaryIIFormatDescriptor()` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds new entries or replaces same-name entries by editing the existing 128-byte record stream in place. Existing payloads are not decoded/re-encoded. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | +| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes named entries (and descendants of named directory entries) by shifting the following record tail left and truncating the stream. | + ### Namespace `FileFormat.Bkf` [`BkfEntry`](#bkfentry) · [`BkfFormatDescriptor`](#bkfformatdescriptor) · [`BkfInPlaceModifier`](#bkfinplacemodifier) · [`BkfReader`](#bkfreader) · [`BkfWriter`](#bkfwriter) · [`BkfWriter.Item`](#bkfwriteritem)