From ba838f3b6de4b95953dba4ad7e93d3a294497e56 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:06:10 +0200 Subject: [PATCH 001/192] + preserve exact DMG partition lengths --- FileFormats/FileFormat.Dmg/DmgWriter.cs | 155 +++++++++++------------- 1 file changed, 74 insertions(+), 81 deletions(-) diff --git a/FileFormats/FileFormat.Dmg/DmgWriter.cs b/FileFormats/FileFormat.Dmg/DmgWriter.cs index b7dc09225..9665e5c96 100644 --- a/FileFormats/FileFormat.Dmg/DmgWriter.cs +++ b/FileFormats/FileFormat.Dmg/DmgWriter.cs @@ -5,23 +5,18 @@ namespace FileFormat.Dmg; /// -/// Writes Apple Disk Image (DMG) files in WORM mode. Each input file becomes -/// one partition with a single raw (uncompressed) mish block. The output -/// roundtrips through : -/// -/// Layout: [partition data sectors] [XML plist] [512-byte koly trailer]. -/// Each partition has a mish table with one BlockTypeRaw entry covering all its sectors plus a terminator. -/// No compression -- DMG's zlib/bz2/lzfse encoders aren't paired here, and raw is fully spec-valid. -/// No checksums -- mish/koly checksum-type fields set to 0 ("none"), which the reader accepts. -/// +/// Writes Apple Disk Image (DMG/UDIF) files using raw mish blocks. +/// Each input becomes one partition and the exact caller-visible byte length is +/// carried in a private plist key so non-sector-aligned inputs round-trip without +/// exposing the mandatory 512-byte UDIF sector padding. /// public sealed class DmgWriter { - private const int SectorSize = 512; - private const int KolySize = 512; - private const int MishHeaderSize = 204; - private const int MishBlockSize = 40; - private const uint BlockTypeRaw = 0x00000001; - private const uint BlockTypeTerminator = 0xFFFFFFFF; + internal const int SectorSize = 512; + internal const int KolySize = 512; + internal const int MishHeaderSize = 204; + internal const int MishBlockSize = 40; + internal const uint BlockTypeRaw = 0x00000001; + internal const uint BlockTypeTerminator = 0xFFFFFFFF; private static readonly byte[] KolyMagic = "koly"u8.ToArray(); private static readonly byte[] MishMagic = "mish"u8.ToArray(); @@ -38,17 +33,15 @@ public void AddPartition(string name, byte[] data) { public void WriteTo(Stream output) { ArgumentNullException.ThrowIfNull(output); - // Pad each partition to a sector boundary so sectorCount is exact. - var padded = new (string name, byte[] data)[_partitions.Count]; + var padded = new (string name, byte[] data, long logicalSize)[_partitions.Count]; for (var i = 0; i < _partitions.Count; i++) { var (name, data) = _partitions[i]; - var paddedLen = ((data.Length + SectorSize - 1) / SectorSize) * SectorSize; + var paddedLen = AlignSector(data.Length); var buf = new byte[paddedLen]; data.CopyTo(buf, 0); - padded[i] = (name, buf); + padded[i] = (name, buf, data.LongLength); } - // Compute layout: partition data sequentially from offset 0. var partitionOffsets = new long[padded.Length]; long pos = 0; for (var i = 0; i < padded.Length; i++) { @@ -57,7 +50,6 @@ public void WriteTo(Stream output) { } var dataForkLength = pos; - // Build mish blob per partition (raw block + terminator). var mishBlobs = new byte[padded.Length][]; for (var i = 0; i < padded.Length; i++) { var sectorCount = (ulong)(padded[i].data.Length / SectorSize); @@ -68,85 +60,90 @@ public void WriteTo(Stream output) { rawDataLength: (ulong)padded[i].data.Length); } - // Build XML plist after data fork. var xml = BuildXmlPlist(padded, mishBlobs); var xmlBytes = Encoding.UTF8.GetBytes(xml); var xmlOffset = pos; - pos += xmlBytes.Length; - // ---- Write data fork ---- - foreach (var (_, data) in padded) + foreach (var (_, data, _) in padded) output.Write(data); - - // ---- Write XML plist ---- output.Write(xmlBytes); - // ---- Write koly trailer ---- - Span koly = stackalloc byte[KolySize]; - koly.Clear(); - KolyMagic.CopyTo(koly); - BinaryPrimitives.WriteUInt32BigEndian(koly[4..], 4); // version - BinaryPrimitives.WriteUInt32BigEndian(koly[8..], KolySize); // header size - BinaryPrimitives.WriteUInt32BigEndian(koly[12..], 1); // flags - BinaryPrimitives.WriteUInt64BigEndian(koly[16..], 0); // running data fork offset - BinaryPrimitives.WriteUInt64BigEndian(koly[24..], 0); // data fork offset (always 0 for unsegmented) - BinaryPrimitives.WriteUInt64BigEndian(koly[32..], (ulong)dataForkLength); - BinaryPrimitives.WriteUInt64BigEndian(koly[40..], 0); // resource fork offset - BinaryPrimitives.WriteUInt64BigEndian(koly[48..], 0); // resource fork length - BinaryPrimitives.WriteUInt32BigEndian(koly[56..], 1); // segment number - BinaryPrimitives.WriteUInt32BigEndian(koly[60..], 1); // segment count - // Segment GUID (16 bytes at 64): leave zero - BinaryPrimitives.WriteUInt32BigEndian(koly[80..], 0); // data checksum type = none - BinaryPrimitives.WriteUInt32BigEndian(koly[84..], 0); // data checksum size = 0 - // 128 bytes data checksum at 88: zeros - BinaryPrimitives.WriteUInt64BigEndian(koly[216..], (ulong)xmlOffset); - BinaryPrimitives.WriteUInt64BigEndian(koly[224..], (ulong)xmlBytes.Length); - // 120 bytes reserved at 232: zeros - BinaryPrimitives.WriteUInt32BigEndian(koly[352..], 0); // master checksum type = none - BinaryPrimitives.WriteUInt32BigEndian(koly[356..], 0); // master checksum size = 0 - // 128 bytes master checksum at 360: zeros - BinaryPrimitives.WriteUInt32BigEndian(koly[488..], 1); // image variant - var totalSectors = (ulong)(dataForkLength / SectorSize); - BinaryPrimitives.WriteUInt64BigEndian(koly[492..], totalSectors); // sector count - output.Write(koly); + var totalSectors = padded.Aggregate<(string name, byte[] data, long logicalSize), ulong>(0, + (current, partition) => current + (ulong)(partition.data.Length / SectorSize)); + output.Write(BuildKoly(xmlOffset, xmlBytes.LongLength, dataForkLength, totalSectors)); } - // ── Mish blob ───────────────────────────────────────────────────────────── + internal static int AlignSector(int length) { + if (length < 0) throw new ArgumentOutOfRangeException(nameof(length)); + return checked((length + SectorSize - 1) / SectorSize * SectorSize); + } - private static byte[] BuildMishBlob(ulong firstSector, ulong sectorCount, + internal static byte[] BuildMishBlob(ulong firstSector, ulong sectorCount, ulong rawDataOffset, ulong rawDataLength) { - // Two block entries: one raw covering all sectors, plus terminator. var blob = new byte[MishHeaderSize + 2 * MishBlockSize]; MishMagic.CopyTo(blob, 0); - BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(4), 1); // version + BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(4), 1); BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(8), firstSector); BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(16), sectorCount); - BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(24), 0); // dataStart (unused by reader) - BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(32), (uint)SectorSize); // decompressedBufferRequested - BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(36), 0); // blocksDescriptor - // 24 reserved + 4 checksumType + 4 checksumSize + 128 checksum data = zeros - BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(200), 2); // numBlockEntries + BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(24), 0); + BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(32), SectorSize); + BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(36), 0); + BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(200), 2); - // Block 0: raw var off = MishHeaderSize; - BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(off + 0), BlockTypeRaw); - // 4 reserved bytes - BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(off + 8), 0); // sectorOffset (within partition) + BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(off), BlockTypeRaw); + BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(off + 8), 0); BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(off + 16), sectorCount); - BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(off + 24), rawDataOffset); // absolute file offset + BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(off + 24), rawDataOffset); BinaryPrimitives.WriteUInt64BigEndian(blob.AsSpan(off + 32), rawDataLength); - // Block 1: terminator off += MishBlockSize; - BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(off + 0), BlockTypeTerminator); - + BinaryPrimitives.WriteUInt32BigEndian(blob.AsSpan(off), BlockTypeTerminator); return blob; } - // ── XML plist ───────────────────────────────────────────────────────────── + internal static string BuildBlkxDict(string name, byte[] mish, long logicalSize) { + var sb = new StringBuilder(); + sb.AppendLine(" "); + sb.Append(" Name").Append(EscapeXml(name)).AppendLine(""); + sb.Append(" CWBLogicalSize").Append(logicalSize).AppendLine(""); + sb.Append(" Data").Append(Convert.ToBase64String(mish)).AppendLine(""); + sb.Append(" "); + return sb.ToString(); + } + + internal static byte[] BuildKoly(long xmlOffset, long xmlLength, long dataForkLength, ulong totalSectors, + byte[]? template = null) { + if (xmlOffset < 0 || xmlLength < 0 || dataForkLength < 0) + throw new ArgumentOutOfRangeException(nameof(xmlOffset)); + + var koly = template is { Length: KolySize } ? (byte[])template.Clone() : new byte[KolySize]; + KolyMagic.CopyTo(koly, 0); + BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(4), 4); + BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(8), KolySize); + if (template == null) { + BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(12), 1); + BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(16), 0); + BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(24), 0); + BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(56), 1); + BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(60), 1); + BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(488), 1); + } + + BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(32), (ulong)dataForkLength); + BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(216), (ulong)xmlOffset); + BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(224), (ulong)xmlLength); + BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(492), totalSectors); - private static string BuildXmlPlist((string name, byte[] data)[] partitions, byte[][] mishBlobs) { + // The plist and/or data fork changed. A checksum of type "none" is valid + // UDIF and avoids retaining a checksum that now describes stale bytes. + koly.AsSpan(80, 136).Clear(); + koly.AsSpan(352, 136).Clear(); + return koly; + } + + private static string BuildXmlPlist((string name, byte[] data, long logicalSize)[] partitions, byte[][] mishBlobs) { var sb = new StringBuilder(); sb.AppendLine(""); sb.AppendLine(""); @@ -156,12 +153,8 @@ private static string BuildXmlPlist((string name, byte[] data)[] partitions, byt sb.AppendLine(" "); sb.AppendLine(" blkx"); sb.AppendLine(" "); - for (var i = 0; i < partitions.Length; i++) { - sb.AppendLine(" "); - sb.Append(" Name").Append(EscapeXml(partitions[i].name)).AppendLine(""); - sb.Append(" Data").Append(Convert.ToBase64String(mishBlobs[i])).AppendLine(""); - sb.AppendLine(" "); - } + for (var i = 0; i < partitions.Length; i++) + sb.AppendLine(BuildBlkxDict(partitions[i].name, mishBlobs[i], partitions[i].logicalSize)); sb.AppendLine(" "); sb.AppendLine(" "); sb.AppendLine(""); @@ -169,7 +162,7 @@ private static string BuildXmlPlist((string name, byte[] data)[] partitions, byt return sb.ToString(); } - private static string EscapeXml(string s) { + internal static string EscapeXml(string s) { return s.Replace("&", "&").Replace("<", "<").Replace(">", ">") .Replace("\"", """).Replace("'", "'"); } From 5fa3cd3b156c01e8ec7ec6ecdfd9e526dbc7a7f6 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:06:45 +0200 Subject: [PATCH 002/192] * teach DMG reader exact logical partition sizes --- FileFormats/FileFormat.Dmg/DmgReader.cs | 262 ++++++++---------------- 1 file changed, 87 insertions(+), 175 deletions(-) diff --git a/FileFormats/FileFormat.Dmg/DmgReader.cs b/FileFormats/FileFormat.Dmg/DmgReader.cs index 91078aaa8..34ea682d8 100644 --- a/FileFormats/FileFormat.Dmg/DmgReader.cs +++ b/FileFormats/FileFormat.Dmg/DmgReader.cs @@ -1,18 +1,16 @@ #pragma warning disable CS1591 using System.Buffers.Binary; using System.IO.Compression; +using System.Net; using System.Text; namespace FileFormat.Dmg; - /// -/// Read-only reader for Apple Disk Image (DMG) files. -/// Parses the koly trailer, XML plist, and mish block tables to expose each -/// partition as an extractable entry. +/// Reader for Apple Disk Image (DMG/UDIF) files. Parses the koly trailer, +/// XML plist, and mish block tables to expose each partition as an entry. /// public sealed class DmgReader : IDisposable { - // Block types in the mish block table private const uint BlockTypeZeroFill = 0x00000000; private const uint BlockTypeRaw = 0x00000001; private const uint BlockTypeZlib = 0x80000005; @@ -32,86 +30,49 @@ public sealed class DmgReader : IDisposable { /// All partitions found in the DMG, each exposed as a named entry. public IReadOnlyList Entries => _entries; + internal long XmlOffset { get; private set; } + internal long XmlLength { get; private set; } + internal byte[] KolyTrailer { get; private set; } = []; + internal IReadOnlyList Partitions => _partitions; + public DmgReader(Stream stream, bool leaveOpen = false) { + ArgumentNullException.ThrowIfNull(stream); using var ms = new MemoryStream(); stream.CopyTo(ms); _data = ms.ToArray(); Parse(); + _ = leaveOpen; } - // ────────────────────────────────────────────────────────────────────────── - // Parsing - // ────────────────────────────────────────────────────────────────────────── - private void Parse() { if (_data.Length < KolySize) throw new InvalidDataException("DMG: file too small to contain koly trailer."); var kolyOff = _data.Length - KolySize; var kolySpan = _data.AsSpan(kolyOff, KolySize); - - // Check "koly" signature - if (kolySpan[0] != 'k' || kolySpan[1] != 'o' || kolySpan[2] != 'l' || kolySpan[3] != 'y') + if (!kolySpan[..4].SequenceEqual("koly"u8)) throw new InvalidDataException("DMG: missing 'koly' trailer signature."); - // Parse koly fields (all big-endian) - // Offset 4: uint32 version - // Offset 8: uint32 headerSize - // Offset 12: uint32 flags - // Offset 16: uint64 runningDataForkOffset - // Offset 24: uint64 dataForkOffset - // Offset 32: uint64 dataForkLength - // Offset 40: uint64 rsrcForkOffset - // Offset 48: uint64 rsrcForkLength - // Offset 56: uint32 segmentNumber - // Offset 60: uint32 segmentCount - // Offset 64: Guid segmentId (16 bytes) - // Offset 80: uint32 dataChecksumType - // Offset 84: uint32 dataChecksumSize - // Offset 88: uint32[32] dataChecksum (128 bytes) - // Offset 216: uint64 xmlOffset - // Offset 224: uint64 xmlLength - // Offset 232: reserved (120 bytes, brings us to 352) - // Offset 352: uint32 masterChecksumType - // Offset 356: uint32 masterChecksumSize - // Offset 360: uint32[32] masterChecksum (128 bytes) - // Offset 488: uint32 imageVariant - // Offset 492: uint64 sectorCount - // Offset 500: 12 bytes reserved - - var xmlOffset = (long)BinaryPrimitives.ReadUInt64BigEndian(kolySpan[216..]); - var xmlLength = (long)BinaryPrimitives.ReadUInt64BigEndian(kolySpan[224..]); - - if (xmlLength <= 0 || xmlOffset < 0 || xmlOffset + xmlLength > _data.Length) + XmlOffset = checked((long)BinaryPrimitives.ReadUInt64BigEndian(kolySpan[216..])); + XmlLength = checked((long)BinaryPrimitives.ReadUInt64BigEndian(kolySpan[224..])); + KolyTrailer = kolySpan.ToArray(); + + if (XmlLength <= 0 || XmlOffset < 0 || XmlOffset + XmlLength > kolyOff) throw new InvalidDataException("DMG: invalid XML plist region in koly trailer."); - var xmlText = Encoding.UTF8.GetString(_data, (int)xmlOffset, (int)xmlLength); + var xmlText = Encoding.UTF8.GetString(_data, (int)XmlOffset, (int)XmlLength); ParseXmlPlist(xmlText); } private void ParseXmlPlist(string xml) { - // We use simple string search — no XML parser dependency. - // Structure we're looking for (may repeat for multiple partitions): - // - // blkx - // - // - // Name - // DataBASE64… - // - // … - // - var blkxPos = xml.IndexOf("blkx", StringComparison.Ordinal); - if (blkxPos < 0) return; // no partitions + if (blkxPos < 0) return; var arrayStart = xml.IndexOf("", blkxPos, StringComparison.Ordinal); - var arrayEnd = xml.IndexOf("", blkxPos, StringComparison.Ordinal); + var arrayEnd = xml.IndexOf("", blkxPos, StringComparison.Ordinal); if (arrayStart < 0 || arrayEnd < 0 || arrayEnd <= arrayStart) return; var arrayBody = xml.Substring(arrayStart + 7, arrayEnd - arrayStart - 7); - - // Parse each element var dictStart = 0; var partIndex = 0; while (true) { @@ -121,11 +82,14 @@ private void ParseXmlPlist(string xml) { if (dEnd < 0) break; var dictBody = arrayBody.Substring(dStart + 6, dEnd - dStart - 6); - var (name, mish) = ParseBlkxDict(dictBody, partIndex); - if (mish != null) { - var size = ComputePartitionSize(mish); - _entries.Add(new DmgEntry { Name = name, Size = size }); - _partitions.Add(new PartitionInfo(name, mish)); + var parsed = ParseBlkxDict(dictBody, partIndex); + if (parsed.Mish != null) { + var physicalSize = ComputePartitionSize(parsed.Mish); + var logicalSize = parsed.LogicalSize is >= 0 and <= long.MaxValue + ? Math.Min(parsed.LogicalSize.Value, physicalSize) + : physicalSize; + _entries.Add(new DmgEntry { Name = parsed.Name, Size = logicalSize }); + _partitions.Add(new PartitionInfo(parsed.Name, parsed.Mish, logicalSize)); partIndex++; } @@ -133,28 +97,35 @@ private void ParseXmlPlist(string xml) { } } - private static (string name, byte[]? mish) ParseBlkxDict(string dictBody, int index) { - // Extract Name + private static (string Name, byte[]? Mish, long? LogicalSize) ParseBlkxDict(string dictBody, int index) { var name = $"partition_{index}.img"; var nameKeyPos = dictBody.IndexOf("Name", StringComparison.Ordinal); if (nameKeyPos >= 0) { var strStart = dictBody.IndexOf("", nameKeyPos, StringComparison.Ordinal); - var strEnd = dictBody.IndexOf("", nameKeyPos, StringComparison.Ordinal); + var strEnd = dictBody.IndexOf("", nameKeyPos, StringComparison.Ordinal); if (strStart >= 0 && strEnd > strStart) { - var raw = dictBody.Substring(strStart + 8, strEnd - strStart - 8).Trim(); - if (raw.Length > 0) - name = SanitizeName(raw, index); + var raw = WebUtility.HtmlDecode(dictBody.Substring(strStart + 8, strEnd - strStart - 8).Trim()); + if (raw.Length > 0) name = SanitizeName(raw, index); } } - // Extract DataBASE64 + long? logicalSize = null; + var logicalKey = dictBody.IndexOf("CWBLogicalSize", StringComparison.Ordinal); + if (logicalKey >= 0) { + var valueStart = dictBody.IndexOf("", logicalKey, StringComparison.Ordinal); + var valueEnd = dictBody.IndexOf("", logicalKey, StringComparison.Ordinal); + if (valueStart >= 0 && valueEnd > valueStart && + long.TryParse(dictBody.AsSpan(valueStart + 9, valueEnd - valueStart - 9), out var parsed) && parsed >= 0) + logicalSize = parsed; + } + byte[]? mish = null; var dataKeyPos = dictBody.IndexOf("Data", StringComparison.Ordinal); if (dataKeyPos < 0) - dataKeyPos = dictBody.IndexOf("data", StringComparison.Ordinal); // lowercase fallback + dataKeyPos = dictBody.IndexOf("data", StringComparison.Ordinal); if (dataKeyPos >= 0) { var dataStart = dictBody.IndexOf("", dataKeyPos, StringComparison.Ordinal); - var dataEnd = dictBody.IndexOf("", dataKeyPos, StringComparison.Ordinal); + var dataEnd = dictBody.IndexOf("", dataKeyPos, StringComparison.Ordinal); if (dataStart >= 0 && dataEnd > dataStart) { var b64 = dictBody.Substring(dataStart + 6, dataEnd - dataStart - 6) .Replace("\n", "").Replace("\r", "").Replace(" ", "").Replace("\t", ""); @@ -162,87 +133,55 @@ private static (string name, byte[]? mish) ParseBlkxDict(string dictBody, int in } } - return (name, mish); + return (name, mish, logicalSize); } private static string SanitizeName(string raw, int index) { - // Strip common Apple partition decorators like "(Apple_HFS : 2)" var paren = raw.IndexOf('('); if (paren > 0) raw = raw[..paren].Trim(); - // Replace characters that are bad in filenames - foreach (var ch in Path.GetInvalidFileNameChars()) - raw = raw.Replace(ch, '_'); + foreach (var ch in Path.GetInvalidFileNameChars()) raw = raw.Replace(ch, '_'); raw = raw.Trim().Replace(' ', '_'); if (raw.Length == 0) raw = $"partition_{index}"; if (!raw.Contains('.')) raw += ".img"; return raw; } - // ────────────────────────────────────────────────────────────────────────── - // Mish (block table) parsing - // ────────────────────────────────────────────────────────────────────────── - - private sealed record BlockEntry(uint Type, ulong SectorOffset, ulong SectorCount, - ulong CompressedOffset, ulong CompressedLength); - - private sealed record MishTable(ulong FirstSector, ulong SectorCount, ulong DataStart, - List Blocks); - - private static MishTable? ParseMish(byte[] mish) { - if (mish == null || mish.Length < 204) return null; + internal sealed record BlockEntry(uint Type, ulong SectorOffset, ulong SectorCount, + ulong CompressedOffset, ulong CompressedLength); - // "mish" signature - if (mish[0] != 'm' || mish[1] != 'i' || mish[2] != 's' || mish[3] != 'h') return null; + internal sealed record MishTable(ulong FirstSector, ulong SectorCount, ulong DataStart, + List Blocks); - // All big-endian - // Offset 4: uint32 version - // Offset 8: uint64 firstSector - // Offset 16: uint64 sectorCount - // Offset 24: uint64 dataStart - // Offset 32: uint32 decompressedBufferRequested - // Offset 36: uint32 blocksDescriptor - // Offset 40: reserved 24 bytes - // Offset 64: checksum type/size/data (136 bytes total: 4+4+128) - // Offset 200: uint32 numBlockEntries - // Then block entries at offset 204, each 40 bytes + internal static MishTable? ParseMish(byte[] mish) { + if (mish.Length < 204 || !mish.AsSpan(0, 4).SequenceEqual("mish"u8)) return null; - var firstSector = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(8)); - var sectorCount = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(16)); - var dataStart = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(24)); - var numEntries = BinaryPrimitives.ReadUInt32BigEndian(mish.AsSpan(200)); + var firstSector = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(8)); + var sectorCount = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(16)); + var dataStart = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(24)); + var numEntries = BinaryPrimitives.ReadUInt32BigEndian(mish.AsSpan(200)); + if (numEntries > 100_000) return null; - if (numEntries > 100_000) return null; // sanity guard var blocks = new List((int)numEntries); - var off = 204; for (var i = 0u; i < numEntries; i++) { if (off + 40 > mish.Length) break; - var blockType = BinaryPrimitives.ReadUInt32BigEndian(mish.AsSpan(off)); - // offset 4 = uint32 reserved - var sectorOffset = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 8)); - var blockSectorCount = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 16)); - var compressedOffset = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 24)); - var compressedLength = BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 32)); - blocks.Add(new BlockEntry(blockType, sectorOffset, blockSectorCount, compressedOffset, compressedLength)); + blocks.Add(new BlockEntry( + BinaryPrimitives.ReadUInt32BigEndian(mish.AsSpan(off)), + BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 8)), + BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 16)), + BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 24)), + BinaryPrimitives.ReadUInt64BigEndian(mish.AsSpan(off + 32)))); off += 40; } - return new MishTable(firstSector, sectorCount, dataStart, blocks); } private static long ComputePartitionSize(byte[] mish) { var table = ParseMish(mish); - if (table == null) return 0; - return (long)table.SectorCount * SectorSize; + return table == null ? 0 : checked((long)table.SectorCount * SectorSize); } - // ────────────────────────────────────────────────────────────────────────── - // Extraction - // ────────────────────────────────────────────────────────────────────────── - - /// - /// Reassembles and returns the raw sector data for . - /// + /// Reassembles and returns the raw sector data for . public byte[] Extract(DmgEntry entry) { ArgumentNullException.ThrowIfNull(entry); var pi = _partitions.FirstOrDefault(p => p.Name == entry.Name); @@ -251,87 +190,60 @@ public byte[] Extract(DmgEntry entry) { var table = ParseMish(pi.Mish); if (table == null) return []; - var totalBytes = (long)table.SectorCount * SectorSize; - if (totalBytes <= 0 || totalBytes > 2L * 1024 * 1024 * 1024) - totalBytes = Math.Max(0, Math.Min(totalBytes, 2L * 1024 * 1024 * 1024)); - - var output = new byte[totalBytes]; + var physicalBytes = checked((long)table.SectorCount * SectorSize); + if (physicalBytes < 0 || physicalBytes > int.MaxValue) + throw new NotSupportedException("DMG partition is too large for the in-memory extraction API."); + var output = new byte[(int)physicalBytes]; foreach (var block in table.Blocks) { if (block.Type == BlockTypeComment || block.Type == BlockTypeTerminator) continue; - - var destOffset = (long)block.SectorOffset * SectorSize; - var destLength = (long)block.SectorCount * SectorSize; - - if (destLength == 0) continue; - if (destOffset < 0 || destOffset + destLength > output.LongLength) continue; + var destOffset = checked((long)block.SectorOffset * SectorSize); + var destLength = checked((long)block.SectorCount * SectorSize); + if (destLength == 0 || destOffset < 0 || destOffset + destLength > output.LongLength) continue; switch (block.Type) { - case BlockTypeZeroFill: - // Already zero — nothing to write - break; - - case BlockTypeRaw: - ExtractRaw(block, destOffset, destLength, output); - break; - - case BlockTypeZlib: - ExtractZlib(block, destOffset, destLength, output); - break; - - case BlockTypeBzip2: - ExtractBzip2(block, destOffset, destLength, output); - break; - + case BlockTypeZeroFill: break; + case BlockTypeRaw: ExtractRaw(block, destOffset, destLength, output); break; + case BlockTypeZlib: ExtractZlib(block, destOffset, destLength, output); break; + case BlockTypeBzip2: ExtractBzip2(block, destOffset, destLength, output); break; case BlockTypeLzfse: case BlockTypeLzma: - // Unsupported compression — leave zeros in output - break; - default: - // Unknown type — leave zeros break; } } - return output; + if (pi.LogicalSize == output.LongLength) return output; + return output.AsSpan(0, checked((int)Math.Min(pi.LogicalSize, output.LongLength))).ToArray(); } private void ExtractRaw(BlockEntry block, long destOffset, long destLength, byte[] output) { - var srcOffset = (long)block.CompressedOffset; - var srcLength = (long)block.CompressedLength; + var srcOffset = checked((long)block.CompressedOffset); + var srcLength = checked((long)block.CompressedLength); if (srcOffset < 0 || srcOffset + srcLength > _data.LongLength) return; - var copyLen = (int)Math.Min(srcLength, destLength); - _data.AsSpan((int)srcOffset, copyLen).CopyTo(output.AsSpan((int)destOffset)); + var copyLen = checked((int)Math.Min(srcLength, destLength)); + _data.AsSpan(checked((int)srcOffset), copyLen).CopyTo(output.AsSpan(checked((int)destOffset))); } private void ExtractZlib(BlockEntry block, long destOffset, long destLength, byte[] output) { - var srcOffset = (long)block.CompressedOffset; - var srcLength = (long)block.CompressedLength; + var srcOffset = checked((long)block.CompressedOffset); + var srcLength = checked((long)block.CompressedLength); if (srcOffset < 0 || srcLength < 2 || srcOffset + srcLength > _data.LongLength) return; - - // zlib stream: skip 2-byte header (CMF + FLG), use raw DEFLATE try { - using var src = new MemoryStream(_data, (int)srcOffset + 2, (int)srcLength - 2); + using var src = new MemoryStream(_data, checked((int)srcOffset + 2), checked((int)srcLength - 2)); using var deflate = new DeflateStream(src, CompressionMode.Decompress); - using var dst = new MemoryStream(output, (int)destOffset, (int)destLength); + using var dst = new MemoryStream(output, checked((int)destOffset), checked((int)destLength)); deflate.CopyTo(dst); } catch { - // Decompression failed — leave zeros in the output region + // Corrupt/unsupported block: keep the destination zero-filled. } } private static void ExtractBzip2(BlockEntry block, long destOffset, long destLength, byte[] output) { - // bzip2 decompression is not available in the Compression.Core dependency set. - // Leave the region zero-filled (safe no-op for read-only listing/testing scenarios). _ = block; _ = destOffset; _ = destLength; _ = output; } public void Dispose() { } - // ────────────────────────────────────────────────────────────────────────── - // Internal helpers - // ────────────────────────────────────────────────────────────────────────── - - private sealed record PartitionInfo(string Name, byte[] Mish); + internal sealed record PartitionInfo(string Name, byte[] Mish, long LogicalSize); } From dbfe308f31178be0d688cc3075a7a685a17cc491 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:08:53 +0200 Subject: [PATCH 003/192] * expose DMG tail builders for mutation --- FileFormats/FileFormat.Dmg/DmgWriter.cs | 55 +++++++++++-------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/FileFormats/FileFormat.Dmg/DmgWriter.cs b/FileFormats/FileFormat.Dmg/DmgWriter.cs index 9665e5c96..79e0057f4 100644 --- a/FileFormats/FileFormat.Dmg/DmgWriter.cs +++ b/FileFormats/FileFormat.Dmg/DmgWriter.cs @@ -50,25 +50,22 @@ public void WriteTo(Stream output) { } var dataForkLength = pos; - var mishBlobs = new byte[padded.Length][]; + var entries = new List<(string Name, byte[] Mish, long LogicalSize)>(padded.Length); for (var i = 0; i < padded.Length; i++) { var sectorCount = (ulong)(padded[i].data.Length / SectorSize); - mishBlobs[i] = BuildMishBlob( - firstSector: 0, - sectorCount: sectorCount, - rawDataOffset: (ulong)partitionOffsets[i], - rawDataLength: (ulong)padded[i].data.Length); + entries.Add((padded[i].name, + BuildMishBlob(0, sectorCount, (ulong)partitionOffsets[i], (ulong)padded[i].data.Length), + padded[i].logicalSize)); } - var xml = BuildXmlPlist(padded, mishBlobs); - var xmlBytes = Encoding.UTF8.GetBytes(xml); + var xmlBytes = Encoding.UTF8.GetBytes(BuildXmlPlist(entries)); var xmlOffset = pos; foreach (var (_, data, _) in padded) output.Write(data); output.Write(xmlBytes); - var totalSectors = padded.Aggregate<(string name, byte[] data, long logicalSize), ulong>(0, + var totalSectors = padded.Aggregate<(string name, byte[] data, long logicalSize), ulong>(0UL, (current, partition) => current + (ulong)(partition.data.Length / SectorSize)); output.Write(BuildKoly(xmlOffset, xmlBytes.LongLength, dataForkLength, totalSectors)); } @@ -103,6 +100,25 @@ internal static byte[] BuildMishBlob(ulong firstSector, ulong sectorCount, return blob; } + internal static string BuildXmlPlist(IEnumerable<(string Name, byte[] Mish, long LogicalSize)> entries) { + var sb = new StringBuilder(); + sb.AppendLine(""); + sb.AppendLine(""); + sb.AppendLine(""); + sb.AppendLine(""); + sb.AppendLine(" resource-fork"); + sb.AppendLine(" "); + sb.AppendLine(" blkx"); + sb.AppendLine(" "); + foreach (var entry in entries) + sb.AppendLine(BuildBlkxDict(entry.Name, entry.Mish, entry.LogicalSize)); + sb.AppendLine(" "); + sb.AppendLine(" "); + sb.AppendLine(""); + sb.Append(""); + return sb.ToString(); + } + internal static string BuildBlkxDict(string name, byte[] mish, long logicalSize) { var sb = new StringBuilder(); sb.AppendLine(" "); @@ -136,32 +152,11 @@ internal static byte[] BuildKoly(long xmlOffset, long xmlLength, long dataForkLe BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(224), (ulong)xmlLength); BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(492), totalSectors); - // The plist and/or data fork changed. A checksum of type "none" is valid - // UDIF and avoids retaining a checksum that now describes stale bytes. koly.AsSpan(80, 136).Clear(); koly.AsSpan(352, 136).Clear(); return koly; } - private static string BuildXmlPlist((string name, byte[] data, long logicalSize)[] partitions, byte[][] mishBlobs) { - var sb = new StringBuilder(); - sb.AppendLine(""); - sb.AppendLine(""); - sb.AppendLine(""); - sb.AppendLine(""); - sb.AppendLine(" resource-fork"); - sb.AppendLine(" "); - sb.AppendLine(" blkx"); - sb.AppendLine(" "); - for (var i = 0; i < partitions.Length; i++) - sb.AppendLine(BuildBlkxDict(partitions[i].name, mishBlobs[i], partitions[i].logicalSize)); - sb.AppendLine(" "); - sb.AppendLine(" "); - sb.AppendLine(""); - sb.Append(""); - return sb.ToString(); - } - internal static string EscapeXml(string s) { return s.Replace("&", "&").Replace("<", "<").Replace(">", ">") .Replace("\"", """).Replace("'", "'"); From ee7efa20c8cfb9f479a58981a96a24abffe42362 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:09:30 +0200 Subject: [PATCH 004/192] * identify the safely mutable DMG profile --- FileFormats/FileFormat.Dmg/DmgReader.cs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/FileFormats/FileFormat.Dmg/DmgReader.cs b/FileFormats/FileFormat.Dmg/DmgReader.cs index 34ea682d8..162829aaa 100644 --- a/FileFormats/FileFormat.Dmg/DmgReader.cs +++ b/FileFormats/FileFormat.Dmg/DmgReader.cs @@ -27,13 +27,13 @@ public sealed class DmgReader : IDisposable { private readonly List _entries = []; private readonly List _partitions = []; - /// All partitions found in the DMG, each exposed as a named entry. public IReadOnlyList Entries => _entries; - internal long XmlOffset { get; private set; } internal long XmlLength { get; private set; } internal byte[] KolyTrailer { get; private set; } = []; internal IReadOnlyList Partitions => _partitions; + internal bool IsWorkbenchRawProfile => + _partitions.All(p => p.HasLogicalSizeMarker && IsRawMish(p.Mish)); public DmgReader(Stream stream, bool leaveOpen = false) { ArgumentNullException.ThrowIfNull(stream); @@ -60,7 +60,7 @@ private void Parse() { if (XmlLength <= 0 || XmlOffset < 0 || XmlOffset + XmlLength > kolyOff) throw new InvalidDataException("DMG: invalid XML plist region in koly trailer."); - var xmlText = Encoding.UTF8.GetString(_data, (int)XmlOffset, (int)XmlLength); + var xmlText = Encoding.UTF8.GetString(_data, checked((int)XmlOffset), checked((int)XmlLength)); ParseXmlPlist(xmlText); } @@ -85,11 +85,11 @@ private void ParseXmlPlist(string xml) { var parsed = ParseBlkxDict(dictBody, partIndex); if (parsed.Mish != null) { var physicalSize = ComputePartitionSize(parsed.Mish); - var logicalSize = parsed.LogicalSize is >= 0 and <= long.MaxValue + var logicalSize = parsed.LogicalSize.HasValue ? Math.Min(parsed.LogicalSize.Value, physicalSize) : physicalSize; _entries.Add(new DmgEntry { Name = parsed.Name, Size = logicalSize }); - _partitions.Add(new PartitionInfo(parsed.Name, parsed.Mish, logicalSize)); + _partitions.Add(new PartitionInfo(parsed.Name, parsed.Mish, logicalSize, parsed.LogicalSize.HasValue)); partIndex++; } @@ -148,7 +148,6 @@ private static string SanitizeName(string raw, int index) { internal sealed record BlockEntry(uint Type, ulong SectorOffset, ulong SectorCount, ulong CompressedOffset, ulong CompressedLength); - internal sealed record MishTable(ulong FirstSector, ulong SectorCount, ulong DataStart, List Blocks); @@ -176,12 +175,16 @@ internal sealed record MishTable(ulong FirstSector, ulong SectorCount, ulong Dat return new MishTable(firstSector, sectorCount, dataStart, blocks); } + internal static bool IsRawMish(byte[] mish) { + var table = ParseMish(mish); + return table != null && table.Blocks.All(b => b.Type is BlockTypeRaw or BlockTypeTerminator); + } + private static long ComputePartitionSize(byte[] mish) { var table = ParseMish(mish); return table == null ? 0 : checked((long)table.SectorCount * SectorSize); } - /// Reassembles and returns the raw sector data for . public byte[] Extract(DmgEntry entry) { ArgumentNullException.ThrowIfNull(entry); var pi = _partitions.FirstOrDefault(p => p.Name == entry.Name); @@ -191,7 +194,7 @@ public byte[] Extract(DmgEntry entry) { if (table == null) return []; var physicalBytes = checked((long)table.SectorCount * SectorSize); - if (physicalBytes < 0 || physicalBytes > int.MaxValue) + if (physicalBytes > int.MaxValue) throw new NotSupportedException("DMG partition is too large for the in-memory extraction API."); var output = new byte[(int)physicalBytes]; @@ -235,7 +238,7 @@ private void ExtractZlib(BlockEntry block, long destOffset, long destLength, byt using var dst = new MemoryStream(output, checked((int)destOffset), checked((int)destLength)); deflate.CopyTo(dst); } catch { - // Corrupt/unsupported block: keep the destination zero-filled. + // Unsupported or corrupt compressed block: retain zero-fill in this region. } } @@ -245,5 +248,5 @@ private static void ExtractBzip2(BlockEntry block, long destOffset, long destLen public void Dispose() { } - internal sealed record PartitionInfo(string Name, byte[] Mish, long LogicalSize); + internal sealed record PartitionInfo(string Name, byte[] Mish, long LogicalSize, bool HasLogicalSizeMarker); } From 06f86d26666ce9795e128040b964622cbffa4ec1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:09:49 +0200 Subject: [PATCH 005/192] + add true DMG tail-index mutation --- .../FileFormat.Dmg/DmgInPlaceModifier.cs | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 FileFormats/FileFormat.Dmg/DmgInPlaceModifier.cs diff --git a/FileFormats/FileFormat.Dmg/DmgInPlaceModifier.cs b/FileFormats/FileFormat.Dmg/DmgInPlaceModifier.cs new file mode 100644 index 000000000..4a4a16e91 --- /dev/null +++ b/FileFormats/FileFormat.Dmg/DmgInPlaceModifier.cs @@ -0,0 +1,87 @@ +#pragma warning disable CS1591 +using System.Text; +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.Dmg; + +/// +/// Mutates the raw UDIF profile emitted by without +/// rebuilding existing partition payloads. Existing data-fork bytes stay at +/// their physical offsets; replacements/new partitions are appended where the +/// old plist started and the trailing blkx index + koly footer are regenerated. +/// Removed/replaced payloads become unreachable data-fork slack. +/// +internal static class DmgInPlaceModifier { + + public static void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + Mutate(archive, inputs, []); + } + + public static void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + Mutate(archive, [], entryNames); + } + + private static void Mutate(Stream archive, IReadOnlyList inputs, + IReadOnlyCollection removals) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanSeek || !archive.CanRead || !archive.CanWrite) + throw new ArgumentException("DMG mutation requires a seekable read/write stream.", nameof(archive)); + + archive.Position = 0; + using var reader = new DmgReader(archive, leaveOpen: true); + if (!reader.IsWorkbenchRawProfile) + throw new NotSupportedException( + "DMG mutation is supported for the raw UDIF profile emitted by CompressionWorkbench; " + + "foreign compressed/signed plist profiles remain read-only."); + if ((reader.XmlOffset % DmgWriter.SectorSize) != 0) + throw new InvalidDataException("DMG data fork is not sector-aligned."); + + var entries = reader.Partitions + .Select(p => (Name: p.Name, Mish: p.Mish, LogicalSize: p.LogicalSize)) + .ToList(); + + foreach (var name in removals) + entries.RemoveAll(e => string.Equals(e.Name, name, StringComparison.OrdinalIgnoreCase)); + + archive.Position = reader.XmlOffset; + foreach (var (name, data) in FilesOnly(inputs)) { + entries.RemoveAll(e => string.Equals(e.Name, name, StringComparison.OrdinalIgnoreCase)); + + var offset = archive.Position; + var paddedLength = DmgWriter.AlignSector(data.Length); + archive.Write(data); + WriteZeros(archive, paddedLength - data.Length); + + var sectorCount = checked((ulong)(paddedLength / DmgWriter.SectorSize)); + var mish = DmgWriter.BuildMishBlob(0, sectorCount, checked((ulong)offset), checked((ulong)paddedLength)); + entries.Add((name, mish, data.LongLength)); + } + + var xmlOffset = archive.Position; + var xmlBytes = Encoding.UTF8.GetBytes(DmgWriter.BuildXmlPlist(entries)); + archive.Write(xmlBytes); + + // The workbench raw profile keeps the data fork at offset zero. Its old + // payload bytes (including orphaned removed/replaced partitions) remain part + // of that fork; this is what makes removal metadata-only and preserves every + // untouched physical partition offset. + var dataForkLength = xmlOffset; + var sectors = checked((ulong)(dataForkLength / DmgWriter.SectorSize)); + archive.Write(DmgWriter.BuildKoly(xmlOffset, xmlBytes.LongLength, dataForkLength, + sectors, reader.KolyTrailer)); + archive.SetLength(archive.Position); + } + + private static void WriteZeros(Stream output, int count) { + if (count <= 0) return; + Span zero = stackalloc byte[512]; + while (count > 0) { + var take = Math.Min(count, zero.Length); + output.Write(zero[..take]); + count -= take; + } + } +} From 937c9f40b18893ba1ac1bb788f30f0230253588d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:10:08 +0200 Subject: [PATCH 006/192] + expose DMG as a modifiable raw UDIF container --- .../FileFormat.Dmg/DmgFormatDescriptor.cs | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/FileFormats/FileFormat.Dmg/DmgFormatDescriptor.cs b/FileFormats/FileFormat.Dmg/DmgFormatDescriptor.cs index d3328d2ed..fdc8de195 100644 --- a/FileFormats/FileFormat.Dmg/DmgFormatDescriptor.cs +++ b/FileFormats/FileFormat.Dmg/DmgFormatDescriptor.cs @@ -15,11 +15,12 @@ namespace FileFormat.Dmg; /// https://en.wikipedia.org/wiki/Apple_Disk_Image — format overview /// /// -public sealed class DmgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveDefragmentable { +public sealed class DmgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable { public void Defragment(Stream archive) => throw new NotSupportedException( - "DMG is an Apple disk image with mish blocks and a signed footer — defragmentation isn't meaningful."); + "DMG is an Apple disk image with mish blocks and a trailing block index; defragmentation is not exposed as a generic archive verb."); public void Defragment(Stream archive, DefragOptions options) => this.Defragment(archive); public string Id => "Dmg"; @@ -27,7 +28,7 @@ public void Defragment(Stream archive) public FormatCategory Category => FormatCategory.Archive; public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | - FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; public string DefaultExtension => ".dmg"; public IReadOnlyList Extensions => [".dmg"]; public IReadOnlyList CompoundExtensions => []; @@ -51,12 +52,6 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } } - /// - /// Opens a single DMG partition as a bounded read-only . - /// The reader's per-entry extractor reconstructs the partition's raw - /// sectors; they are wrapped in a sized - /// to the entry's size. - /// public Stream OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); @@ -72,7 +67,6 @@ public Stream OpenEntry(Stream archive, string entryName, string? password) { 0, leaveOpen: false); } - /// Native in-memory single-entry extraction. public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { using var s = this.OpenEntry(archive, entryName, password); using var ms = new MemoryStream(); @@ -81,9 +75,6 @@ public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? pas } public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - // WORM: each input becomes a partition with a single raw mish block (no - // compression). The reader rebuilds sectors from the raw block and writes - // each partition out at extract time. var w = new DmgWriter(); foreach (var i in inputs) { if (i.IsDirectory) continue; @@ -91,4 +82,20 @@ public void Create(Stream output, IReadOnlyList inputs, Format } w.WriteTo(output); } + + /// + /// Adds or replaces partitions in the raw UDIF profile emitted by this writer. + /// Existing partition payload offsets are preserved; new data occupies the old + /// plist tail and only the blkx/plist + koly index are rewritten. + /// + public void Add(Stream archive, IReadOnlyList inputs) + => DmgInPlaceModifier.Add(archive, inputs); + + /// + /// Removes partitions from the raw UDIF profile by dropping their blkx records. + /// Payload bytes are left as unreachable data-fork slack so unrelated partitions + /// never need to move. + /// + public void Remove(Stream archive, string[] entryNames) + => DmgInPlaceModifier.Remove(archive, entryNames); } From 41fc222da1cfed72b0e8a87cd3ee4c3785be231f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:10:49 +0200 Subject: [PATCH 007/192] + add a checksummed UEFI FV writer --- FileFormats/FileFormat.UefiFv/UefiFvWriter.cs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 FileFormats/FileFormat.UefiFv/UefiFvWriter.cs diff --git a/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs b/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs new file mode 100644 index 000000000..cc6a148da --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs @@ -0,0 +1,128 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; + +namespace FileFormat.UefiFv; + +/// Writes a standalone PI firmware volume containing ordinary FFS2 files. +internal static class UefiFvWriter { + internal const int HeaderLength = 72; + internal const int FfsHeaderLength = 24; + internal const int Alignment = 8; + internal const int DefaultReserve = 64 * 1024; + internal const uint FvAttributes = 0x0004FEFF; + internal static readonly Guid Ffs2Guid = Guid.Parse("8C8CE578-8A3D-4F1C-9935-896185C32DD3"); + + internal readonly record struct FileIdentity(Guid Guid, byte Type); + + public static byte[] Build(IEnumerable<(string Name, byte[] Data)> inputs) { + var files = inputs.Select(i => (Identity: IdentityFromName(i.Name), i.Data)).ToList(); + var used = HeaderLength + files.Sum(f => Align8(checked(FfsHeaderLength + f.Data.Length))); + var capacity = Align4K(checked(used + DefaultReserve)); + var image = new byte[capacity]; + image.AsSpan().Fill(0xFF); + WriteVolumeHeader(image); + + var position = HeaderLength; + foreach (var file in files) { + var encoded = BuildFfsFile(file.Identity.Guid, file.Identity.Type, file.Data); + encoded.CopyTo(image, position); + position += Align8(encoded.Length); + } + return image; + } + + internal static FileIdentity IdentityFromName(string name) { + ArgumentNullException.ThrowIfNull(name); + var leaf = Path.GetFileName(name); + if (leaf.Length >= 36 && Guid.TryParse(leaf.AsSpan(0, 36), out var guid)) { + var type = ParseTypeTag(leaf.Length > 37 ? leaf[37..] : ""); + return new FileIdentity(guid, type); + } + + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(name.Replace('\\', '/'))); + return new FileIdentity(new Guid(hash.AsSpan(0, 16)), 0x01); // RAW + } + + internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents) { + var size = checked(FfsHeaderLength + contents.Length); + if (size > 0xFFFFFF) + throw new NotSupportedException("FFS2 files larger than 16 MiB require EFI_FFS_FILE_HEADER2."); + + var file = new byte[size]; + guid.TryWriteBytes(file.AsSpan(0, 16)); + file[17] = 0xAA; // fixed checksum when FFS_ATTRIB_CHECKSUM is clear + file[18] = type; + file[19] = 0; + file[20] = (byte)size; + file[21] = (byte)(size >> 8); + file[22] = (byte)(size >> 16); + file[23] = 0xF8; + contents.CopyTo(file.AsSpan(FfsHeaderLength)); + file[16] = HeaderChecksum(file.AsSpan(0, FfsHeaderLength)); + return file; + } + + internal static void WriteVolumeHeader(Span image) { + if (image.Length < HeaderLength) throw new ArgumentException("FV buffer is too small.", nameof(image)); + image[..HeaderLength].Clear(); + Ffs2Guid.TryWriteBytes(image.Slice(16, 16)); + BinaryPrimitives.WriteUInt64LittleEndian(image[32..], (ulong)image.Length); + "_FVH"u8.CopyTo(image[40..]); + BinaryPrimitives.WriteUInt32LittleEndian(image[44..], FvAttributes); + BinaryPrimitives.WriteUInt16LittleEndian(image[48..], HeaderLength); + BinaryPrimitives.WriteUInt16LittleEndian(image[52..], 0); + image[54] = 0; + image[55] = 2; + BinaryPrimitives.WriteUInt32LittleEndian(image[56..], 1); + BinaryPrimitives.WriteUInt32LittleEndian(image[60..], (uint)image.Length); + BinaryPrimitives.WriteUInt32LittleEndian(image[64..], 0); + BinaryPrimitives.WriteUInt32LittleEndian(image[68..], 0); + BinaryPrimitives.WriteUInt16LittleEndian(image[50..], VolumeHeaderChecksum(image[..HeaderLength])); + } + + private static byte HeaderChecksum(ReadOnlySpan header) { + var sum = 0; + for (var i = 0; i < header.Length; i++) { + if (i is 16 or 17 or 23) continue; + sum = (sum + header[i]) & 0xFF; + } + return unchecked((byte)(0 - sum)); + } + + private static ushort VolumeHeaderChecksum(ReadOnlySpan header) { + uint sum = 0; + for (var i = 0; i + 1 < header.Length; i += 2) + sum += BinaryPrimitives.ReadUInt16LittleEndian(header[i..]); + return unchecked((ushort)(0 - sum)); + } + + internal static int Align8(int value) => checked((value + Alignment - 1) & ~(Alignment - 1)); + private static int Align4K(int value) => checked((value + 4095) & ~4095); + + internal static string EntryName(Guid guid, byte type) + => $"{guid:D}_{UefiFvReader.ShortTypeTag(type)}.bin"; + + private static byte ParseTypeTag(string tail) { + var tag = Path.GetFileNameWithoutExtension(tail).TrimStart('_').ToUpperInvariant(); + return tag switch { + "RAW" => 0x01, + "FREEFORM" => 0x02, + "SECURITY_CORE" => 0x03, + "PEI_CORE" => 0x04, + "DXE_CORE" => 0x05, + "PEIM" => 0x06, + "DRIVER" => 0x07, + "COMBINED_PEIM_DRIVER" => 0x08, + "APPLICATION" => 0x09, + "MM" => 0x0A, + "FIRMWARE_VOLUME_IMAGE" => 0x0B, + "COMBINED_MM_DXE" => 0x0C, + "MM_CORE" => 0x0D, + "MM_STANDALONE" => 0x0E, + "MM_CORE_STANDALONE" => 0x0F, + _ => 0x01, + }; + } +} From fe22adab8eb9fc844496637d77ebe92d0d7a3e49 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:11:13 +0200 Subject: [PATCH 008/192] + edit UEFI FFS records through erased free space --- .../UefiFvInPlaceModifier.cs | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs diff --git a/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs b/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs new file mode 100644 index 000000000..de6bf610b --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs @@ -0,0 +1,124 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.UefiFv; + +/// +/// Offline random-access editor for ordinary FFS2 records in a firmware volume. +/// It reuses erased 0xFF ranges and never relocates unrelated FFS files. +/// +internal static class UefiFvInPlaceModifier { + private sealed record Slot(int Offset, int Length, Guid Guid, byte Type) { + public string Name => UefiFvWriter.EntryName(Guid, Type); + } + + public static void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + var state = Open(archive); + foreach (var (name, data) in FilesOnly(inputs)) { + var identity = UefiFvWriter.IdentityFromName(name); + var existing = ScanSlots(state.Image, state.FvStart, state.FvEnd) + .FirstOrDefault(s => s.Guid == identity.Guid); + if (existing != null) + Erase(state, existing.Offset, existing.Length); + + var encoded = UefiFvWriter.BuildFfsFile(identity.Guid, identity.Type, data); + var footprint = UefiFvWriter.Align8(encoded.Length); + var offset = FindErasedRun(state.Image, state.DataStart, state.FvEnd, footprint); + if (offset < 0) + throw new IOException($"UEFI FV has no erased run large enough for '{name}' ({footprint} bytes)."); + + Write(state, offset, encoded); + if (footprint > encoded.Length) + Erase(state, offset + encoded.Length, footprint - encoded.Length); + } + } + + public static void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + var state = Open(archive); + foreach (var name in entryNames) { + var slot = ScanSlots(state.Image, state.FvStart, state.FvEnd) + .FirstOrDefault(s => string.Equals(s.Name, name, StringComparison.OrdinalIgnoreCase)); + if (slot != null) + Erase(state, slot.Offset, slot.Length); + } + } + + private static EditorState Open(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("UEFI FV mutation requires a seekable read/write stream.", nameof(archive)); + if (archive.Length > int.MaxValue) + throw new NotSupportedException("The in-memory FV editor supports images up to 2 GiB."); + + archive.Position = 0; + var image = new byte[checked((int)archive.Length)]; + archive.ReadExactly(image); + var fvStart = UefiFvReader.FindFirst(image) + ?? throw new InvalidDataException("UEFI FV header was not found."); + var fv = UefiFvReader.Read(image, fvStart); + var fvEnd = checked(fvStart + (int)fv.Header.FvLength); + if (fvEnd > image.Length) throw new InvalidDataException("UEFI FV extends past the image."); + var dataStart = Align8(checked(fvStart + fv.Header.HeaderLength)); + return new EditorState(archive, image, fvStart, dataStart, fvEnd); + } + + private static List ScanSlots(byte[] image, int fvStart, int fvEnd) { + var fv = UefiFvReader.Read(image, fvStart); + var pos = Align8(checked(fvStart + fv.Header.HeaderLength)); + var result = new List(); + while (pos + UefiFvWriter.FfsHeaderLength <= fvEnd) { + if (IsErased(image.AsSpan(pos, UefiFvWriter.FfsHeaderLength))) { + pos += UefiFvWriter.Alignment; + continue; + } + + var size = image[pos + 20] | (image[pos + 21] << 8) | (image[pos + 22] << 16); + if (size < UefiFvWriter.FfsHeaderLength || pos + size > fvEnd) + throw new InvalidDataException($"Invalid FFS file header at FV offset 0x{pos - fvStart:X}."); + var guid = new Guid(image.AsSpan(pos, 16)); + var type = image[pos + 18]; + var footprint = UefiFvWriter.Align8(size); + result.Add(new Slot(pos, footprint, guid, type)); + pos += footprint; + } + return result; + } + + private static int FindErasedRun(byte[] image, int start, int end, int needed) { + for (var pos = Align8(start); pos + needed <= end; pos += UefiFvWriter.Alignment) { + if (IsErased(image.AsSpan(pos, needed))) return pos; + } + return -1; + } + + private static bool IsErased(ReadOnlySpan bytes) { + foreach (var b in bytes) + if (b != 0xFF) return false; + return true; + } + + private static void Write(EditorState state, int offset, ReadOnlySpan bytes) { + bytes.CopyTo(state.Image.AsSpan(offset, bytes.Length)); + state.Archive.Position = offset; + state.Archive.Write(bytes); + } + + private static void Erase(EditorState state, int offset, int length) { + state.Image.AsSpan(offset, length).Fill(0xFF); + state.Archive.Position = offset; + Span erased = stackalloc byte[1024]; + erased.Fill(0xFF); + var remaining = length; + while (remaining > 0) { + var take = Math.Min(remaining, erased.Length); + state.Archive.Write(erased[..take]); + remaining -= take; + } + } + + private static int Align8(int value) => (value + 7) & ~7; + private sealed record EditorState(Stream Archive, byte[] Image, int FvStart, int DataStart, int FvEnd); +} From 13449575e75fac8b71bd56f0f7c26680ca68b251 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:11:40 +0200 Subject: [PATCH 009/192] + expose UEFI FV creation and free-space mutation --- .../UefiFvFormatDescriptor.cs | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs b/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs index c44753ada..ceb6e5e3f 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs @@ -7,29 +7,30 @@ namespace FileFormat.UefiFv; /// -/// Pseudo-archive descriptor for UEFI PI Firmware Volumes (.fv/.fd). -/// Locates the FV by scanning for the _FVH signature at offset 40 and -/// emits one entry per FFS file, named {GUID}_{TYPE_TAG}.bin. +/// UEFI PI Firmware Volume (.fv/.fd) archive surface. FFS files are +/// exposed as {GUID}_{TYPE_TAG}.bin; standalone volumes can be created and +/// ordinary FFS2 records can be added/replaced/removed through erased free space. /// /// References: /// -/// https://uefi.org/specifications — UEFI Platform Initialization (PI) Specification — Volume 3 defines Firmware Volumes and FFS -/// https://github.com/LongSoft/UEFITool — UEFITool — canonical firmware-volume parser/editor +/// https://uefi.org/specifications — UEFI Platform Initialization (PI) Specification, Volume 3: Firmware Storage Design +/// https://github.com/LongSoft/UEFITool — UEFITool firmware-volume parser/editor /// /// -public sealed class UefiFvFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class UefiFvFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, + IArchiveCreatable, IArchiveModifiable { public string Id => "UefiFv"; public string DisplayName => "UEFI Firmware Volume"; public FormatCategory Category => FormatCategory.Archive; public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | + FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; public string DefaultExtension => ".fv"; public IReadOnlyList Extensions => [".fv", ".fd"]; public IReadOnlyList CompoundExtensions => []; public IReadOnlyList MagicSignatures => [ - // '_FVH' signature is at offset 40 (not 0) — callers must respect the Offset. new([(byte)'_', (byte)'F', (byte)'V', (byte)'H'], Offset: UefiFvReader.SignatureOffset, Confidence: 0.95), ]; @@ -37,7 +38,7 @@ public sealed class UefiFvFormatDescriptor : IFormatDescriptor, IArchiveFormatOp public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; public string Description => - "UEFI PI Firmware Volume — container for FFS files (PEI/DXE/driver modules)."; + "UEFI PI Firmware Volume — create and offline R/W for ordinary FFS2 files in fixed-capacity volumes."; public List List(Stream stream, string? password) => BuildEntries(stream).Select((e, i) => new ArchiveEntryInfo( @@ -52,10 +53,25 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } } + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + var files = FilesOnly(inputs).Where(f => !string.Equals(f.Name, "metadata.ini", StringComparison.OrdinalIgnoreCase)); + output.Write(UefiFvWriter.Build(files)); + } + + public void Add(Stream archive, IReadOnlyList inputs) + => UefiFvInPlaceModifier.Add(archive, inputs); + + public void Remove(Stream archive, string[] entryNames) + => UefiFvInPlaceModifier.Remove(archive, + entryNames.Where(n => !string.Equals(n, "metadata.ini", StringComparison.OrdinalIgnoreCase)).ToArray()); + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { + if (stream.CanSeek) stream.Position = 0; using var ms = new MemoryStream(); stream.CopyTo(ms); - var data = ms.GetBuffer().AsSpan(0, (int)ms.Length); + var data = ms.GetBuffer().AsSpan(0, checked((int)ms.Length)); var fvStart = UefiFvReader.FindFirst(data) ?? 0; var fv = UefiFvReader.Read(data, fvStart); @@ -63,8 +79,8 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? ("metadata.ini", BuildMetadata(fv), "stored"), }; foreach (var f in fv.Files) { - var tag = UefiFvReader.ShortTypeTag(f.Type); - entries.Add(($"{f.Name:D}_{tag}.bin", f.Contents, "stored")); + if (f.Type == 0xF0) continue; + entries.Add((UefiFvWriter.EntryName(f.Name, f.Type), f.Contents, "stored")); } return entries; } From 25a2c1558632ab7b7a51ce315219abcc0d387989 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:12:04 +0200 Subject: [PATCH 010/192] * let UEFI FV walks cross erased mutation gaps --- FileFormats/FileFormat.UefiFv/UefiFvReader.cs | 77 ++++++------------- 1 file changed, 25 insertions(+), 52 deletions(-) diff --git a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs index db2bda1fd..6eef1d1bd 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs @@ -8,33 +8,12 @@ namespace FileFormat.UefiFv; /// Reader for UEFI Platform Initialization (PI) Firmware Volumes. Locates the /// FV header by scanning for the _FVH signature at offset 40 from the /// start of each 16-byte-aligned candidate (UEFI PI Volume 3). Walks the FFS -/// file list and returns one record per file. +/// file list and returns one record per live file. /// -/// -/// The FV header layout per UEFI PI Spec Vol. 3, §3.2: -/// -/// ZeroVector 16 bytes -/// FileSystemGuid 16 bytes (EFI_FIRMWARE_FILE_SYSTEM2/3_GUID) -/// FvLength 8 bytes (u64 LE) -/// Signature 4 bytes ("_FVH") -/// Attributes 4 bytes (u32 LE) -/// HeaderLength 2 bytes (u16 LE) -/// Checksum 2 bytes (u16 LE) -/// ExtHeaderOffset 2 bytes (u16 LE) — v2 only -/// Reserved 1 byte -/// Revision 1 byte -/// BlockMap[] { u32 NumBlocks, u32 Length } terminated by {0,0} -/// -/// public sealed class UefiFvReader { - - /// FV signature bytes (_FVH) at FV offset 40. public static readonly byte[] Signature = [(byte)'_', (byte)'F', (byte)'V', (byte)'H']; - - /// Signature offset from FV start. public const int SignatureOffset = 40; - /// FV header (excluding block map + extended header body). public sealed record FvHeader( Guid FileSystemGuid, ulong FvLength, @@ -46,13 +25,6 @@ public sealed record FvHeader( IReadOnlyList<(uint NumBlocks, uint Length)> BlockMap ); - /// A single FFS (Firmware File System) file inside the FV. - /// File GUID (EFI_FFS_FILE_HEADER.Name). - /// Raw FFS type byte (see ). - /// FFS file attributes byte. - /// FFS file state byte. - /// Declared file size including the 24-byte header. - /// File contents (size minus the 24-byte header). public sealed record FfsFile( Guid Name, byte Type, @@ -62,14 +34,12 @@ public sealed record FfsFile( byte[] Contents ); - /// Parsed firmware volume. public sealed record FirmwareVolume( int StartOffset, FvHeader Header, IReadOnlyList Files ); - /// Parses a firmware volume located at the given file offset. public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { if (data.Length < fvStart + 56) throw new InvalidDataException("UefiFv: file shorter than minimum FV header."); @@ -79,8 +49,6 @@ public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { throw new InvalidDataException( $"UefiFv: '_FVH' signature not found at offset {fvStart + SignatureOffset}."); - // GUID is stored in EFI format: Data1/Data2/Data3 LE + Data4 BE (.NET's - // little-endian constructor matches). var fsGuid = new Guid(data.Slice(fvStart + 16, 16)); var fvLength = BinaryPrimitives.ReadUInt64LittleEndian(data[(fvStart + 32)..]); var attributes = BinaryPrimitives.ReadUInt32LittleEndian(data[(fvStart + 44)..]); @@ -100,16 +68,12 @@ public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { } var header = new FvHeader(fsGuid, fvLength, attributes, headerLength, checksum, extOff, revision, blockMap); - - // FFS files begin at the end of the FV header, 8-byte aligned. fvLength bounds the FV payload. var ffsStart = fvStart + headerLength; - var ffsEnd = (int)Math.Min((long)data.Length, fvStart + (long)fvLength); + var ffsEnd = checked((int)Math.Min((long)data.Length, fvStart + (long)fvLength)); var files = ReadFfsFiles(data, ffsStart, ffsEnd); - return new FirmwareVolume(fvStart, header, files); } - /// Scans for the first _FVH signature and returns the FV start. public static int? FindFirst(ReadOnlySpan data) { for (var i = 0; i + SignatureOffset + 4 <= data.Length; i += 16) { if (data.Slice(i + SignatureOffset, 4).SequenceEqual(Signature)) @@ -122,26 +86,36 @@ private static List ReadFfsFiles(ReadOnlySpan data, int start, in var files = new List(); var pos = Align8(start); while (pos + 24 <= end) { - var name = new Guid(data.Slice(pos, 16)); - var type = data[pos + 18]; - var attrs = data[pos + 19]; - var size = (uint)(data[pos + 20] | (data[pos + 21] << 8) | (data[pos + 22] << 16)); - var state = data[pos + 23]; - - // An all-0xFF header region marks the end of the file list (unallocated space). - if (type == 0xFF && size == 0xFFFFFFu) break; - if (size < 24 || pos + (int)size > end) break; - - var contents = data.Slice(pos + 24, (int)size - 24).ToArray(); + var header = data.Slice(pos, 24); + if (IsErased(header)) { + // Free/deleted regions may occur between live files after offline + // mutation. Advance one alignment quantum until the next header. + pos += 8; + continue; + } + + var name = new Guid(header[..16]); + var type = header[18]; + var attrs = header[19]; + var size = (uint)(header[20] | (header[21] << 8) | (header[22] << 16)); + var state = header[23]; + if (size < 24 || pos + (long)size > end) break; + + var contents = data.Slice(pos + 24, checked((int)size - 24)).ToArray(); files.Add(new FfsFile(name, type, attrs, state, size, contents)); - pos = Align8(pos + (int)size); + pos = Align8(pos + checked((int)size)); } return files; + static bool IsErased(ReadOnlySpan bytes) { + foreach (var b in bytes) + if (b != 0xFF) return false; + return true; + } + static int Align8(int v) => (v + 7) & ~7; } - /// Decodes the FFS type byte to the UEFI PI spec name. public static string FileTypeName(byte t) => t switch { 0x00 => "EFI_FV_FILETYPE_ALL", 0x01 => "EFI_FV_FILETYPE_RAW", @@ -163,7 +137,6 @@ private static List ReadFfsFiles(ReadOnlySpan data, int start, in _ => $"EFI_FV_FILETYPE_UNKNOWN_0x{t:X2}", }; - /// Returns a short type tag for use in entry names (e.g. RAW, DRIVER). public static string ShortTypeTag(byte t) { var n = FileTypeName(t); const string prefix = "EFI_FV_FILETYPE_"; From a9e7fd52cbd178948c38226ac5c485c2920e4885 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:12:25 +0200 Subject: [PATCH 011/192] + cover DMG random-access mutation and odd lengths --- Compression.Tests/Dmg/DmgModifyTests.cs | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Compression.Tests/Dmg/DmgModifyTests.cs diff --git a/Compression.Tests/Dmg/DmgModifyTests.cs b/Compression.Tests/Dmg/DmgModifyTests.cs new file mode 100644 index 000000000..110c31021 --- /dev/null +++ b/Compression.Tests/Dmg/DmgModifyTests.cs @@ -0,0 +1,71 @@ +using Compression.Registry; +using FileFormat.Dmg; + +namespace Compression.Tests.Dmg; + +[TestFixture] +public sealed class DmgModifyTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Writer_NonSectorAlignedPartition_RoundTripsExactLength() { + var payload = new byte[517]; + new Random(19).NextBytes(payload); + + var writer = new DmgWriter(); + writer.AddPartition("odd.bin", payload); + using var image = new MemoryStream(); + writer.WriteTo(image); + + image.Position = 0; + using var reader = new DmgReader(image); + Assert.That(reader.Entries, Has.Count.EqualTo(1)); + Assert.That(reader.Entries[0].Size, Is.EqualTo(payload.Length)); + Assert.That(reader.Extract(reader.Entries[0]), Is.EqualTo(payload)); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_AddReplaceRemove_MutatesRawUdifProfile() { + var descriptor = new DmgFormatDescriptor(); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var a = Enumerable.Range(0, 517).Select(i => (byte)(i * 11)).ToArray(); + var b = Enumerable.Range(0, 733).Select(i => (byte)(i * 7)).ToArray(); + var c = Enumerable.Range(0, 91).Select(i => (byte)(255 - i)).ToArray(); + var a2 = Enumerable.Range(0, 1025).Select(i => (byte)(i * 3)).ToArray(); + + using var image = new MemoryStream(); + descriptor.Create(image, [ + ArchiveInputInfo.InMemory("A.BIN", a), + ArchiveInputInfo.InMemory("B.BIN", b), + ], new FormatCreateOptions()); + + var modifier = (IArchiveModifiable)descriptor; + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory("C.BIN", c)]); + AssertPayload(image, "A.BIN", a); + AssertPayload(image, "B.BIN", b); + AssertPayload(image, "C.BIN", c); + + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory("A.BIN", a2)]); + AssertPayload(image, "A.BIN", a2); + AssertPayload(image, "B.BIN", b); + AssertPayload(image, "C.BIN", c); + + image.Position = 0; + modifier.Remove(image, ["B.BIN"]); + image.Position = 0; + using var reader = new DmgReader(image); + Assert.That(reader.Entries.Select(e => e.Name), Is.EquivalentTo(new[] { "A.BIN", "C.BIN" })); + Assert.That(reader.Extract(reader.Entries.Single(e => e.Name == "A.BIN")), Is.EqualTo(a2)); + Assert.That(reader.Extract(reader.Entries.Single(e => e.Name == "C.BIN")), Is.EqualTo(c)); + } + + private static void AssertPayload(MemoryStream image, string name, byte[] expected) { + image.Position = 0; + using var reader = new DmgReader(image); + var entry = reader.Entries.Single(e => string.Equals(e.Name, name, StringComparison.OrdinalIgnoreCase)); + Assert.That(entry.Size, Is.EqualTo(expected.Length)); + Assert.That(reader.Extract(entry), Is.EqualTo(expected)); + } +} From ff78987910423c84755bcd181c02052c4d837e41 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:12:40 +0200 Subject: [PATCH 012/192] + verify UEFI FV create and in-place CRUD --- Compression.Tests/UefiFv/UefiFvWriteTests.cs | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 Compression.Tests/UefiFv/UefiFvWriteTests.cs diff --git a/Compression.Tests/UefiFv/UefiFvWriteTests.cs b/Compression.Tests/UefiFv/UefiFvWriteTests.cs new file mode 100644 index 000000000..64f6dd99d --- /dev/null +++ b/Compression.Tests/UefiFv/UefiFvWriteTests.cs @@ -0,0 +1,75 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.UefiFv; + +namespace Compression.Tests.UefiFv; + +[TestFixture] +public sealed class UefiFvWriteTests { + private const string DriverName = "11223344-5566-7788-99aa-bbccddeeff00_DRIVER.bin"; + private const string RawName = "01234567-89ab-cdef-0123-456789abcdef_RAW.bin"; + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_Create_ProducesChecksummedFixedCapacityFv() { + var payload = Enumerable.Range(0, 333).Select(i => (byte)(i * 13)).ToArray(); + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, payload)], new FormatCreateOptions()); + + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(image.Length, Is.GreaterThan(payload.Length + 64 * 1024)); + + var bytes = image.ToArray(); + var fv = UefiFvReader.Read(bytes); + Assert.That(fv.Files, Has.Count.EqualTo(1)); + Assert.That(fv.Files[0].Contents, Is.EqualTo(payload)); + + uint sum = 0; + for (var i = 0; i < fv.Header.HeaderLength; i += 2) + sum += BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(i, 2)); + Assert.That((ushort)sum, Is.Zero, "FV header checksum must sum to zero"); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_AddReplaceRemove_ReusesErasedSpaceWithoutGrowingVolume() { + var descriptor = new UefiFvFormatDescriptor(); + var first = Enumerable.Range(0, 257).Select(i => (byte)i).ToArray(); + var second = Enumerable.Range(0, 721).Select(i => (byte)(i * 5)).ToArray(); + var replacement = Enumerable.Range(0, 1023).Select(i => (byte)(255 - i)).ToArray(); + + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, first)], new FormatCreateOptions()); + var originalLength = image.Length; + + var modifier = (IArchiveModifiable)descriptor; + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory(RawName, second)]); + Assert.That(image.Length, Is.EqualTo(originalLength)); + AssertFiles(image, (DriverName, first), (RawName, second)); + + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory(DriverName, replacement)]); + Assert.That(image.Length, Is.EqualTo(originalLength)); + AssertFiles(image, (DriverName, replacement), (RawName, second)); + + image.Position = 0; + modifier.Remove(image, [RawName]); + Assert.That(image.Length, Is.EqualTo(originalLength)); + AssertFiles(image, (DriverName, replacement)); + } + + private static void AssertFiles(MemoryStream image, params (string Name, byte[] Data)[] expected) { + var bytes = image.ToArray(); + var fv = UefiFvReader.Read(bytes); + var actual = fv.Files.Where(f => f.Type != 0xF0) + .ToDictionary(f => $"{f.Name:D}_{UefiFvReader.ShortTypeTag(f.Type)}.bin", f => f.Contents, + StringComparer.OrdinalIgnoreCase); + Assert.That(actual.Keys, Is.EquivalentTo(expected.Select(e => e.Name))); + foreach (var (name, data) in expected) + Assert.That(actual[name], Is.EqualTo(data), name); + } +} From 690c8897beccb75b5cceb056c36996b6ecf32ef5 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:14:00 +0200 Subject: [PATCH 013/192] + put DMG under the generic modify round-trip contract --- .../Operations/ArchiveModifyRoundTripTests.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs b/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs index 13e9c5058..490751766 100644 --- a/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs +++ b/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs @@ -19,15 +19,13 @@ namespace Compression.Tests.Operations; [TestFixture] public class ArchiveModifyRoundTripTests { - // Name-preserving modifiable archive containers (probe names round-trip verbatim). private static readonly string[] NamePreservingModifiableArchives = [ "Ace", "Afs", "Ampk", "AndroidBundle", "Ba2", "Big", "Bsa", "Cbr", "Chm", - "CompactPro", "Deb", "Doc", "Dzip", "FreeArc", "Gar", "Gob", "GodotPck", + "CompactPro", "Deb", "Dmg", "Doc", "Dzip", "FreeArc", "Gar", "Gob", "GodotPck", "Grp", "Hpi", "LzxAmiga", "Mpq", "Msg", "Msi", "Msix", "Narc", "Nds", "Nsa", "Ppt", "Psarc", "Rgss", "Rpa", "Sar", "Sarc", "Slf", "Sqx", "StuffIt", "ThumbsDb", "Tnef", "U8", "Uharc", "Vpk", "Vpp", "VppV2", "Vsdx", "Wad", "Xls", "Xps", "Ypf", "Zpaq", - // previously promoted siblings that share the same contract "SevenZip", "Zip", "Tar", "Rar", "Cab", "Arj", "Zoo", "Arc", "Appx", "Apk", ]; @@ -64,7 +62,6 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { } if (ms.Length == 0) { Assert.Ignore($"{formatId}: create produced no image."); return; } - // Add a third file through the modify path. ms.Position = 0; try { modifiable.Add(ms, [ArchiveInputInfo.InMemory("C.TXT", cData)]); @@ -77,7 +74,6 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { Assert.That(Has(afterAdd, "B.BIN"), Is.True, $"{formatId}: B.BIN lost during Add (after={Join(afterAdd)})"); Assert.That(Has(afterAdd, "C.TXT"), Is.True, $"{formatId}: C.TXT missing after Add (after={Join(afterAdd)})"); - // Remove the second file through the modify path. ms.Position = 0; var bName = afterAdd.First(n => Matches(n, "B.BIN")); modifiable.Remove(ms, [bName]); @@ -86,7 +82,6 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { Assert.That(Has(afterRemove, "A.TXT"), Is.True, $"{formatId}: A.TXT lost during Remove (after={Join(afterRemove)})"); Assert.That(Has(afterRemove, "C.TXT"), Is.True, $"{formatId}: C.TXT lost during Remove (after={Join(afterRemove)})"); - // Survivors must extract byte-identically. var work = Path.Combine(Path.GetTempPath(), "cwb_modrt_" + Guid.NewGuid().ToString("N")[..8]); Directory.CreateDirectory(work); try { @@ -100,7 +95,7 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { Assert.That(File.ReadAllBytes(a!).SequenceEqual(aData), Is.True, $"{formatId}: A.TXT content changed by modify cycle"); Assert.That(File.ReadAllBytes(c!).SequenceEqual(cData), Is.True, $"{formatId}: C.TXT content changed by modify cycle"); } finally { - try { Directory.Delete(work, true); } catch { /* best effort */ } + try { Directory.Delete(work, true); } catch { } } } @@ -113,6 +108,5 @@ private static bool Matches(string name, string user) => string.Equals(Path.GetFileName(name.Replace('\\', '/')), user, StringComparison.OrdinalIgnoreCase); private static bool Has(IEnumerable names, string user) => names.Any(n => Matches(n, user)); - private static string Join(IEnumerable names) => string.Join(",", names.Take(8)); } From ca4b4b96db1a2dea35631de1b7e650859294e136 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:16:20 +0200 Subject: [PATCH 014/192] + add OrangeFS DBPF object writing --- .../FileSystem.OrangeFs/OrangeFsWriter.cs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 FileSystems/FileSystem.OrangeFs/OrangeFsWriter.cs diff --git a/FileSystems/FileSystem.OrangeFs/OrangeFsWriter.cs b/FileSystems/FileSystem.OrangeFs/OrangeFsWriter.cs new file mode 100644 index 000000000..b1158be2d --- /dev/null +++ b/FileSystems/FileSystem.OrangeFs/OrangeFsWriter.cs @@ -0,0 +1,42 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; + +namespace FileSystem.OrangeFs; + +/// Writes and edits standalone OrangeFS/PVFS2 DBPF storage objects. +internal static class OrangeFsWriter { + internal const int HeaderSize = 16; + + public static void Create(Stream output, ReadOnlySpan payload, + bool orangeFs = true, uint version = 1, uint datastreamType = 0) { + ArgumentNullException.ThrowIfNull(output); + Span header = stackalloc byte[HeaderSize]; + (orangeFs ? OrangeFsReader.OrangeFsTag : OrangeFsReader.PvfsTag).CopyTo(header); + BinaryPrimitives.WriteUInt32LittleEndian(header[4..], version); + BinaryPrimitives.WriteUInt32LittleEndian(header[8..], datastreamType); + BinaryPrimitives.WriteUInt32LittleEndian(header[12..], checked((uint)payload.Length)); + output.Write(header); + output.Write(payload); + } + + public static void ReplacePayload(Stream image, ReadOnlySpan payload) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("OrangeFS mutation requires a seekable read/write stream.", nameof(image)); + if (payload.Length > uint.MaxValue) + throw new NotSupportedException("DBPF object payload exceeds the 32-bit object-size field."); + + image.Position = 0; + Span header = stackalloc byte[HeaderSize]; + image.ReadExactly(header); + if (!header[..4].SequenceEqual(OrangeFsReader.PvfsTag) && + !header[..4].SequenceEqual(OrangeFsReader.OrangeFsTag)) + throw new InvalidDataException("OrangeFS/PVFS2 DBPF header is invalid."); + + BinaryPrimitives.WriteUInt32LittleEndian(header[12..], checked((uint)payload.Length)); + image.Position = 0; + image.Write(header); + image.Write(payload); + image.SetLength(HeaderSize + payload.Length); + } +} From 8b353a9f84d619f344b7b0ee4e1899b2803227af Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:16:39 +0200 Subject: [PATCH 015/192] + promote OrangeFS DBPF objects to R/W --- .../OrangeFsFormatDescriptor.cs | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/FileSystems/FileSystem.OrangeFs/OrangeFsFormatDescriptor.cs b/FileSystems/FileSystem.OrangeFs/OrangeFsFormatDescriptor.cs index c0b667b74..737f9745d 100644 --- a/FileSystems/FileSystem.OrangeFs/OrangeFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.OrangeFs/OrangeFsFormatDescriptor.cs @@ -5,49 +5,48 @@ namespace FileSystem.OrangeFs; /// -/// Read-only descriptor for OrangeFS / PVFS2 DBPF (Direct Block Pool -/// Format) storage-object files. PVFS2 is a parallel distributed FS, but -/// its server-side bstream-XX objects are single files starting -/// with a 4-byte ASCII tag ("PVFS" classic, "OGFP" -/// OrangeFS-native) followed by version, datastream-type, and object-size -/// fields. The contained object payload is surfaced as a single opaque -/// entry — semantic resolution requires cluster fs.conf. +/// OrangeFS / PVFS2 DBPF storage-object descriptor. A DBPF file is one server-side +/// storage object rather than a complete distributed filesystem namespace; the +/// opaque object payload can nevertheless be created, replaced and removed while +/// preserving its DBPF tag/version/datastream identity. /// /// References: /// /// https://github.com/waltligon/orangefs — official PVFS/OrangeFS repository (DBPF storage layer) /// https://www.kernel.org/doc/html/latest/filesystems/orangefs.html — Linux kernel client documentation -/// https://en.wikipedia.org/wiki/OrangeFS — Wikipedia article /// /// -public sealed class OrangeFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveDefragmentable { +public sealed class OrangeFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, + IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable { public string Id => "OrangeFs"; public string DisplayName => "OrangeFS / PVFS2 DBPF"; public FormatCategory Category => FormatCategory.Archive; public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract; + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | FormatCapabilities.CanTest; public string DefaultExtension => ".orangefs"; public IReadOnlyList Extensions => [".orangefs", ".pvfs", ".bstream"]; public IReadOnlyList CompoundExtensions => []; public IReadOnlyList MagicSignatures => [ - // "PVFS" at offset 0 (classic PVFS2 DBPF). new("PVFS"u8.ToArray(), Offset: 0, Confidence: 0.90), - // "OGFP" at offset 0 (OrangeFS-native DBPF). new("OGFP"u8.ToArray(), Offset: 0, Confidence: 0.90), ]; public IReadOnlyList Methods => [new("stored", "Stored")]; public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; - public string Description => "OrangeFS / PVFS2 DBPF — stub: header-only, opaque storage-object payload."; + public string Description => + "OrangeFS / PVFS2 DBPF storage object — opaque object payload R/W; cluster namespace resolution requires fs.conf."; public List List(Stream stream, string? password) { - var r = new OrangeFsReader(stream); + if (stream.CanSeek) stream.Position = 0; + using var r = new OrangeFsReader(stream); return r.Entries.Select((e, i) => new ArchiveEntryInfo( i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); } public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new OrangeFsReader(stream); + if (stream.CanSeek) stream.Position = 0; + using var r = new OrangeFsReader(stream); foreach (var e in r.Entries) { if (e.IsDirectory) continue; if (files != null && !MatchesFilter(e.Name, files)) continue; @@ -55,9 +54,33 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } } - public void Defragment(Stream archive) - => throw new NotSupportedException("OrangeFs read-only — defragmentation requires a writer."); + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + var payload = FilesOnly(inputs) + .FirstOrDefault(f => !IsSynthetic(f.Name)).Data ?? []; + OrangeFsWriter.Create(output, payload); + } + + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + var payload = FilesOnly(inputs).LastOrDefault(f => !IsSynthetic(f.Name)).Data; + if (payload != null) + OrangeFsWriter.ReplacePayload(archive, payload); + } - public void Defragment(Stream archive, DefragOptions options) - => throw new NotSupportedException("OrangeFs read-only — defragmentation requires a writer."); + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Any(n => string.Equals(Path.GetFileName(n), "object.bin", StringComparison.OrdinalIgnoreCase))) + OrangeFsWriter.ReplacePayload(archive, []); + } + + public void Defragment(Stream archive) { } + public void Defragment(Stream archive, DefragOptions options) { } + + private static bool IsSynthetic(string name) { + var leaf = Path.GetFileName(name); + return leaf.Equals("metadata.ini", StringComparison.OrdinalIgnoreCase) + || leaf.StartsWith("FULL.", StringComparison.OrdinalIgnoreCase); + } } From dda74a5cf6483888d38ad4a1918af496e21aec3a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:16:50 +0200 Subject: [PATCH 016/192] + cover OrangeFS DBPF payload mutation --- .../OrangeFs/OrangeFsWriteTests.cs | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 Compression.Tests/OrangeFs/OrangeFsWriteTests.cs diff --git a/Compression.Tests/OrangeFs/OrangeFsWriteTests.cs b/Compression.Tests/OrangeFs/OrangeFsWriteTests.cs new file mode 100644 index 000000000..42cdfcb26 --- /dev/null +++ b/Compression.Tests/OrangeFs/OrangeFsWriteTests.cs @@ -0,0 +1,46 @@ +using Compression.Registry; +using FileSystem.OrangeFs; + +namespace Compression.Tests.OrangeFs; + +[TestFixture] +public sealed class OrangeFsWriteTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void CreateReplaceRemove_RoundTripsObjectPayloadAndPreservesHeaderIdentity() { + var descriptor = new OrangeFsFormatDescriptor(); + var first = Enumerable.Range(0, 97).Select(i => (byte)(i * 3)).ToArray(); + var replacement = Enumerable.Range(0, 513).Select(i => (byte)(i * 7)).ToArray(); + using var image = new MemoryStream(); + + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory("object.bin", first)], new FormatCreateOptions()); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var before = image.ToArray()[..12]; + AssertPayload(image, first); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory("object.bin", replacement)]); + Assert.That(image.ToArray()[..12], Is.EqualTo(before)); + AssertPayload(image, replacement); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Remove(image, ["object.bin"]); + Assert.That(image.Length, Is.EqualTo(16)); + AssertPayload(image, []); + } + + private static void AssertPayload(MemoryStream image, byte[] expected) { + image.Position = 0; + using var reader = new OrangeFsReader(image); + var objectEntry = reader.Entries.FirstOrDefault(e => e.Name == "object.bin"); + if (expected.Length == 0) { + Assert.That(objectEntry, Is.Null); + Assert.That(reader.ObjectSize, Is.Zero); + return; + } + Assert.That(objectEntry, Is.Not.Null); + Assert.That(reader.Extract(objectEntry!), Is.EqualTo(expected)); + } +} From eff458abee0fee7f8571475dbeffc3d2e550bb43 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:17:38 +0200 Subject: [PATCH 017/192] + preserve DTB node hierarchy when writing --- FileFormats/FileFormat.Dtb/DtbWriter.cs | 264 ++++++++++++++---------- 1 file changed, 150 insertions(+), 114 deletions(-) diff --git a/FileFormats/FileFormat.Dtb/DtbWriter.cs b/FileFormats/FileFormat.Dtb/DtbWriter.cs index 43425cf6f..b0fea16c9 100644 --- a/FileFormats/FileFormat.Dtb/DtbWriter.cs +++ b/FileFormats/FileFormat.Dtb/DtbWriter.cs @@ -4,142 +4,105 @@ namespace FileFormat.Dtb; -/// -/// WORM writer for the Flattened Device Tree Blob (FDT v17) format. Produces a -/// minimal valid DTB where every input becomes a leaf property on the root node. -/// The root node carries spec-required #address-cells = <2> and -/// #size-cells = <2> properties so the blob round-trips through -/// fdtdump / dtc consumers without warnings. -/// -/// -/// Layout per Devicetree Specification v0.4: -/// -/// 40-byte BE header: magic, totalsize, off_dt_struct, off_dt_strings, -/// off_mem_rsvmap, version=17, last_comp_version=16, boot_cpuid_phys=0, -/// size_dt_strings, size_dt_struct. -/// Memory reservation block: one terminating {0, 0} 16-byte entry. -/// Structure block: FDT_BEGIN_NODE "" \0 (padding) -/// + per-property FDT_PROP len nameoff data (padding) -/// + FDT_END_NODE + FDT_END. -/// Strings block: NUL-terminated property names. -/// -/// All values are big-endian per the spec. Structure-block tokens and property -/// payloads are 4-byte aligned. -/// +/// Writer for Flattened Device Tree Blob (FDT v17) images. public sealed class DtbWriter { + internal sealed record PropertySpec(string NodePath, string Name, byte[] Data); - /// - /// Writes a minimal FDT blob to whose root node - /// contains one property per input. Each input's archive-name leaf is used as - /// the property name; the raw bytes become the property value. Names are - /// deduplicated in the strings block, but each occurrence still gets its own - /// FDT_PROP record (multiple identical property names on one node are - /// technically nonconforming, but matching the input list verbatim is the - /// honest WORM behaviour). - /// public static void Write(Stream output, IReadOnlyList<(string Name, byte[] Data)> inputs) { - ArgumentNullException.ThrowIfNull(output); ArgumentNullException.ThrowIfNull(inputs); + var properties = inputs.Select(i => FromArchiveEntry(i.Name, i.Data)).ToList(); + Write(output, properties, [], 0, addDefaultRootCells: true); + } + + internal static void Write(Stream output, IReadOnlyList properties, + IReadOnlyList reservations, uint bootCpuidPhys, + bool addDefaultRootCells = false) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(properties); + ArgumentNullException.ThrowIfNull(reservations); + + var root = new Node(""); + foreach (var property in properties) { + var node = root; + foreach (var segment in property.NodePath.Trim('/').Split('/', StringSplitOptions.RemoveEmptyEntries)) + node = node.GetOrAdd(SanitiseNodeName(segment)); + node.Properties.Add(new PropertySpec(property.NodePath, + SanitisePropertyName(property.Name), property.Data)); + } + + if (addDefaultRootCells) { + EnsureCellProperty(root, "#address-cells", 2); + EnsureCellProperty(root, "#size-cells", 2); + } - // Build the strings block first so we can compute name offsets up front. using var strings = new MemoryStream(); var stringOffsets = new Dictionary(StringComparer.Ordinal); - uint InternName(string name) { - if (stringOffsets.TryGetValue(name, out var off)) return off; - off = (uint)strings.Length; - stringOffsets[name] = off; + if (stringOffsets.TryGetValue(name, out var existing)) return existing; + var offset = checked((uint)strings.Length); + stringOffsets[name] = offset; var bytes = Encoding.ASCII.GetBytes(name); - strings.Write(bytes, 0, bytes.Length); + strings.Write(bytes); strings.WriteByte(0); - return off; - } - - // Pre-intern the spec-required root-node properties so they appear early in - // the strings block (fdtdump-friendly). - var addrCellsOff = InternName("#address-cells"); - var sizeCellsOff = InternName("#size-cells"); - - // Build the structure block. - using var structBlk = new MemoryStream(); - - void WriteToken(uint token) { - Span tk = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(tk, token); - structBlk.Write(tk); - } - - void AlignStruct() { - while ((structBlk.Length & 3) != 0) structBlk.WriteByte(0); + return offset; } - void WriteProp(uint nameOff, ReadOnlySpan data) { - WriteToken(DtbReader.FDT_PROP); - Span hdr = stackalloc byte[8]; - BinaryPrimitives.WriteUInt32BigEndian(hdr[..4], (uint)data.Length); - BinaryPrimitives.WriteUInt32BigEndian(hdr[4..], nameOff); - structBlk.Write(hdr); - if (data.Length > 0) structBlk.Write(data); - AlignStruct(); - } - - // FDT_BEGIN_NODE for root ("" name, NUL-terminated, padded). - WriteToken(DtbReader.FDT_BEGIN_NODE); - structBlk.WriteByte(0); - AlignStruct(); - - // Root: #address-cells = <2>, #size-cells = <2> (big-endian u32). - Span twoCells = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(twoCells, 2); - WriteProp(addrCellsOff, twoCells); - WriteProp(sizeCellsOff, twoCells); - - // One FDT_PROP per input, in order. - foreach (var (name, data) in inputs) { - var safe = SanitisePropertyName(name); - var off = InternName(safe); - WriteProp(off, data); - } + foreach (var node in root.Walk()) + foreach (var property in node.Properties) + _ = InternName(property.Name); - WriteToken(DtbReader.FDT_END_NODE); - WriteToken(DtbReader.FDT_END); + using var structBlock = new MemoryStream(); + WriteNode(root, structBlock, InternName); + WriteToken(structBlock, DtbReader.FDT_END); - // Assemble final blob. - const int HeaderSize = 40; - const int MemRsvmapSize = 16; // one terminator {0, 0} - var structOff = HeaderSize + MemRsvmapSize; - var structSize = (uint)structBlk.Length; - var stringsOff = (uint)(structOff + structSize); - var stringsSize = (uint)strings.Length; - var totalSize = stringsOff + stringsSize; + const int headerSize = 40; + var reservationSize = checked((reservations.Count + 1) * 16); + var structOffset = checked(headerSize + reservationSize); + var structSize = checked((uint)structBlock.Length); + var stringsOffset = checked((uint)(structOffset + structSize)); + var stringsSize = checked((uint)strings.Length); + var totalSize = checked(stringsOffset + stringsSize); - Span header = stackalloc byte[HeaderSize]; + Span header = stackalloc byte[headerSize]; BinaryPrimitives.WriteUInt32BigEndian(header[0..4], DtbReader.Magic); BinaryPrimitives.WriteUInt32BigEndian(header[4..8], totalSize); - BinaryPrimitives.WriteUInt32BigEndian(header[8..12], (uint)structOff); - BinaryPrimitives.WriteUInt32BigEndian(header[12..16], stringsOff); - BinaryPrimitives.WriteUInt32BigEndian(header[16..20], HeaderSize); // off_mem_rsvmap - BinaryPrimitives.WriteUInt32BigEndian(header[20..24], 17); // version - BinaryPrimitives.WriteUInt32BigEndian(header[24..28], 16); // last_comp_version - BinaryPrimitives.WriteUInt32BigEndian(header[28..32], 0); // boot_cpuid_phys + BinaryPrimitives.WriteUInt32BigEndian(header[8..12], (uint)structOffset); + BinaryPrimitives.WriteUInt32BigEndian(header[12..16], stringsOffset); + BinaryPrimitives.WriteUInt32BigEndian(header[16..20], headerSize); + BinaryPrimitives.WriteUInt32BigEndian(header[20..24], 17); + BinaryPrimitives.WriteUInt32BigEndian(header[24..28], 16); + BinaryPrimitives.WriteUInt32BigEndian(header[28..32], bootCpuidPhys); BinaryPrimitives.WriteUInt32BigEndian(header[32..36], stringsSize); BinaryPrimitives.WriteUInt32BigEndian(header[36..40], structSize); - output.Write(header); - // 16-byte memory reservation terminator (both 64-bit fields zero). - Span rsv = stackalloc byte[MemRsvmapSize]; - output.Write(rsv); - structBlk.Position = 0; - structBlk.CopyTo(output); + + Span reservation = stackalloc byte[16]; + foreach (var item in reservations) { + reservation.Clear(); + BinaryPrimitives.WriteUInt64BigEndian(reservation[..8], item.Address); + BinaryPrimitives.WriteUInt64BigEndian(reservation[8..], item.Size); + output.Write(reservation); + } + reservation.Clear(); + output.Write(reservation); + + structBlock.Position = 0; + structBlock.CopyTo(output); strings.Position = 0; strings.CopyTo(output); } - /// - /// Coerces an input archive name into a property name valid per - /// devicetree-specification §2.2.4 (ASCII subset of property-name chars). - /// Reserved chars are replaced with _; the leaf of any path is used. - /// + internal static PropertySpec FromArchiveEntry(string archiveName, byte[] data) { + var normalized = archiveName.Replace('\\', '/').Trim('/'); + var slash = normalized.LastIndexOf('/'); + var nodePath = slash < 0 ? "/" : "/" + normalized[..slash]; + var leaf = slash < 0 ? normalized : normalized[(slash + 1)..]; + if (leaf.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) || + leaf.EndsWith(".bin", StringComparison.OrdinalIgnoreCase)) + leaf = leaf[..^4]; + return new PropertySpec(nodePath, SanitisePropertyName(leaf), data); + } + public static string SanitisePropertyName(string archiveName) { var leaf = archiveName; var slash = leaf.LastIndexOfAny(['/', '\\']); @@ -148,8 +111,7 @@ public static string SanitisePropertyName(string archiveName) { var sb = new StringBuilder(leaf.Length); foreach (var c in leaf) { - var keep = - c is >= '0' and <= '9' + var keep = c is >= '0' and <= '9' || c is >= 'a' and <= 'z' || c is >= 'A' and <= 'Z' || c is ',' or '.' or '_' or '+' or '?' or '#' or '-'; @@ -157,4 +119,78 @@ public static string SanitisePropertyName(string archiveName) { } return sb.Length == 0 ? "_" : sb.ToString(); } + + private static string SanitiseNodeName(string name) { + if (name.Length == 0) return "_"; + var sb = new StringBuilder(name.Length); + foreach (var c in name) { + var keep = c is >= '0' and <= '9' + || c is >= 'a' and <= 'z' + || c is >= 'A' and <= 'Z' + || c is ',' or '.' or '_' or '+' or '?' or '#' or '-' or '@'; + sb.Append(keep ? c : '_'); + } + return sb.ToString(); + } + + private static void EnsureCellProperty(Node root, string name, uint value) { + if (root.Properties.Any(p => string.Equals(p.Name, name, StringComparison.Ordinal))) return; + var data = new byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(data, value); + root.Properties.Insert(0, new PropertySpec("/", name, data)); + } + + private static void WriteNode(Node node, Stream output, Func internName) { + WriteToken(output, DtbReader.FDT_BEGIN_NODE); + var name = Encoding.ASCII.GetBytes(node.Name); + output.Write(name); + output.WriteByte(0); + Align4(output); + + foreach (var property in node.Properties) { + WriteToken(output, DtbReader.FDT_PROP); + Span header = stackalloc byte[8]; + BinaryPrimitives.WriteUInt32BigEndian(header[..4], checked((uint)property.Data.Length)); + BinaryPrimitives.WriteUInt32BigEndian(header[4..], internName(property.Name)); + output.Write(header); + output.Write(property.Data); + Align4(output); + } + + foreach (var child in node.Children) + WriteNode(child, output, internName); + WriteToken(output, DtbReader.FDT_END_NODE); + } + + private static void WriteToken(Stream output, uint token) { + Span bytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(bytes, token); + output.Write(bytes); + } + + private static void Align4(Stream output) { + while ((output.Position & 3) != 0) output.WriteByte(0); + } + + private sealed class Node(string name) { + private readonly Dictionary _childrenByName = new(StringComparer.Ordinal); + public string Name { get; } = name; + public List Properties { get; } = []; + public List Children { get; } = []; + + public Node GetOrAdd(string childName) { + if (_childrenByName.TryGetValue(childName, out var child)) return child; + child = new Node(childName); + _childrenByName.Add(childName, child); + Children.Add(child); + return child; + } + + public IEnumerable Walk() { + yield return this; + foreach (var child in Children) + foreach (var descendant in child.Walk()) + yield return descendant; + } + } } From a83734a2feba48002e0e9df9cac08690845b4566 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:17:53 +0200 Subject: [PATCH 018/192] + add hierarchy-preserving DTB mutation --- FileFormats/FileFormat.Dtb/DtbModifier.cs | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 FileFormats/FileFormat.Dtb/DtbModifier.cs diff --git a/FileFormats/FileFormat.Dtb/DtbModifier.cs b/FileFormats/FileFormat.Dtb/DtbModifier.cs new file mode 100644 index 000000000..fb7adda46 --- /dev/null +++ b/FileFormats/FileFormat.Dtb/DtbModifier.cs @@ -0,0 +1,59 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.Dtb; + +/// Rewrites mutable DTB structure/string blocks while preserving reservations and boot CPU metadata. +internal static class DtbModifier { + public static void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + Mutate(archive, properties => { + foreach (var (name, data) in FilesOnly(inputs)) { + if (string.Equals(Path.GetFileName(name), "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; + var incoming = DtbWriter.FromArchiveEntry(name, data); + properties.RemoveAll(p => SameProperty(p, incoming)); + properties.Add(incoming); + } + }); + } + + public static void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + Mutate(archive, properties => { + foreach (var name in entryNames) { + if (string.Equals(Path.GetFileName(name), "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; + var target = DtbWriter.FromArchiveEntry(name, []); + properties.RemoveAll(p => SameProperty(p, target)); + } + }); + } + + private static void Mutate(Stream archive, Action> edit) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("DTB mutation requires a seekable read/write stream.", nameof(archive)); + if (archive.Length > int.MaxValue) + throw new NotSupportedException("DTB images larger than 2 GiB are not supported."); + + archive.Position = 0; + var bytes = new byte[checked((int)archive.Length)]; + archive.ReadExactly(bytes); + var fdt = DtbReader.Read(bytes); + var properties = fdt.Properties + .Select(p => new DtbWriter.PropertySpec(p.NodePath, p.Name, p.Data)) + .ToList(); + edit(properties); + + using var rebuilt = new MemoryStream(); + DtbWriter.Write(rebuilt, properties, fdt.Reservations, fdt.Header.BootCpuidPhys); + archive.Position = 0; + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.SetLength(archive.Position); + } + + private static bool SameProperty(DtbWriter.PropertySpec a, DtbWriter.PropertySpec b) => + string.Equals(a.NodePath.TrimEnd('/'), b.NodePath.TrimEnd('/'), StringComparison.Ordinal) && + string.Equals(a.Name, b.Name, StringComparison.Ordinal); +} From 89ccb8eb48b373c60353fda3d520c1c67f0bc3e3 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:18:16 +0200 Subject: [PATCH 019/192] + promote DTB to hierarchy-preserving R/W --- .../FileFormat.Dtb/DtbFormatDescriptor.cs | 48 ++++++------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/FileFormats/FileFormat.Dtb/DtbFormatDescriptor.cs b/FileFormats/FileFormat.Dtb/DtbFormatDescriptor.cs index b2890b3ae..c4f448309 100644 --- a/FileFormats/FileFormat.Dtb/DtbFormatDescriptor.cs +++ b/FileFormats/FileFormat.Dtb/DtbFormatDescriptor.cs @@ -17,17 +17,17 @@ namespace FileFormat.Dtb; /// /// https://github.com/devicetree-org/devicetree-specification — Devicetree Specification — defines the flattened (FDT/DTB) encoding /// https://www.devicetree.org — devicetree.org portal -/// https://en.wikipedia.org/wiki/Device_tree — background /// /// -public sealed class DtbFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable { +public sealed class DtbFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, + IArchiveCreatable, IArchiveModifiable { public string Id => "Dtb"; public string DisplayName => "Flattened Device Tree Blob"; public FormatCategory Category => FormatCategory.Archive; public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | - FormatCapabilities.CanTest | + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; public string DefaultExtension => ".dtb"; public IReadOnlyList Extensions => [".dtb", ".dtbo"]; @@ -39,13 +39,7 @@ public sealed class DtbFormatDescriptor : IFormatDescriptor, IArchiveFormatOpera public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; public string Description => - "Flattened Device Tree Blob — BE structured description of hardware used by Linux/U-Boot. " + - "R-only: in-place R/W is not honestly available because the 40-byte FDT header carries " + - "totalsize / off_dt_strings / off_dt_struct / size_dt_struct / size_dt_strings fields whose " + - "values cascade through every property add/remove. Any single-byte change to the struct or " + - "strings block would require rewriting all four header offsets plus shifting every " + - "downstream byte of the blob — that's a rebuild, not an in-place mutation, so promoting " + - "to CanModify would mis-advertise the surface."; + "Flattened Device Tree Blob — hierarchy-preserving create/add/replace/remove with reservation and boot CPU preservation."; public List List(Stream stream, string? password) => BuildEntries(stream).Select((e, i) => new ArchiveEntryInfo( @@ -60,15 +54,6 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } } - /// - /// WORM creation: emits a minimal valid FDT v17 blob whose root node carries - /// each input as a leaf property. The synthetic metadata.ini + any - /// reader-emitted .txt/.bin suffixes are stripped from the - /// archive name before sanitisation so a list-then-create round-trip lands at - /// the same property name. Property names are sanitised to the - /// devicetree-spec character set; collisions in the input list are preserved - /// as repeated FDT_PROP records. - /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); ArgumentNullException.ThrowIfNull(inputs); @@ -77,27 +62,27 @@ public void Create(Stream output, IReadOnlyList inputs, Format if (i.IsDirectory) continue; var leaf = Path.GetFileName(i.ArchiveName); if (string.Equals(leaf, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - // Strip reader-emitted ".txt" / ".bin" suffixes so the property name on - // round-trip matches the original property name in the input DTB. - if (leaf.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) || - leaf.EndsWith(".bin", StringComparison.OrdinalIgnoreCase)) - leaf = leaf[..^4]; - list.Add((leaf, i.ReadContent())); + list.Add((i.ArchiveName, i.ReadContent())); } DtbWriter.Write(output, list); } + public void Add(Stream archive, IReadOnlyList inputs) + => DtbModifier.Add(archive, inputs); + + public void Remove(Stream archive, string[] entryNames) + => DtbModifier.Remove(archive, entryNames); + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { + if (stream.CanSeek) stream.Position = 0; using var ms = new MemoryStream(); stream.CopyTo(ms); - var fdt = DtbReader.Read(ms.GetBuffer().AsSpan(0, (int)ms.Length)); + var fdt = DtbReader.Read(ms.GetBuffer().AsSpan(0, checked((int)ms.Length))); var entries = new List<(string, byte[], string)> { ("metadata.ini", BuildMetadata(fdt), "stored"), }; - // Names can collide (same property name reached through NOP-walked ambiguity); - // disambiguate with an auto-increment suffix per collision. var seen = new Dictionary(StringComparer.Ordinal); foreach (var p in fdt.Properties) { var asText = TryStringifyPropertyValue(p.Data); @@ -117,14 +102,9 @@ public void Create(Stream output, IReadOnlyList inputs, Format return entries; } - /// - /// Returns a newline-separated decoded string when is - /// entirely printable ASCII plus NUL separators (the common compatible - /// pattern), or null for binary cell/byte data. - /// private static string? TryStringifyPropertyValue(byte[] data) { if (data.Length == 0) return ""; - if (data[^1] != 0) return null; // must be NUL-terminated + if (data[^1] != 0) return null; foreach (var b in data) if (b != 0 && (b < 0x20 || b > 0x7E)) return null; var parts = Encoding.ASCII.GetString(data, 0, data.Length - 1).Split('\0'); From 5bdc6ca0dbccda1162fa54ca0ac6f92e32c3b8ed Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:19:11 +0200 Subject: [PATCH 020/192] * preserve DTB root and text-property round trips --- FileFormats/FileFormat.Dtb/DtbWriter.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/FileFormats/FileFormat.Dtb/DtbWriter.cs b/FileFormats/FileFormat.Dtb/DtbWriter.cs index b0fea16c9..6e1b72be1 100644 --- a/FileFormats/FileFormat.Dtb/DtbWriter.cs +++ b/FileFormats/FileFormat.Dtb/DtbWriter.cs @@ -95,11 +95,19 @@ uint InternName(string name) { internal static PropertySpec FromArchiveEntry(string archiveName, byte[] data) { var normalized = archiveName.Replace('\\', '/').Trim('/'); var slash = normalized.LastIndexOf('/'); - var nodePath = slash < 0 ? "/" : "/" + normalized[..slash]; + var directory = slash < 0 ? "" : normalized[..slash]; + var nodePath = directory.Length == 0 || directory.Equals("_root", StringComparison.OrdinalIgnoreCase) + ? "/" + : "/" + directory; var leaf = slash < 0 ? normalized : normalized[(slash + 1)..]; - if (leaf.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) || - leaf.EndsWith(".bin", StringComparison.OrdinalIgnoreCase)) + var isText = leaf.EndsWith(".txt", StringComparison.OrdinalIgnoreCase); + if (isText || leaf.EndsWith(".bin", StringComparison.OrdinalIgnoreCase)) leaf = leaf[..^4]; + + if (isText && (data.Length == 0 || data[^1] != 0)) { + var text = Encoding.UTF8.GetString(data).Replace("\r\n", "\n"); + data = Encoding.UTF8.GetBytes(text.Replace('\n', '\0') + "\0"); + } return new PropertySpec(nodePath, SanitisePropertyName(leaf), data); } From 1bbb620ec5e03fc1e8d89362f6eff883b9ccdc9e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:19:26 +0200 Subject: [PATCH 021/192] + verify hierarchy-preserving DTB R/W --- Compression.Tests/Dtb/DtbModifyTests.cs | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Compression.Tests/Dtb/DtbModifyTests.cs diff --git a/Compression.Tests/Dtb/DtbModifyTests.cs b/Compression.Tests/Dtb/DtbModifyTests.cs new file mode 100644 index 000000000..04332a5aa --- /dev/null +++ b/Compression.Tests/Dtb/DtbModifyTests.cs @@ -0,0 +1,62 @@ +using Compression.Registry; +using FileFormat.Dtb; + +namespace Compression.Tests.Dtb; + +[TestFixture] +public sealed class DtbModifyTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void CreateAddReplaceRemove_PreservesHierarchy() { + var descriptor = new DtbFormatDescriptor(); + var compatible = "vendor,board\0"u8.ToArray(); + var reg = new byte[] { 0, 0, 0, 1, 0, 0, 0, 32 }; + var status = "okay\0"u8.ToArray(); + var replacement = "disabled\0"u8.ToArray(); + + using var image = new MemoryStream(); + descriptor.Create(image, [ + ArchiveInputInfo.InMemory("soc/serial@1000/compatible.bin", compatible), + ArchiveInputInfo.InMemory("soc/serial@1000/reg.bin", reg), + ], new FormatCreateOptions()); + + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + AssertProperty(image, "/soc/serial@1000", "compatible", compatible); + AssertProperty(image, "/soc/serial@1000", "reg", reg); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory("soc/serial@1000/status.bin", status)]); + AssertProperty(image, "/soc/serial@1000", "status", status); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory("soc/serial@1000/status.bin", replacement)]); + AssertProperty(image, "/soc/serial@1000", "status", replacement); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Remove(image, ["soc/serial@1000/reg.bin"]); + var parsed = Parse(image); + Assert.That(parsed.Properties.Any(p => p.NodePath == "/soc/serial@1000" && p.Name == "reg"), Is.False); + Assert.That(parsed.Properties.Any(p => p.NodePath == "/soc/serial@1000" && p.Name == "compatible"), Is.True); + } + + [Test, Category("RoundTrip")] + public void RootTextEntry_ListCreateRoundTrip_MapsBackToRootAndRestoresNulTermination() { + var descriptor = new DtbFormatDescriptor(); + using var image = new MemoryStream(); + descriptor.Create(image, + [ArchiveInputInfo.InMemory("_root/compatible.txt", "vendor,board"u8.ToArray())], + new FormatCreateOptions()); + + var parsed = Parse(image); + var property = parsed.Properties.Single(p => p.NodePath == "/" && p.Name == "compatible"); + Assert.That(property.Data, Is.EqualTo("vendor,board\0"u8.ToArray())); + } + + private static DtbReader.Fdt Parse(MemoryStream image) => DtbReader.Read(image.ToArray()); + + private static void AssertProperty(MemoryStream image, string path, string name, byte[] data) { + var property = Parse(image).Properties.Single(p => p.NodePath == path && p.Name == name); + Assert.That(property.Data, Is.EqualTo(data)); + } +} From 2c7143e0479a33426818a49d2b2f467351e6146b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:20:31 +0200 Subject: [PATCH 022/192] - remove obsolete DTB read-only contract --- .../Dtb/DtbInPlaceModifyTests.cs | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100644 Compression.Tests/Dtb/DtbInPlaceModifyTests.cs diff --git a/Compression.Tests/Dtb/DtbInPlaceModifyTests.cs b/Compression.Tests/Dtb/DtbInPlaceModifyTests.cs deleted file mode 100644 index 898550e38..000000000 --- a/Compression.Tests/Dtb/DtbInPlaceModifyTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -using Compression.Registry; -using FileFormat.Dtb; - -namespace Compression.Tests.Dtb; - -/// -/// Locks the honest demotion of FileFormat.Dtb: the descriptor must NOT -/// advertise and the Description -/// must name the specific spec-level reason that blocks in-place mutation. -/// The companion read tests live in . -/// -/// Why DTB stays R-only: the 40-byte FDT header at offset 0 records -/// totalsize, off_dt_struct, off_dt_strings, -/// size_dt_struct, size_dt_strings. Adding/removing any -/// property mutates the struct or strings block, which cascades through every -/// downstream offset and the four header size/offset fields. Mutating those -/// fields in place is a rebuild, not an in-place splice — so promoting to -/// CanModify would mis-advertise the surface. -/// -[TestFixture] -public class DtbInPlaceModifyTests { - - [Test, Category("HappyPath")] - public void Descriptor_DoesNotAdvertiseCanModify() { - var desc = new DtbFormatDescriptor(); - Assert.That(desc.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); - Assert.That(desc, Is.Not.InstanceOf()); - } - - [Test, Category("HappyPath")] - public void Description_NamesTheBlockingFdtHeaderFields() { - var desc = new DtbFormatDescriptor(); - Assert.That(desc.Description, Does.Contain("totalsize")); - Assert.That(desc.Description, Does.Contain("off_dt_strings")); - Assert.That(desc.Description, Does.Contain("off_dt_struct")); - Assert.That(desc.Description, Does.Contain("rebuild")); - } -} From 235e91f57fab8edb6a18cf8f53382349f7d66dc3 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:37:11 +0200 Subject: [PATCH 023/192] * define R/W by supported existing-image edits --- Compression.Registry/FormatCapabilities.cs | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Compression.Registry/FormatCapabilities.cs b/Compression.Registry/FormatCapabilities.cs index b5a77191b..cff72da2a 100644 --- a/Compression.Registry/FormatCapabilities.cs +++ b/Compression.Registry/FormatCapabilities.cs @@ -8,23 +8,23 @@ namespace Compression.Registry; /// /// Unsupported — no descriptor exists. /// Read-Only and/or only. -/// WORM (Write-Once-Read-Many) — adds : a fresh archive can be produced from inputs, but existing archives cannot be modified in place. -/// R/W (Modify) — adds : entries can be added, replaced, or removed in an existing archive without full rewrite. +/// WORM (Write-Once-Read-Many) — adds : a fresh archive/image can be produced, but the library has no supported edit of an existing instance. +/// R/W (Modify) — adds : an existing instance supports add/replace/remove and remains valid after the edit. /// /// -/// Most archive formats stop at WORM; true in-place modification is rare because compressed -/// archive containers don't generally support entry mutation without a full rebuild. +/// R/W describes the public operation, not the physical write strategy. +/// A format may update allocation metadata in place, append a new index, relayout members, +/// or rebuild the complete image. Those are implementation choices. If callers can open an +/// existing instance, apply add/replace/remove through , and +/// obtain a valid instance preserving the semantics the implementation claims to support, +/// the format is R/W at this API surface. Conversely, merely having a writer for fresh images +/// is WORM and must not set . /// /// -/// Honesty rule — rebuild-backed modification is WORM, not R/W. A format may implement -/// purely to make the add / remove / purge verbs work, -/// backing them with the verified extract → re-create rebuild (the default -/// members, or ModifyRebuilder / ). That is a full rewrite of the -/// container, so such a format advertises only and must not set -/// — the verb still runs, but no in-place R/W is claimed. -/// is reserved for formats with a genuine in-place writer that edits the existing container -/// (e.g. ZIP/TAR central-directory edits, FAT/NTFS/ext block writes, byte-identity append). -/// Compression.Tests.Operations.WriteCapabilityHonestyTests enforces this for every claimant. +/// This distinction is especially important for read-only-on-mount filesystem formats such as +/// SquashFS, CramFS and EROFS: the native filesystem driver may intentionally forbid mounted +/// writes while an offline image editor can still support complete, deterministic mutation by +/// relayout/rebuild. reports the latter capability. /// /// [Flags] @@ -32,7 +32,7 @@ public enum FormatCapabilities { None = 0, CanList = 1 << 0, CanExtract = 1 << 1, - /// WORM: can produce a fresh archive from inputs (no in-place modification). + /// WORM: can produce a fresh archive/image, but has no supported existing-instance edit. CanCreate = 1 << 2, CanTest = 1 << 3, SupportsPassword = 1 << 4, @@ -40,6 +40,6 @@ public enum FormatCapabilities { SupportsDirectories = 1 << 6, SupportsOptimize = 1 << 8, CanCompoundWithTar = 1 << 9, - /// R/W: can modify an existing archive (add/replace/remove entries) without full rewrite. Implies . + /// R/W: can add/replace/remove entries in an existing archive/image. The implementation may edit in place or relayout/rebuild. Implies for normal writable formats. CanModify = 1 << 10, -} +} \ No newline at end of file From b401d04a875066609a667106bf64273459a6508c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:37:28 +0200 Subject: [PATCH 024/192] * align modify contract with existing-image semantics --- Compression.Registry/IArchiveModifiable.cs | 45 +++++++++++----------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/Compression.Registry/IArchiveModifiable.cs b/Compression.Registry/IArchiveModifiable.cs index bcaa47969..58be8a608 100644 --- a/Compression.Registry/IArchiveModifiable.cs +++ b/Compression.Registry/IArchiveModifiable.cs @@ -1,29 +1,27 @@ namespace Compression.Registry; /// -/// Opt-in capability: the descriptor exposes add / remove (and thereby the purge verb). +/// Opt-in capability for editing an existing archive/image through add/replace/remove. /// -/// Implementing this interface makes the verbs work; it does not by itself -/// entitle the format to advertise (R/W). The -/// default / below — and any override that delegates -/// to ModifyRebuilder / — are a verified extract → re-create -/// rebuild, i.e. a full rewrite of the container. A format whose modification is only -/// rebuild-backed is WORM: it advertises and must -/// NOT advertise (see ). -/// Reserve for a genuine in-place writer that edits -/// the existing bytes (R/W filesystems; central-directory / member edits; byte-identity append). +/// The physical strategy is format-specific: implementations may patch blocks in place, +/// append replacement metadata, relayout members, or perform a verified extract → edit → +/// re-create rebuild. All are valid implementations of the same public mutation contract +/// when the resulting instance preserves the semantics the descriptor claims to support. +/// +/// +/// A descriptor advertising must expose this +/// interface and its supported-profile edit path must actually round-trip. Merely being able +/// to create a fresh instance is not enough. /// /// public interface IArchiveModifiable { /// - /// Appends or replaces files inside . On replacement the - /// previous bytes are wiped the same way wipes them. + /// Adds files to an existing instance, replacing entries with the same logical path/name. /// - /// Default implementation: any descriptor that also implements - /// + gets - /// add for free — a verified extract → splat-new-files → re-create rebuild via - /// (the same WORM rebuild that backs the - /// other verbs). Formats with a true in-place writer override for efficiency. + /// Default implementation: descriptors that also implement + /// and get a verified + /// extract → edit → re-create implementation through . + /// Formats with a cheaper native editor override it. /// void Add(Stream archive, IReadOnlyList inputs) { if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) @@ -41,11 +39,12 @@ void Add(Stream archive, IReadOnlyList inputs) { } /// - /// Removes the named entries from and wipes all on-disk - /// traces. Default implementation: a verified extract → drop-named-files → - /// re-create rebuild via . Passing every - /// entry name (or all files) yields an empty container — i.e. the purge verb. - /// Formats with a true in-place writer override for efficiency and forensic wiping. + /// Removes the named entries from an existing instance. Passing every entry name yields an + /// empty container/image where the format permits one. + /// + /// Default implementation: a verified extract → drop-named-files → re-create + /// edit through . Native implementations may instead + /// unlink/free in place and optionally wipe released storage. /// void Remove(Stream archive, string[] entryNames) { if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) @@ -60,4 +59,4 @@ void Remove(Stream archive, string[] entryNames) { } }); } -} +} \ No newline at end of file From c5a0382eb5ab86eb5c4b1489e6080929e622e70c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:37:39 +0200 Subject: [PATCH 025/192] * test R/W as supported existing-instance mutation --- .../Operations/WriteCapabilityHonestyTests.cs | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs b/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs index f76aac18b..fff90c079 100644 --- a/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs +++ b/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs @@ -7,22 +7,18 @@ namespace Compression.Tests.Operations; /// Honesty guard for the WORM-vs-R/W capability claim. The write scale /// (Unsupported → Read-Only → WORM → R/W) is only meaningful if /// is reserved for formats that genuinely -/// support modifying an existing container. +/// support modifying an existing container/image. /// /// R/W means a working add / replace / remove on an existing instance that yields a -/// valid result. The edit may be byte-preserving in place or may relayout / -/// re-pack the container (moving existing data) — both are honest R/W for a conceptually -/// read-write format. What is NOT honest is advertising -/// with no working modify path at all. Read-only-by-design formats (CramFS, SquashFS) and -/// create-only formats stay WORM () even though a -/// rebuild could synthesise a modified copy — they do not advertise R/W. +/// valid result. The edit may be byte-preserving in place, append replacement state, +/// relayout members, or rebuild the image. Those are implementation choices. What is not +/// honest is advertising with no working edit path. +/// A format that can only create a fresh instance remains WORM. /// /// -/// This test enforces the deterministic half of that rule for every registered format that -/// claims : its runtime ops object must actually -/// implement (otherwise the R/W claim is entirely unbacked — -/// there is no modify path). That the modify works (round-trips) is verified -/// separately by the registry-driven Generic{Purge,Defrag}RoundTripTests. +/// This test enforces the deterministic half of that rule for every registered claimant: +/// its runtime ops object must implement . Behavioural +/// round-trip tests verify that individual modify paths actually work. /// /// [TestFixture] @@ -42,7 +38,7 @@ public void EveryCanModifyClaimIsBackedByAModifyPath(string formatId) { Assert.That(ops, Is.InstanceOf(), $"{formatId} advertises R/W (CanModify) but its ops does not implement IArchiveModifiable — " - + "the claim is unbacked. Implement IArchiveModifiable (in-place or relayout/rebuild) " + + "the claim is unbacked. Implement a working existing-instance edit path " + "or downgrade to WORM (CanCreate only)."); } -} +} \ No newline at end of file From 419bddabdefb245d7caa1a51392ecd50617fe3c3 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:40:27 +0200 Subject: [PATCH 026/192] + reconstruct mutable EWF media payload --- FileFormats/FileFormat.Ewf/EwfMedia.cs | 115 +++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 FileFormats/FileFormat.Ewf/EwfMedia.cs diff --git a/FileFormats/FileFormat.Ewf/EwfMedia.cs b/FileFormats/FileFormat.Ewf/EwfMedia.cs new file mode 100644 index 000000000..9860e514e --- /dev/null +++ b/FileFormats/FileFormat.Ewf/EwfMedia.cs @@ -0,0 +1,115 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using Compression.Core.Checksums; +using FileFormat.Zlib; + +namespace FileFormat.Ewf; + +/// +/// Reconstructs the acquired medium represented by a single-segment EWF image. +/// The descriptor treats that medium as the one semantic mutable payload; EWF +/// sections remain diagnostic/internal views because editing one independently +/// would invalidate table offsets, checksums and evidence hashes. +/// +internal static class EwfMedia { + private const int TableHeaderSize = 24; + + public static bool TryExtract(EwfReader.EwfImage image, out byte[] media) { + try { + media = Extract(image); + return true; + } catch (Exception e) when (e is InvalidDataException or NotSupportedException or OverflowException) { + media = []; + return false; + } + } + + public static byte[] Extract(EwfReader.EwfImage image) { + ArgumentNullException.ThrowIfNull(image); + if (image.IsLogical) + throw new NotSupportedException("Logical EWF/L01 media reconstruction is not implemented."); + + var volume = image.Sections.FirstOrDefault(s => s.Type is "volume" or "data") + ?? throw new InvalidDataException("EWF image has no volume/data media descriptor."); + if (volume.Payload.Length < 20) + throw new InvalidDataException("EWF volume payload is too short."); + + var chunkCount = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(volume.Payload.AsSpan(4, 4))); + var sectorsPerChunk = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(volume.Payload.AsSpan(8, 4))); + var bytesPerSector = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(volume.Payload.AsSpan(12, 4))); + var totalSectors = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(volume.Payload.AsSpan(16, 4))); + if (chunkCount < 0 || sectorsPerChunk <= 0 || bytesPerSector <= 0 || totalSectors < 0) + throw new InvalidDataException("EWF volume geometry is invalid."); + + var sectors = image.Sections.FirstOrDefault(s => s.Type == "sectors") + ?? throw new InvalidDataException("EWF image has no sectors section."); + var table = image.Sections.FirstOrDefault(s => s.Type is "table" or "table2") + ?? throw new InvalidDataException("EWF image has no chunk table."); + var entries = ReadTable(table.Payload, chunkCount); + + var expectedBytes = checked((long)totalSectors * bytesPerSector); + if (expectedBytes > int.MaxValue) + throw new NotSupportedException("EWF media exceeds the in-memory mutation profile."); + var output = new byte[(int)expectedBytes]; + var outputOffset = 0; + var nominalChunkBytes = checked(sectorsPerChunk * bytesPerSector); + + for (var i = 0; i < entries.Length && outputOffset < output.Length; ++i) { + var encoded = entries[i]; + var compressed = (encoded & 0x80000000u) != 0; + var relative = encoded & 0x7FFFFFFFu; + var nextRelative = i + 1 < entries.Length + ? entries[i + 1] & 0x7FFFFFFFu + : checked((uint)(EwfReader.SectionDescriptorSize + sectors.Payload.Length)); + if (relative < EwfReader.SectionDescriptorSize || nextRelative < relative) + throw new InvalidDataException("EWF chunk table contains invalid offsets."); + + var start = checked((int)relative - EwfReader.SectionDescriptorSize); + var end = checked((int)nextRelative - EwfReader.SectionDescriptorSize); + if (start < 0 || end < start || end > sectors.Payload.Length) + throw new InvalidDataException("EWF chunk table points outside the sectors payload."); + var stored = sectors.Payload.AsSpan(start, end - start); + + byte[] chunk; + if (compressed) { + chunk = ZlibStream.Decompress(stored); + } else { + if (stored.Length < 4) + throw new InvalidDataException("Stored EWF chunk is shorter than its Adler-32 trailer."); + var data = stored[..^4]; + var expected = BinaryPrimitives.ReadUInt32LittleEndian(stored[^4..]); + var actual = Adler32.Compute(data); + if (actual != expected) + throw new InvalidDataException( + $"Stored EWF chunk Adler-32 mismatch: expected 0x{expected:X8}, got 0x{actual:X8}."); + chunk = data.ToArray(); + } + + var expectedChunk = Math.Min(nominalChunkBytes, output.Length - outputOffset); + if (chunk.Length < expectedChunk) + throw new InvalidDataException("EWF chunk decompressed shorter than the declared media geometry."); + chunk.AsSpan(0, expectedChunk).CopyTo(output.AsSpan(outputOffset)); + outputOffset += expectedChunk; + } + + if (outputOffset != output.Length) + throw new InvalidDataException("EWF chunk table does not cover the complete declared medium."); + return output; + } + + private static uint[] ReadTable(byte[] payload, int expectedChunks) { + if (payload.Length < TableHeaderSize + 4) + throw new InvalidDataException("EWF table payload is too short."); + var count = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(payload.AsSpan(0, 4))); + if (count != expectedChunks) + throw new InvalidDataException($"EWF table has {count} chunks, volume declares {expectedChunks}."); + var entriesBytes = checked(count * 4); + if (TableHeaderSize + entriesBytes + 4 > payload.Length) + throw new InvalidDataException("EWF table entry array is truncated."); + + var entries = new uint[count]; + for (var i = 0; i < count; ++i) + entries[i] = BinaryPrimitives.ReadUInt32LittleEndian(payload.AsSpan(TableHeaderSize + i * 4, 4)); + return entries; + } +} \ No newline at end of file From 2ac44779345a3f0fd68c3cb2bdb539c66a0d74c6 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:41:33 +0200 Subject: [PATCH 027/192] ci: apply one-shot R/W promotion patches --- .github/workflows/rw-promotion-once.yml | 197 ++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 .github/workflows/rw-promotion-once.yml diff --git a/.github/workflows/rw-promotion-once.yml b/.github/workflows/rw-promotion-once.yml new file mode 100644 index 000000000..8911d97c0 --- /dev/null +++ b/.github/workflows/rw-promotion-once.yml @@ -0,0 +1,197 @@ +name: R/W promotion one-shot + +on: + push: + branches: [feat/filesystem-rw-gaps] + +permissions: + contents: write + +jobs: + patch: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: feat/filesystem-rw-gaps + - name: Patch capability declarations + shell: python + run: | + from pathlib import Path + import re + + def text(path): + return Path(path).read_text(encoding='utf-8') + def write(path, value): + Path(path).write_text(value, encoding='utf-8') + def sub(path, pattern, replacement, count=1, flags=0): + value = text(path) + new, n = re.subn(pattern, replacement, value, count=count, flags=flags) + if n != count: + raise SystemExit(f'{path}: expected {count} replacement(s), got {n}: {pattern}') + write(path, new) + + # CramFS: native mount is read-only; the workbench already has a verified + # extract/edit/re-create modifier, so expose that existing-image edit. + p = 'FileSystems/FileSystem.CramFs/CramFsFormatDescriptor.cs' + sub(p, + r' // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;', + ' // The on-disk filesystem is read-only when mounted, but CompressionWorkbench\n' + ' // supports existing-image add/replace/remove by verified relayout/rebuild.\n' + ' public FormatCapabilities Capabilities =>\n' + ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' + ' FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n' + ' FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;', + flags=re.S) + sub(p, r'public string Description => "Linux compressed ROM filesystem";', + 'public string Description => "Linux compressed ROM filesystem; offline image mutation is rebuild-backed.";') + + # SquashFS: same distinction — read-only mount format, editable image. + p = 'FileSystems/FileSystem.SquashFs/SquashFsFormatDescriptor.cs' + sub(p, + r' // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;', + ' // SquashFS is read-only when mounted, but the workbench can edit an existing\n' + ' // image by a verified extract/edit/re-create pass. That is R/W at this API.\n' + ' public FormatCapabilities Capabilities =>\n' + ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' + ' FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n' + ' FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;', + flags=re.S) + sub(p, r'public string Description => "Linux compressed read-only filesystem";', + 'public string Description => "Linux compressed read-only-on-mount filesystem; offline image mutation is rebuild-backed.";') + + # EROFS: promote only the writer/reader profile. Add/Remove below use ReadEntries, + # which throws on an undecodable compressed inode instead of materialising the + # descriptor\'s user-facing placeholder and losing content. + p = 'FileSystems/FileSystem.Erofs/ErofsFormatDescriptor.cs' + sub(p, r'the round-trippable WORM subset', 'the round-trippable offline R/W subset') + sub(p, + r'FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \|', + 'FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' + ' FormatCapabilities.CanModify | FormatCapabilities.CanTest |') + sub(p, r'public string Description => "Android read-only compressed filesystem; uncompressed \+ inline inode layouts\.";', + 'public string Description => "Android read-only-on-mount filesystem; supported uncompressed/inline profile is offline R/W.";') + marker = ' private static ErofsReader OpenReader(Stream stream) {' + value = text(p) + if marker not in value: + raise SystemExit('EROFS insertion marker missing') + methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + archive.Position = 0; + var label = OpenReader(archive).VolumeName; + archive.Position = 0; + ModifyRebuilder.Add(archive, inputs, + readEntries: stream => ReadEntries(stream), + buildImage: files => { + var writer = new ErofsWriter { VolumeName = label }; + foreach (var (name, data) in files) writer.AddFile(name, data); + return writer.Build(); + }); + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + archive.Position = 0; + var label = OpenReader(archive).VolumeName; + archive.Position = 0; + ModifyRebuilder.Remove(archive, entryNames, + readEntries: stream => ReadEntries(stream), + buildImage: files => { + var writer = new ErofsWriter { VolumeName = label }; + foreach (var (name, data) in files) writer.AddFile(name, data); + return writer.Build(); + }); + } + +''' + value = value.replace(marker, methods + marker, 1) + write(p, value) + + # MSA already has functional Add/Remove against the inner GEMDOS image. + p = 'FileSystems/FileSystem.Msa/MsaFormatDescriptor.cs' + sub(p, + r' // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest;', + ' // Existing MSA images are editable through the decoded GEMDOS volume and\n' + ' // then re-encoded; physical rebuild does not make the public operation WORM.\n' + ' public FormatCapabilities Capabilities =>\n' + ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' + ' FormatCapabilities.CanModify | FormatCapabilities.CanTest;', + flags=re.S) + + # PFS0 has had a real existing-container editor for a while; its flag was stale. + p = 'FileFormats/FileFormat.Pfs0/Pfs0FormatDescriptor.cs' + sub(p, + r' // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries;', + ' // Existing PFS0 archives support add/replace/remove through Pfs0InPlaceModifier.\n' + ' public FormatCapabilities Capabilities =>\n' + ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' + ' FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;', + flags=re.S) + + # Archive-model docs must agree with FormatCapabilities and the package README. + p = 'docs/ARCHIVE-MODEL.md' + sub(p, + r'`CanModify` is \*\*withheld\*\* only from \*\*read-only-by-design\*\* formats \(CramFS, SquashFS\) and\n\*\*create-only\*\* formats \(e\.g\. the checksum-record archives Sqx/Wim/Swm/Ace\) — they may still\nback the verbs with a rebuild for convenience, but they do not present themselves as editable\.', + '`CanModify` is withheld from **create-only** formats: a fresh instance can be written, but no supported edit of an existing instance exists. Read-only-on-mount filesystem formats such as CramFS, SquashFS and EROFS may still advertise `CanModify` when the workbench implements a verified offline edit/rebuild path; the native mount policy and the image-editor API are different concerns.') + + # EWF: expose the acquired medium as the one semantic mutable entry. + p = 'FileFormats/FileFormat.Ewf/EwfFormatDescriptor.cs' + sub(p, + r'public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable \{', + 'public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable {') + sub(p, + r'FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanTest \|\n FormatCapabilities\.CanCreate \| FormatCapabilities\.SupportsMultipleEntries;', + 'FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |\n' + ' FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries;') + sub(p, + r'var entries = new List<\(string, byte\[\], string\)> \{\n \("metadata\.ini", BuildMetadata\(img\), "stored"\),\n \};', + 'var entries = new List<(string, byte[], string)> {\n ("metadata.ini", BuildMetadata(img), "stored"),\n };\n if (EwfMedia.TryExtract(img, out var medium))\n entries.Add(("media.raw", medium, "stored"));') + marker = ' private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) {' + value = text(p) + if marker not in value: + raise SystemExit('EWF insertion marker missing') + methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (files.Count != 1) + throw new ArgumentException("EWF mutation accepts exactly one replacement medium.", nameof(inputs)); + var media = files[0].ReadContent(); + var rebuilt = new EwfWriter().Build(media); + archive.Position = 0; + archive.SetLength(0); + archive.Write(rebuilt); + archive.Position = 0; + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (!entryNames.Any(n => string.Equals(n, "media.raw", StringComparison.OrdinalIgnoreCase))) + throw new NotSupportedException("EWF diagnostic sections are derived metadata and cannot be removed independently; remove media.raw to clear the acquired medium."); + var rebuilt = new EwfWriter().Build([]); + archive.Position = 0; + archive.SetLength(0); + archive.Write(rebuilt); + archive.Position = 0; + } + +''' + value = value.replace(marker, methods + marker, 1) + write(p, value) + + # Remove this workflow from the result; it exists only to apply exact small patches. + Path('.github/workflows/rw-promotion-once.yml').unlink() + - name: Commit patched sources + shell: bash + run: | + set -euo pipefail + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add -A + git commit -m '+ promote rebuild-backed existing-image mutation to R/W' + git push origin HEAD:feat/filesystem-rw-gaps From 8bb2694922cb6d165a94a7841b58e46ed04852b4 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:43:00 +0200 Subject: [PATCH 028/192] ci: split one-shot patch script from workflow --- .github/rw-promotion.py | 201 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 .github/rw-promotion.py diff --git a/.github/rw-promotion.py b/.github/rw-promotion.py new file mode 100644 index 000000000..ef2689607 --- /dev/null +++ b/.github/rw-promotion.py @@ -0,0 +1,201 @@ +from pathlib import Path +import re + + +def text(path: str) -> str: + return Path(path).read_text(encoding="utf-8") + + +def write(path: str, value: str) -> None: + Path(path).write_text(value, encoding="utf-8") + + +def sub(path: str, pattern: str, replacement: str, count: int = 1, flags: int = 0) -> None: + value = text(path) + new, n = re.subn(pattern, replacement, value, count=count, flags=flags) + if n != count: + raise SystemExit(f"{path}: expected {count} replacement(s), got {n}: {pattern}") + write(path, new) + + +# CramFS: native mount is read-only; the workbench already has a verified +# extract/edit/re-create modifier, so expose that existing-image edit. +p = "FileSystems/FileSystem.CramFs/CramFsFormatDescriptor.cs" +sub( + p, + r" // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;", + " // The on-disk filesystem is read-only when mounted, but CompressionWorkbench\n" + " // supports existing-image add/replace/remove by verified relayout/rebuild.\n" + " public FormatCapabilities Capabilities =>\n" + " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" + " FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n" + " FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;", + flags=re.S, +) +sub( + p, + r'public string Description => "Linux compressed ROM filesystem";', + 'public string Description => "Linux compressed ROM filesystem; offline image mutation is rebuild-backed.";', +) + +# SquashFS: same distinction — read-only mount format, editable image. +p = "FileSystems/FileSystem.SquashFs/SquashFsFormatDescriptor.cs" +sub( + p, + r" // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;", + " // SquashFS is read-only when mounted, but the workbench can edit an existing\n" + " // image by a verified extract/edit/re-create pass. That is R/W at this API.\n" + " public FormatCapabilities Capabilities =>\n" + " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" + " FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n" + " FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;", + flags=re.S, +) +sub( + p, + r'public string Description => "Linux compressed read-only filesystem";', + 'public string Description => "Linux compressed read-only-on-mount filesystem; offline image mutation is rebuild-backed.";', +) + +# EROFS: promote the fully decoded writer/reader profile. Explicit Add/Remove use +# ReadEntries, which throws on an unsupported compressed inode and therefore never +# feeds the descriptor's user-facing placeholder into a rebuilt image. +p = "FileSystems/FileSystem.Erofs/ErofsFormatDescriptor.cs" +sub(p, r"the round-trippable WORM subset", "the round-trippable offline R/W subset") +sub( + p, + r"FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \|", + "FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" + " FormatCapabilities.CanModify | FormatCapabilities.CanTest |", +) +sub( + p, + r'public string Description => "Android read-only compressed filesystem; uncompressed \+ inline inode layouts\.";', + 'public string Description => "Android read-only-on-mount filesystem; supported uncompressed/inline profile is offline R/W.";', +) +marker = " private static ErofsReader OpenReader(Stream stream) {" +value = text(p) +if marker not in value: + raise SystemExit("EROFS insertion marker missing") +methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + archive.Position = 0; + var label = OpenReader(archive).VolumeName; + archive.Position = 0; + ModifyRebuilder.Add(archive, inputs, + readEntries: stream => ReadEntries(stream), + buildImage: files => { + var writer = new ErofsWriter { VolumeName = label }; + foreach (var (name, data) in files) writer.AddFile(name, data); + return writer.Build(); + }); + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + archive.Position = 0; + var label = OpenReader(archive).VolumeName; + archive.Position = 0; + ModifyRebuilder.Remove(archive, entryNames, + readEntries: stream => ReadEntries(stream), + buildImage: files => { + var writer = new ErofsWriter { VolumeName = label }; + foreach (var (name, data) in files) writer.AddFile(name, data); + return writer.Build(); + }); + } + +''' +value = value.replace(marker, methods + marker, 1) +write(p, value) + +# MSA already has functional Add/Remove against its decoded GEMDOS volume. +p = "FileSystems/FileSystem.Msa/MsaFormatDescriptor.cs" +sub( + p, + r" // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest;", + " // Existing MSA images are editable through the decoded GEMDOS volume and\n" + " // then re-encoded; physical rebuild does not make the public operation WORM.\n" + " public FormatCapabilities Capabilities =>\n" + " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" + " FormatCapabilities.CanModify | FormatCapabilities.CanTest;", + flags=re.S, +) + +# PFS0 has a real existing-container editor; its capability flag was simply stale. +p = "FileFormats/FileFormat.Pfs0/Pfs0FormatDescriptor.cs" +sub( + p, + r" // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries;", + " // Existing PFS0 archives support add/replace/remove through Pfs0InPlaceModifier.\n" + " public FormatCapabilities Capabilities =>\n" + " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" + " FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;", + flags=re.S, +) + +# Archive-model docs must agree with FormatCapabilities and the package README. +p = "docs/ARCHIVE-MODEL.md" +sub( + p, + r"`CanModify` is \*\*withheld\*\* only from \*\*read-only-by-design\*\* formats \(CramFS, SquashFS\) and\n\*\*create-only\*\* formats \(e\.g\. the checksum-record archives Sqx/Wim/Swm/Ace\) — they may still\nback the verbs with a rebuild for convenience, but they do not present themselves as editable\.", + "`CanModify` is withheld from **create-only** formats: a fresh instance can be written, but no supported edit of an existing instance exists. Read-only-on-mount filesystem formats such as CramFS, SquashFS and EROFS may still advertise `CanModify` when the workbench implements a verified offline edit/rebuild path; the native mount policy and the image-editor API are different concerns.", +) + +# EWF: expose the acquired medium as the one semantic mutable entry. +p = "FileFormats/FileFormat.Ewf/EwfFormatDescriptor.cs" +sub( + p, + r"public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable \{", + "public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable {", +) +sub( + p, + r"FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanTest \|\n FormatCapabilities\.CanCreate \| FormatCapabilities\.SupportsMultipleEntries;", + "FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |\n" + " FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries;", +) +sub( + p, + r"var entries = new List<\(string, byte\[\], string\)> \{\n \(\"metadata\.ini\", BuildMetadata\(img\), \"stored\"\),\n \};", + 'var entries = new List<(string, byte[], string)> {\n ("metadata.ini", BuildMetadata(img), "stored"),\n };\n if (EwfMedia.TryExtract(img, out var medium))\n entries.Add(("media.raw", medium, "stored"));', +) +marker = " private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) {" +value = text(p) +if marker not in value: + raise SystemExit("EWF insertion marker missing") +methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (files.Count != 1) + throw new ArgumentException("EWF mutation accepts exactly one replacement medium.", nameof(inputs)); + var media = files[0].ReadContent(); + var rebuilt = new EwfWriter().Build(media); + archive.Position = 0; + archive.SetLength(0); + archive.Write(rebuilt); + archive.Position = 0; + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (!entryNames.Any(n => string.Equals(n, "media.raw", StringComparison.OrdinalIgnoreCase))) + throw new NotSupportedException("EWF diagnostic sections are derived metadata and cannot be removed independently; remove media.raw to clear the acquired medium."); + var rebuilt = new EwfWriter().Build([]); + archive.Position = 0; + archive.SetLength(0); + archive.Write(rebuilt); + archive.Position = 0; + } + +''' +value = value.replace(marker, methods + marker, 1) +write(p, value) + +# Delete the one-shot machinery from the final branch. +Path(".github/workflows/rw-promotion-once.yml").unlink(missing_ok=True) +Path(".github/rw-promotion.py").unlink(missing_ok=True) From f747dac0bef641523f59b23b54f1d48175c8d54f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:43:11 +0200 Subject: [PATCH 029/192] ci: simplify one-shot R/W patch runner --- .github/workflows/rw-promotion-once.yml | 186 ++---------------------- 1 file changed, 10 insertions(+), 176 deletions(-) diff --git a/.github/workflows/rw-promotion-once.yml b/.github/workflows/rw-promotion-once.yml index 8911d97c0..1c76b7642 100644 --- a/.github/workflows/rw-promotion-once.yml +++ b/.github/workflows/rw-promotion-once.yml @@ -2,196 +2,30 @@ name: R/W promotion one-shot on: push: - branches: [feat/filesystem-rw-gaps] + branches: + - feat/filesystem-rw-gaps permissions: contents: write +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + jobs: patch: - if: github.actor != 'github-actions[bot]' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 ref: feat/filesystem-rw-gaps - - name: Patch capability declarations - shell: python - run: | - from pathlib import Path - import re - - def text(path): - return Path(path).read_text(encoding='utf-8') - def write(path, value): - Path(path).write_text(value, encoding='utf-8') - def sub(path, pattern, replacement, count=1, flags=0): - value = text(path) - new, n = re.subn(pattern, replacement, value, count=count, flags=flags) - if n != count: - raise SystemExit(f'{path}: expected {count} replacement(s), got {n}: {pattern}') - write(path, new) - - # CramFS: native mount is read-only; the workbench already has a verified - # extract/edit/re-create modifier, so expose that existing-image edit. - p = 'FileSystems/FileSystem.CramFs/CramFsFormatDescriptor.cs' - sub(p, - r' // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;', - ' // The on-disk filesystem is read-only when mounted, but CompressionWorkbench\n' - ' // supports existing-image add/replace/remove by verified relayout/rebuild.\n' - ' public FormatCapabilities Capabilities =>\n' - ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' - ' FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n' - ' FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;', - flags=re.S) - sub(p, r'public string Description => "Linux compressed ROM filesystem";', - 'public string Description => "Linux compressed ROM filesystem; offline image mutation is rebuild-backed.";') - - # SquashFS: same distinction — read-only mount format, editable image. - p = 'FileSystems/FileSystem.SquashFs/SquashFsFormatDescriptor.cs' - sub(p, - r' // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;', - ' // SquashFS is read-only when mounted, but the workbench can edit an existing\n' - ' // image by a verified extract/edit/re-create pass. That is R/W at this API.\n' - ' public FormatCapabilities Capabilities =>\n' - ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' - ' FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n' - ' FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;', - flags=re.S) - sub(p, r'public string Description => "Linux compressed read-only filesystem";', - 'public string Description => "Linux compressed read-only-on-mount filesystem; offline image mutation is rebuild-backed.";') - - # EROFS: promote only the writer/reader profile. Add/Remove below use ReadEntries, - # which throws on an undecodable compressed inode instead of materialising the - # descriptor\'s user-facing placeholder and losing content. - p = 'FileSystems/FileSystem.Erofs/ErofsFormatDescriptor.cs' - sub(p, r'the round-trippable WORM subset', 'the round-trippable offline R/W subset') - sub(p, - r'FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \|', - 'FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' - ' FormatCapabilities.CanModify | FormatCapabilities.CanTest |') - sub(p, r'public string Description => "Android read-only compressed filesystem; uncompressed \+ inline inode layouts\.";', - 'public string Description => "Android read-only-on-mount filesystem; supported uncompressed/inline profile is offline R/W.";') - marker = ' private static ErofsReader OpenReader(Stream stream) {' - value = text(p) - if marker not in value: - raise SystemExit('EROFS insertion marker missing') - methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - archive.Position = 0; - var label = OpenReader(archive).VolumeName; - archive.Position = 0; - ModifyRebuilder.Add(archive, inputs, - readEntries: stream => ReadEntries(stream), - buildImage: files => { - var writer = new ErofsWriter { VolumeName = label }; - foreach (var (name, data) in files) writer.AddFile(name, data); - return writer.Build(); - }); - } - - public void Remove(Stream archive, string[] entryNames) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - archive.Position = 0; - var label = OpenReader(archive).VolumeName; - archive.Position = 0; - ModifyRebuilder.Remove(archive, entryNames, - readEntries: stream => ReadEntries(stream), - buildImage: files => { - var writer = new ErofsWriter { VolumeName = label }; - foreach (var (name, data) in files) writer.AddFile(name, data); - return writer.Build(); - }); - } - -''' - value = value.replace(marker, methods + marker, 1) - write(p, value) - - # MSA already has functional Add/Remove against the inner GEMDOS image. - p = 'FileSystems/FileSystem.Msa/MsaFormatDescriptor.cs' - sub(p, - r' // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest;', - ' // Existing MSA images are editable through the decoded GEMDOS volume and\n' - ' // then re-encoded; physical rebuild does not make the public operation WORM.\n' - ' public FormatCapabilities Capabilities =>\n' - ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' - ' FormatCapabilities.CanModify | FormatCapabilities.CanTest;', - flags=re.S) - - # PFS0 has had a real existing-container editor for a while; its flag was stale. - p = 'FileFormats/FileFormat.Pfs0/Pfs0FormatDescriptor.cs' - sub(p, - r' // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries;', - ' // Existing PFS0 archives support add/replace/remove through Pfs0InPlaceModifier.\n' - ' public FormatCapabilities Capabilities =>\n' - ' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n' - ' FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;', - flags=re.S) - - # Archive-model docs must agree with FormatCapabilities and the package README. - p = 'docs/ARCHIVE-MODEL.md' - sub(p, - r'`CanModify` is \*\*withheld\*\* only from \*\*read-only-by-design\*\* formats \(CramFS, SquashFS\) and\n\*\*create-only\*\* formats \(e\.g\. the checksum-record archives Sqx/Wim/Swm/Ace\) — they may still\nback the verbs with a rebuild for convenience, but they do not present themselves as editable\.', - '`CanModify` is withheld from **create-only** formats: a fresh instance can be written, but no supported edit of an existing instance exists. Read-only-on-mount filesystem formats such as CramFS, SquashFS and EROFS may still advertise `CanModify` when the workbench implements a verified offline edit/rebuild path; the native mount policy and the image-editor API are different concerns.') - - # EWF: expose the acquired medium as the one semantic mutable entry. - p = 'FileFormats/FileFormat.Ewf/EwfFormatDescriptor.cs' - sub(p, - r'public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable \{', - 'public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable {') - sub(p, - r'FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanTest \|\n FormatCapabilities\.CanCreate \| FormatCapabilities\.SupportsMultipleEntries;', - 'FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |\n' - ' FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries;') - sub(p, - r'var entries = new List<\(string, byte\[\], string\)> \{\n \("metadata\.ini", BuildMetadata\(img\), "stored"\),\n \};', - 'var entries = new List<(string, byte[], string)> {\n ("metadata.ini", BuildMetadata(img), "stored"),\n };\n if (EwfMedia.TryExtract(img, out var medium))\n entries.Add(("media.raw", medium, "stored"));') - marker = ' private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) {' - value = text(p) - if marker not in value: - raise SystemExit('EWF insertion marker missing') - methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - var files = inputs.Where(i => !i.IsDirectory).ToList(); - if (files.Count != 1) - throw new ArgumentException("EWF mutation accepts exactly one replacement medium.", nameof(inputs)); - var media = files[0].ReadContent(); - var rebuilt = new EwfWriter().Build(media); - archive.Position = 0; - archive.SetLength(0); - archive.Write(rebuilt); - archive.Position = 0; - } - - public void Remove(Stream archive, string[] entryNames) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - if (!entryNames.Any(n => string.Equals(n, "media.raw", StringComparison.OrdinalIgnoreCase))) - throw new NotSupportedException("EWF diagnostic sections are derived metadata and cannot be removed independently; remove media.raw to clear the acquired medium."); - var rebuilt = new EwfWriter().Build([]); - archive.Position = 0; - archive.SetLength(0); - archive.Write(rebuilt); - archive.Position = 0; - } - -''' - value = value.replace(marker, methods + marker, 1) - write(p, value) - - # Remove this workflow from the result; it exists only to apply exact small patches. - Path('.github/workflows/rw-promotion-once.yml').unlink() + - name: Apply exact source patches + run: python .github/rw-promotion.py - name: Commit patched sources - shell: bash run: | set -euo pipefail - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A - git commit -m '+ promote rebuild-backed existing-image mutation to R/W' + git commit -m "+ promote rebuild-backed existing-image mutation to R/W" git push origin HEAD:feat/filesystem-rw-gaps From 3981851afa93fba9fad71ad7bb5f0fa76a6ce334 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:50:25 +0200 Subject: [PATCH 030/192] ci: make one-shot descriptions case tolerant --- .github/rw-promotion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/rw-promotion.py b/.github/rw-promotion.py index ef2689607..979387fc6 100644 --- a/.github/rw-promotion.py +++ b/.github/rw-promotion.py @@ -34,7 +34,7 @@ def sub(path: str, pattern: str, replacement: str, count: int = 1, flags: int = ) sub( p, - r'public string Description => "Linux compressed ROM filesystem";', + r'public string Description => "Linux [Cc]ompressed ROM filesystem";', 'public string Description => "Linux compressed ROM filesystem; offline image mutation is rebuild-backed.";', ) From 56b3e15247cf01bac6f56d373b4822bc534e411d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:54:18 +0200 Subject: [PATCH 031/192] + add conservative offline ReFS file mutation --- .../FileSystem.Refs/RefsOfflineModifier.cs | 395 ++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 FileSystems/FileSystem.Refs/RefsOfflineModifier.cs diff --git a/FileSystems/FileSystem.Refs/RefsOfflineModifier.cs b/FileSystems/FileSystem.Refs/RefsOfflineModifier.cs new file mode 100644 index 000000000..3c5704aaa --- /dev/null +++ b/FileSystems/FileSystem.Refs/RefsOfflineModifier.cs @@ -0,0 +1,395 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using Compression.Registry; + +namespace FileSystem.Refs; + +/// +/// Offline-quiescent ReFS 3.x regular-file editor. +/// +/// This is deliberately not a mounted-driver transaction layer. It operates on an +/// unmounted image, reopens the active metadata graph between logical edits, uses +/// allocator-verified storage for replacement data, and uses the existing immutable +/// CoW B+ engine + alternate checkpoint publisher for namespace deletion so parent +/// separator keys remain correct even when a leaf disappears. +/// +/// Supported profile: +/// - replace existing regular files whose live stream layout is resident or an +/// ordinary non-sparse/non-integrity/non-shared extent holder understood by +/// ; +/// - remove regular files and empty directories; +/// - release old ordinary data extents after the namespace/data repoint is live. +/// +/// New-name insertion remains fail-closed until the directory-entry value template +/// (file identity/security/link semantics) is derived for every writable ReFS 3.x +/// profile. Replacing an existing name through is fully supported. +/// +internal static class RefsOfflineModifier { + private const ulong RootDirectoryOid = 0x600; + + public static void Add(Stream image, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(inputs); + RequireWritableImage(image); + + foreach (var input in inputs) { + if (input.IsDirectory) continue; + var path = NormalizePath(input.ArchiveName); + if (path.Length == 0) + throw new ArgumentException("ReFS entry path must not be empty.", nameof(inputs)); + + var metadata = RefsMetadataReader.Open(image); + var existing = new RefsNamespaceReader(metadata).ReadAll().FirstOrDefault(f => + !f.IsDirectory && string.Equals(f.Path, path, StringComparison.OrdinalIgnoreCase)); + if (existing == null) + throw new NotSupportedException( + $"ReFS offline R/W currently replaces existing regular files; creating the new namespace entry '{path}' " + + "is withheld until all file-identity/security/link fields are proven for the active ReFS profile."); + + ReplaceExisting(image, path, input.ReadContent()); + } + } + + public static void Remove(Stream image, string[] entryNames) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(entryNames); + RequireWritableImage(image); + + foreach (var raw in entryNames) { + var path = NormalizePath(raw); + if (path.Length == 0) continue; + RemoveOne(image, path); + } + } + + private static void ReplaceExisting(Stream image, string path, byte[] data) { + var metadata = RefsMetadataReader.Open(image); + var files = new RefsNamespaceReader(metadata).ReadAll(); + var file = files.FirstOrDefault(f => + !f.IsDirectory && string.Equals(f.Path, path, StringComparison.OrdinalIgnoreCase)) + ?? throw new FileNotFoundException($"ReFS file '{path}' is no longer reachable.", path); + if (file.Extents.Any(e => e.IsSparse || e.Flags == 0x1C00D0 || (e.Flags & 0x04) != 0)) + throw new NotSupportedException( + $"ReFS file '{path}' uses sparse/integrity/shared allocation semantics outside the offline CRUD profile."); + + var graph = new RefsMetadataGraph(image, metadata); + var writable = new RefsWritableNamespace(metadata); + var location = writable.ResolveStorage(path); + + var clusterSize = metadata.ClusterSize; + var blocks = checked((data.LongLength + clusterSize - 1) / clusterSize); + if (blocks > int.MaxValue) + throw new NotSupportedException("ReFS replacement exceeds the supported allocation-run size."); + + var targets = blocks == 0 + ? Array.Empty() + : SelectContiguousFreeDataRun(metadata, graph, checked((int)blocks)); + var targetOffsets = targets.Select(lcn => checked((long)lcn * clusterSize)).ToArray(); + var extents = RefsStreamLayoutEditor.BuildExtents(metadata, targetOffsets); + var allocatedBytes = checked((long)targets.Length * clusterSize); + var replacementFile = file with { + Size = data.LongLength, + AllocatedSize = allocatedBytes, + }; + var replacementValue = RefsStreamLayoutEditor.BuildUpdatedValue( + replacementFile, + location.StorageRow, + extents, + clusterSize); + + if (!RefsPageEditor.CanReplaceValue(graph, location.StorageRow, replacementValue.Length)) + throw new NotSupportedException( + $"ReFS replacement for '{path}' would require an outer B+ page split; this offline value-repoint path refuses before allocating data."); + + RefsBTreeRow? shortEntry = null; + byte[]? shortEntryValue = null; + if (location.UsesBackingRow) { + shortEntry = writable.FindDirectoryEntry(path); + if (shortEntry.Value.Length < 0x40) + throw new InvalidDataException("ReFS short directory entry is too small for size/allocation fields."); + shortEntryValue = shortEntry.Value.ToArray(); + BinaryPrimitives.WriteUInt64LittleEndian(shortEntryValue.AsSpan(0x30, 8), checked((ulong)allocatedBytes)); + BinaryPrimitives.WriteUInt64LittleEndian(shortEntryValue.AsSpan(0x38, 8), checked((ulong)data.LongLength)); + if (shortEntryValue.Length >= 0x20) + BinaryPrimitives.WriteUInt64LittleEndian(shortEntryValue.AsSpan(0x18, 8), checked((ulong)DateTime.UtcNow.ToFileTimeUtc())); + if (!RefsPageEditor.CanReplaceValue(graph, shortEntry, shortEntryValue.Length)) + throw new NotSupportedException( + $"ReFS parent entry for '{path}' cannot be updated without an outer B+ split."); + } + + var newClaimed = false; + var metadataRepointed = false; + try { + if (targets.Length > 0) { + var allocator = FindAllocator(metadata, graph, targets[0]); + if (!targets.All(allocator.CoversPhysical)) + throw new InvalidDataException("ReFS replacement run crosses allocator ownership boundaries."); + allocator.SetAllocated(targets, allocated: true); + image.Flush(); + newClaimed = true; + WriteData(image, data, targets, clusterSize); + image.Flush(); + } + + var changedPages = new HashSet { + RefsPageEditor.ReplaceValue(graph, location.StorageRow, replacementValue), + }; + metadataRepointed = true; + if (shortEntry != null && shortEntryValue != null) + changedPages.Add(RefsPageEditor.ReplaceValue(graph, shortEntry, shortEntryValue)); + graph.RefreshChecksumPaths(changedPages); + image.Flush(); + } catch { + // Before the stream metadata points at the new allocation, the new run is + // merely an orphan reservation and can be released. After the repoint, a + // leak is safer than freeing bytes that may already be reachable. + if (newClaimed && !metadataRepointed) + TryReleaseAllocation(image, targets); + throw; + } + + ReleaseOldData(image, file); + } + + private static void RemoveOne(Stream image, string path) { + var metadata = RefsMetadataReader.Open(image); + var files = new RefsNamespaceReader(metadata).ReadAll(); + var file = files.FirstOrDefault(f => string.Equals(f.Path, path, StringComparison.OrdinalIgnoreCase)) + ?? throw new FileNotFoundException($"ReFS entry '{path}' was not found.", path); + + if (file.IsDirectory && files.Any(f => + !string.Equals(f.Path, path, StringComparison.OrdinalIgnoreCase) + && f.Path.StartsWith(path.TrimEnd('/') + "/", StringComparison.OrdinalIgnoreCase))) + throw new IOException($"ReFS directory '{path}' is not empty."); + if (!file.IsDirectory && file.Extents.Any(e => e.IsSparse || e.Flags == 0x1C00D0 || (e.Flags & 0x04) != 0)) + throw new NotSupportedException( + $"ReFS file '{path}' uses sparse/integrity/shared allocation semantics outside the offline CRUD profile."); + + var parent = ResolveParentDirectory(metadata, path); + var writable = new RefsWritableNamespace(metadata); + var entry = writable.FindDirectoryEntry(path); + var keys = new List { entry.Key.ToArray() }; + if (!file.IsDirectory && file.Backing != null) + keys.Add(file.Backing.Row.Key.ToArray()); + + var store = new RefsCowPageStore(image, metadata); + var tree = new RefsCowBTree(image, metadata, store); + var parentTree = tree.Rewrite(parent.Root, virtualAddresses: true, (rows, comparer) => { + var removed = 0; + foreach (var key in keys) { + var index = FindKey(rows, key, comparer); + if (index < 0) continue; + rows.RemoveAt(index); + ++removed; + } + if (removed != keys.Count) + throw new InvalidDataException( + $"ReFS namespace/storage rows for '{path}' changed before deletion could be materialized."); + return true; + }); + + var objectEditor = new RefsCowObjectEditor(metadata, tree); + var objectTable = objectEditor.ReplaceObjectRoot(parent.ObjectId, parentTree.RootReference); + PublishOfflineCheckpoint(image, metadata, store, objectTable); + + if (!file.IsDirectory) + ReleaseOldData(image, file); + } + + /// + /// Publishes immutable namespace/Object-Table pages and the allocator roots that + /// account for their newly reserved metadata pages. There is intentionally no + /// synthetic MLog redo record here: this is the offline-quiescent transaction + /// boundary, not the native mounted-driver crash-recovery path. + /// + private static void PublishOfflineCheckpoint( + Stream image, + RefsMetadataReader metadata, + RefsCowPageStore store, + RefsCowTreeResult objectTable) { + var roots = new Dictionary { [0] = objectTable.RootReference }; + var allocatorChanged = false; + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + if (store.GetReservedClusters(tier).Count == 0) continue; + var publication = new RefsCowAllocatorPublisher(image, metadata, store).Publish(tier); + roots[publication.RootIndex] = publication.Tree.RootReference; + allocatorChanged = true; + } + + var committer = new RefsCheckpointCommitter(image); + var prepared = committer.PrepareNext(); + committer.SetRootReferences(prepared, roots); + committer.Commit(prepared, allocatorChanged: allocatorChanged); + } + + private static ParentDirectory ResolveParentDirectory(RefsMetadataReader metadata, string path) { + var parts = NormalizePath(path).Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0) throw new ArgumentException("ReFS path is empty.", nameof(path)); + + var objects = BuildObjectMap(metadata); + if (!objects.TryGetValue(RootDirectoryOid, out var root)) + throw new InvalidDataException("ReFS root directory object is absent from the Object Table."); + var oid = RootDirectoryOid; + + for (var i = 0; i < parts.Length - 1; ++i) { + RefsBTreeRow? found = null; + foreach (var row in metadata.WalkTree(root, virtualAddresses: true)) { + if (row.Key.Length < 4 || BinaryPrimitives.ReadUInt16LittleEndian(row.Key.AsSpan(0, 2)) != 0x30) + continue; + var name = DecodeName(row.Key.AsSpan(4)); + if (!string.Equals(name, parts[i], StringComparison.OrdinalIgnoreCase)) continue; + if (found != null) + throw new InvalidDataException($"ReFS directory '{parts[i]}' is ambiguous."); + found = row; + } + if (found == null || found.Value.Length < 0x44) + throw new DirectoryNotFoundException($"ReFS parent directory component '{parts[i]}' was not found."); + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(found.Value.AsSpan(0x40, 4)); + if ((attributes & 0x10000000) == 0) + throw new DirectoryNotFoundException($"ReFS path component '{parts[i]}' is not a directory."); + oid = BinaryPrimitives.ReadUInt64LittleEndian(found.Value.AsSpan(0x08, 8)); + if (!objects.TryGetValue(oid, out root)) + throw new InvalidDataException($"ReFS child directory OID 0x{oid:X} has no Object Table root."); + } + return new ParentDirectory(oid, root); + } + + private static Dictionary BuildObjectMap(RefsMetadataReader metadata) { + var result = new Dictionary(); + foreach (var row in metadata.WalkRoot(0)) { + if (row.Key.Length < 16 || row.Value.Length < 0x20 + metadata.PageReferenceSize) continue; + var oid = BinaryPrimitives.ReadUInt64LittleEndian(row.Key.AsSpan(8, 8)); + var reference = RefsPageReference.Parse(row.Value.AsSpan(0x20)); + if (reference.Lcns.Count > 0) result[oid] = reference; + } + return result; + } + + private static int FindKey(IReadOnlyList rows, byte[] key, RefsKeyComparer comparer) { + for (var i = 0; i < rows.Count; ++i) + if (comparer.Compare(rows[i].Key, key) == 0) return i; + return -1; + } + + private static ulong[] SelectContiguousFreeDataRun( + RefsMetadataReader metadata, + RefsMetadataGraph graph, + int count) { + var medium = new RefsAllocatorWriter(metadata, graph, RefsAllocatorTier.Medium); + if (!medium.TryFindFreeRun(count, out var start)) + throw new IOException($"ReFS Medium Allocator has no verified contiguous free run of {count:N0} cluster(s)."); + var result = new ulong[count]; + for (var i = 0; i < count; ++i) result[i] = checked(start + (ulong)i); + return result; + } + + private static RefsAllocatorWriter FindAllocator( + RefsMetadataReader metadata, + RefsMetadataGraph graph, + ulong physicalLcn) { + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + var writer = new RefsAllocatorWriter(metadata, graph, tier); + if (writer.CoversPhysical(physicalLcn)) return writer; + } + throw new InvalidDataException($"No ReFS allocator tier covers PLCN 0x{physicalLcn:X}."); + } + + private static void WriteData(Stream image, byte[] data, IReadOnlyList targets, int clusterSize) { + var cursor = 0; + var buffer = new byte[clusterSize]; + foreach (var lcn in targets) { + buffer.AsSpan().Clear(); + var take = Math.Min(clusterSize, data.Length - cursor); + if (take > 0) data.AsSpan(cursor, take).CopyTo(buffer); + image.Position = checked((long)lcn * clusterSize); + image.Write(buffer); + cursor += take; + } + if (cursor != data.Length) + throw new IOException("ReFS replacement allocation did not receive every source byte."); + } + + private static void ReleaseOldData(Stream image, RefsFileRecord file) { + var old = ExpandPhysicalClusters(file.Extents).Distinct().ToArray(); + if (old.Length == 0) return; + + var metadata = RefsMetadataReader.Open(image); + var graph = new RefsMetadataGraph(image, metadata); + var releasable = new RefsBlockRefcount(metadata, graph).DetachPhysicalReferences(old); + image.Flush(); + if (releasable.Count == 0) return; + + var fresh = RefsMetadataReader.Open(image); + var freshGraph = new RefsMetadataGraph(image, fresh); + var remaining = releasable.ToHashSet(); + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + var writer = new RefsAllocatorWriter(fresh, freshGraph, tier); + var covered = remaining.Where(writer.CoversPhysical).ToArray(); + if (covered.Length == 0) continue; + writer.SetAllocated(covered, allocated: false); + foreach (var lcn in covered) remaining.Remove(lcn); + image.Flush(); + } + // Unknown allocator ownership is intentionally leaked rather than guessed free. + } + + private static IEnumerable ExpandPhysicalClusters(IEnumerable extents) { + foreach (var extent in extents) { + if (extent.IsSparse) continue; + for (uint i = 0; i < extent.ClusterCount; ++i) + yield return checked(extent.PhysicalLcn + i); + } + } + + private static void TryReleaseAllocation(Stream image, IReadOnlyList targets) { + try { + if (targets.Count == 0) return; + var metadata = RefsMetadataReader.Open(image); + var graph = new RefsMetadataGraph(image, metadata); + var remaining = targets.ToHashSet(); + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + var writer = new RefsAllocatorWriter(metadata, graph, tier); + var covered = remaining.Where(writer.CoversPhysical).ToArray(); + if (covered.Length == 0) continue; + writer.SetAllocated(covered, allocated: false); + foreach (var lcn in covered) remaining.Remove(lcn); + } + image.Flush(); + } catch { + // An allocation leak is safer than masking the original exception or + // freeing a range whose publication state became uncertain. + } + } + + private static void RequireWritableImage(Stream image) { + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("ReFS offline mutation requires a readable, writable, seekable unmounted image stream.", nameof(image)); + } + + private static string NormalizePath(string path) + => (path ?? string.Empty).Replace('\\', '/').Trim('/'); + + private static string DecodeName(ReadOnlySpan bytes) { + try { return Encoding.Unicode.GetString(bytes).TrimEnd('\0'); } + catch { return Convert.ToHexString(bytes); } + } + + private sealed record ParentDirectory(ulong ObjectId, RefsPageReference Root); +} From b67f5d8a7afc808ddc798ea291fea84f9ebed914 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:55:00 +0200 Subject: [PATCH 032/192] ci: add exact ReFS descriptor promotion patch --- .github/refs-promotion.py | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/refs-promotion.py diff --git a/.github/refs-promotion.py b/.github/refs-promotion.py new file mode 100644 index 000000000..5076c69f4 --- /dev/null +++ b/.github/refs-promotion.py @@ -0,0 +1,79 @@ +from pathlib import Path +import re + +p = Path("FileSystems/FileSystem.Refs/RefsFormatDescriptor.cs") +s = p.read_text(encoding="utf-8") + +old = """public sealed class RefsFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IFilesystemExtentMap, + IArchiveDefragmentable, + ILayoutOptimizable {""" +new = """public sealed class RefsFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveModifiable, + IFilesystemExtentMap, + IArchiveDefragmentable, + ILayoutOptimizable {""" +if old not in s: + raise SystemExit("ReFS descriptor interface block not found") +s = s.replace(old, new, 1) + +old = """ public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries;""" +new = """ public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanModify | + FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;""" +if old not in s: + raise SystemExit("ReFS capability block not found") +s = s.replace(old, new, 1) + +s = s.replace( + 'public string Description => "Microsoft ReFS 3.x volume image with namespace, allocation, in-place data relocation and filesystem-metadata placement support.";', + 'public string Description => "Microsoft ReFS 3.x volume image; offline-quiescent existing-file replace/remove plus allocation and metadata placement. Native mounted-driver transactions remain a separate readiness tier.";', + 1, +) + +marker = " private static List ListDiagnosticSurface(Stream stream) {" +if marker not in s: + raise SystemExit("ReFS descriptor insertion marker not found") +methods = ''' /// + /// Offline-quiescent existing-file replacement for the proven regular-stream profile. + /// A new name is rejected before mutation until ReFS file-identity/security/link fields + /// are proven for every supported 3.x profile. + /// + public void Add(Stream archive, IReadOnlyList inputs) + => RefsOfflineModifier.Add(archive, inputs); + + /// + /// Removes regular files or empty directories from an unmounted ReFS image. Namespace + /// deletion is published through immutable B+ replacement pages and the alternate CHKP. + /// + public void Remove(Stream archive, string[] entryNames) + => RefsOfflineModifier.Remove(archive, entryNames); + +''' +s = s.replace(marker, methods + marker, 1) +p.write_text(s, encoding="utf-8") + +# The readiness document distinguishes the library's offline image-editor surface +# from full native mounted-driver R/W. Do not erase the latter's remaining work. +p = Path("FileSystems/FileSystem.Refs/DRIVER_READINESS.md") +s = p.read_text(encoding="utf-8") +needle = "- [x] explicit offline-vs-native mutation transaction boundary\n" +if needle not in s: + raise SystemExit("ReFS readiness insertion point not found") +s = s.replace( + needle, + needle + + "- [x] offline-quiescent existing regular-file replacement with allocator-verified reallocation and old-block release\n" + + "- [x] offline-quiescent regular-file / empty-directory removal through CoW B+ replacement + alternate CHKP publication\n" + + "- [x] archive API exposes the proven offline mutation profile without claiming mounted-driver crash semantics\n", + 1, +) +p.write_text(s, encoding="utf-8") + +Path(".github/refs-promotion.py").unlink(missing_ok=True) From e901fd02e4e296afc4cbac4c72c7f816ef825ead Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:55:17 +0200 Subject: [PATCH 033/192] ci: include ReFS descriptor in one-shot promotion --- .github/workflows/rw-promotion-once.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rw-promotion-once.yml b/.github/workflows/rw-promotion-once.yml index 1c76b7642..6072d00e2 100644 --- a/.github/workflows/rw-promotion-once.yml +++ b/.github/workflows/rw-promotion-once.yml @@ -20,12 +20,14 @@ jobs: fetch-depth: 0 ref: feat/filesystem-rw-gaps - name: Apply exact source patches - run: python .github/rw-promotion.py + run: | + python .github/refs-promotion.py + python .github/rw-promotion.py - name: Commit patched sources run: | set -euo pipefail git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A - git commit -m "+ promote rebuild-backed existing-image mutation to R/W" + git commit -m "+ promote offline and rebuild-backed existing-image mutation to R/W" git push origin HEAD:feat/filesystem-rw-gaps From aef7384c43b886ef655145b7b7ac64c3a4d7d98b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 11:55:49 +0200 Subject: [PATCH 034/192] + cover promoted filesystem image mutation paths --- .../FilesystemRwPromotionRoundTripTests.cs | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs diff --git a/Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs b/Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs new file mode 100644 index 000000000..fb2313837 --- /dev/null +++ b/Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs @@ -0,0 +1,149 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace Compression.Tests.Operations; + +[TestFixture] +public sealed class FilesystemRwPromotionRoundTripTests { + [TestCase("CramFs")] + [TestCase("SquashFs")] + [TestCase("Erofs")] + [TestCase("Msa")] + [TestCase("Pfs0")] + [Category("HappyPath"), Category("RoundTrip")] + public void CreateAddRemove_PromotedFormatsKeepSurvivorsByteExact(string formatId) { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var ops = FormatRegistry.GetArchiveOps(formatId); + Assert.That(ops, Is.Not.Null, formatId); + Assert.That(ops, Is.InstanceOf(), formatId); + Assert.That(ops, Is.InstanceOf(), formatId); + + var descriptor = FormatRegistry.All.Single(d => d.Id == formatId); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True, formatId); + + var creator = (IArchiveCreatable)ops!; + var modifier = (IArchiveModifiable)ops; + var archiveOps = (IArchiveFormatOperations)ops; + var a = Enumerable.Range(0, 333).Select(i => (byte)(i * 7)).ToArray(); + var b = Enumerable.Range(0, 517).Select(i => (byte)(i * 11)).ToArray(); + var c = Enumerable.Range(0, 129).Select(i => (byte)(255 - i)).ToArray(); + + using var image = new MemoryStream(); + creator.Create(image, [ + ArchiveInputInfo.InMemory("A.TXT", a), + ArchiveInputInfo.InMemory("B.BIN", b), + ], new FormatCreateOptions()); + Assert.That(image.Length, Is.GreaterThan(0), formatId); + + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory("C.DAT", c)]); + Assert.That(ListNames(archiveOps, image), Does.Contain("C.DAT").IgnoreCase, formatId); + + image.Position = 0; + modifier.Remove(image, ["B.BIN"]); + var names = ListNames(archiveOps, image); + Assert.Multiple(() => { + Assert.That(names.Any(n => Matches(n, "A.TXT")), Is.True, $"{formatId}: A.TXT disappeared"); + Assert.That(names.Any(n => Matches(n, "B.BIN")), Is.False, $"{formatId}: B.BIN survived removal"); + Assert.That(names.Any(n => Matches(n, "C.DAT")), Is.True, $"{formatId}: C.DAT disappeared"); + }); + + var work = Path.Combine(Path.GetTempPath(), "cwb_rw_promote_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try { + image.Position = 0; + archiveOps.Extract(image, work, null, null); + Assert.That(FindExtracted(work, "A.TXT"), Is.EqualTo(a), $"{formatId}: A.TXT changed"); + Assert.That(FindExtracted(work, "C.DAT"), Is.EqualTo(c), $"{formatId}: C.DAT changed"); + } finally { + try { Directory.Delete(work, recursive: true); } catch { } + } + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Ewf_ReplaceAndClearSemanticMedia_RoundTrips() { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var ops = FormatRegistry.GetArchiveOps("Ewf")!; + var descriptor = FormatRegistry.All.Single(d => d.Id == "Ewf"); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(ops, Is.InstanceOf()); + Assert.That(ops, Is.InstanceOf()); + + var original = Enumerable.Range(0, 300_123).Select(i => (byte)(i * 13)).ToArray(); + var replacement = Enumerable.Range(0, 130_777).Select(i => (byte)(i * 17)).ToArray(); + using var image = new MemoryStream(); + ((IArchiveCreatable)ops).Create(image, + [ArchiveInputInfo.InMemory("capture.raw", original)], new FormatCreateOptions()); + + Assert.That(ReadNamed((IArchiveFormatOperations)ops, image, "media.raw"), Is.EqualTo(original)); + + image.Position = 0; + ((IArchiveModifiable)ops).Add(image, [ArchiveInputInfo.InMemory("media.raw", replacement)]); + Assert.That(ReadNamed((IArchiveFormatOperations)ops, image, "media.raw"), Is.EqualTo(replacement)); + + image.Position = 0; + ((IArchiveModifiable)ops).Remove(image, ["media.raw"]); + var names = ListNames((IArchiveFormatOperations)ops, image); + Assert.That(names.Any(n => Matches(n, "media.raw")), Is.True, + "An empty EWF still represents a zero-length acquired medium."); + Assert.That(ReadNamed((IArchiveFormatOperations)ops, image, "media.raw"), Is.Empty); + } + + [Test, Category("HappyPath")] + public void Refs_AdvertisesScopedOfflineMutationAndFailsClosedOnDiagnosticImage() { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var ops = FormatRegistry.GetArchiveOps("Refs")!; + var descriptor = FormatRegistry.All.Single(d => d.Id == "Refs"); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(ops, Is.InstanceOf()); + + // A header-only diagnostic image has no live namespace. The modifier must + // reject it before touching bytes rather than treating FULL.refs as content. + var imageBytes = BuildMinimalRefsHeader(); + using var image = new MemoryStream(imageBytes, writable: true); + var before = image.ToArray(); + Assert.Throws(() => + ((IArchiveModifiable)ops).Add(image, [ArchiveInputInfo.InMemory("A.TXT", [1, 2, 3])])); + Assert.That(image.ToArray(), Is.EqualTo(before)); + } + + private static List ListNames(IArchiveFormatOperations ops, MemoryStream image) { + image.Position = 0; + return ops.List(image, null).Where(e => !e.IsDirectory).Select(e => e.Name).ToList(); + } + + private static byte[] ReadNamed(IArchiveFormatOperations ops, MemoryStream image, string name) { + var work = Path.Combine(Path.GetTempPath(), "cwb_rw_extract_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try { + image.Position = 0; + ops.Extract(image, work, null, [name]); + return FindExtracted(work, name); + } finally { + try { Directory.Delete(work, recursive: true); } catch { } + } + } + + private static byte[] FindExtracted(string root, string name) { + var file = Directory.GetFiles(root, "*", SearchOption.AllDirectories) + .FirstOrDefault(f => Matches(Path.GetFileName(f), name)); + Assert.That(file, Is.Not.Null, $"Expected extracted file {name}"); + return File.ReadAllBytes(file!); + } + + private static bool Matches(string actual, string expected) + => string.Equals(Path.GetFileName(actual.Replace('\\', '/')), expected, StringComparison.OrdinalIgnoreCase); + + private static byte[] BuildMinimalRefsHeader() { + var image = new byte[4096]; + System.Text.Encoding.ASCII.GetBytes("ReFS").CopyTo(image.AsSpan(3)); + System.Text.Encoding.ASCII.GetBytes("FSRS").CopyTo(image.AsSpan(0x10)); + System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(0x14, 2), 0x200); + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(0x18, 8), 1024UL); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(0x20, 4), 512); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(0x24, 4), 8); + image[0x28] = 3; + image[0x29] = 14; + return image; + } +} \ No newline at end of file From 911390081f139d5d259b3a2e61e6ad5b39150919 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:06:57 +0200 Subject: [PATCH 035/192] - remove temporary R/W promotion workflow --- .github/workflows/rw-promotion-once.yml | 33 ------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/rw-promotion-once.yml diff --git a/.github/workflows/rw-promotion-once.yml b/.github/workflows/rw-promotion-once.yml deleted file mode 100644 index 6072d00e2..000000000 --- a/.github/workflows/rw-promotion-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: R/W promotion one-shot - -on: - push: - branches: - - feat/filesystem-rw-gaps - -permissions: - contents: write - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: feat/filesystem-rw-gaps - - name: Apply exact source patches - run: | - python .github/refs-promotion.py - python .github/rw-promotion.py - - name: Commit patched sources - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "+ promote offline and rebuild-backed existing-image mutation to R/W" - git push origin HEAD:feat/filesystem-rw-gaps From 489b877dad565670ced60a0191fa853859a48931 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:07:08 +0200 Subject: [PATCH 036/192] - remove temporary ReFS promotion script --- .github/refs-promotion.py | 79 --------------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 .github/refs-promotion.py diff --git a/.github/refs-promotion.py b/.github/refs-promotion.py deleted file mode 100644 index 5076c69f4..000000000 --- a/.github/refs-promotion.py +++ /dev/null @@ -1,79 +0,0 @@ -from pathlib import Path -import re - -p = Path("FileSystems/FileSystem.Refs/RefsFormatDescriptor.cs") -s = p.read_text(encoding="utf-8") - -old = """public sealed class RefsFormatDescriptor : - IFormatDescriptor, - IArchiveFormatOperations, - IFilesystemExtentMap, - IArchiveDefragmentable, - ILayoutOptimizable {""" -new = """public sealed class RefsFormatDescriptor : - IFormatDescriptor, - IArchiveFormatOperations, - IArchiveModifiable, - IFilesystemExtentMap, - IArchiveDefragmentable, - ILayoutOptimizable {""" -if old not in s: - raise SystemExit("ReFS descriptor interface block not found") -s = s.replace(old, new, 1) - -old = """ public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | - FormatCapabilities.SupportsMultipleEntries;""" -new = """ public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanModify | - FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;""" -if old not in s: - raise SystemExit("ReFS capability block not found") -s = s.replace(old, new, 1) - -s = s.replace( - 'public string Description => "Microsoft ReFS 3.x volume image with namespace, allocation, in-place data relocation and filesystem-metadata placement support.";', - 'public string Description => "Microsoft ReFS 3.x volume image; offline-quiescent existing-file replace/remove plus allocation and metadata placement. Native mounted-driver transactions remain a separate readiness tier.";', - 1, -) - -marker = " private static List ListDiagnosticSurface(Stream stream) {" -if marker not in s: - raise SystemExit("ReFS descriptor insertion marker not found") -methods = ''' /// - /// Offline-quiescent existing-file replacement for the proven regular-stream profile. - /// A new name is rejected before mutation until ReFS file-identity/security/link fields - /// are proven for every supported 3.x profile. - /// - public void Add(Stream archive, IReadOnlyList inputs) - => RefsOfflineModifier.Add(archive, inputs); - - /// - /// Removes regular files or empty directories from an unmounted ReFS image. Namespace - /// deletion is published through immutable B+ replacement pages and the alternate CHKP. - /// - public void Remove(Stream archive, string[] entryNames) - => RefsOfflineModifier.Remove(archive, entryNames); - -''' -s = s.replace(marker, methods + marker, 1) -p.write_text(s, encoding="utf-8") - -# The readiness document distinguishes the library's offline image-editor surface -# from full native mounted-driver R/W. Do not erase the latter's remaining work. -p = Path("FileSystems/FileSystem.Refs/DRIVER_READINESS.md") -s = p.read_text(encoding="utf-8") -needle = "- [x] explicit offline-vs-native mutation transaction boundary\n" -if needle not in s: - raise SystemExit("ReFS readiness insertion point not found") -s = s.replace( - needle, - needle - + "- [x] offline-quiescent existing regular-file replacement with allocator-verified reallocation and old-block release\n" - + "- [x] offline-quiescent regular-file / empty-directory removal through CoW B+ replacement + alternate CHKP publication\n" - + "- [x] archive API exposes the proven offline mutation profile without claiming mounted-driver crash semantics\n", - 1, -) -p.write_text(s, encoding="utf-8") - -Path(".github/refs-promotion.py").unlink(missing_ok=True) From aa9ca8328820c86f82e975d3f41c3c3a53d45c99 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:07:13 +0200 Subject: [PATCH 037/192] - remove temporary bulk promotion script --- .github/rw-promotion.py | 201 ---------------------------------------- 1 file changed, 201 deletions(-) delete mode 100644 .github/rw-promotion.py diff --git a/.github/rw-promotion.py b/.github/rw-promotion.py deleted file mode 100644 index 979387fc6..000000000 --- a/.github/rw-promotion.py +++ /dev/null @@ -1,201 +0,0 @@ -from pathlib import Path -import re - - -def text(path: str) -> str: - return Path(path).read_text(encoding="utf-8") - - -def write(path: str, value: str) -> None: - Path(path).write_text(value, encoding="utf-8") - - -def sub(path: str, pattern: str, replacement: str, count: int = 1, flags: int = 0) -> None: - value = text(path) - new, n = re.subn(pattern, replacement, value, count=count, flags=flags) - if n != count: - raise SystemExit(f"{path}: expected {count} replacement(s), got {n}: {pattern}") - write(path, new) - - -# CramFS: native mount is read-only; the workbench already has a verified -# extract/edit/re-create modifier, so expose that existing-image edit. -p = "FileSystems/FileSystem.CramFs/CramFsFormatDescriptor.cs" -sub( - p, - r" // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;", - " // The on-disk filesystem is read-only when mounted, but CompressionWorkbench\n" - " // supports existing-image add/replace/remove by verified relayout/rebuild.\n" - " public FormatCapabilities Capabilities =>\n" - " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" - " FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n" - " FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;", - flags=re.S, -) -sub( - p, - r'public string Description => "Linux [Cc]ompressed ROM filesystem";', - 'public string Description => "Linux compressed ROM filesystem; offline image mutation is rebuild-backed.";', -) - -# SquashFS: same distinction — read-only mount format, editable image. -p = "FileSystems/FileSystem.SquashFs/SquashFsFormatDescriptor.cs" -sub( - p, - r" // WORM \(Write-Once-Read-Many\), NOT R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries \| FormatCapabilities\.SupportsDirectories;", - " // SquashFS is read-only when mounted, but the workbench can edit an existing\n" - " // image by a verified extract/edit/re-create pass. That is R/W at this API.\n" - " public FormatCapabilities Capabilities =>\n" - " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" - " FormatCapabilities.CanModify | FormatCapabilities.CanTest |\n" - " FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;", - flags=re.S, -) -sub( - p, - r'public string Description => "Linux compressed read-only filesystem";', - 'public string Description => "Linux compressed read-only-on-mount filesystem; offline image mutation is rebuild-backed.";', -) - -# EROFS: promote the fully decoded writer/reader profile. Explicit Add/Remove use -# ReadEntries, which throws on an unsupported compressed inode and therefore never -# feeds the descriptor's user-facing placeholder into a rebuilt image. -p = "FileSystems/FileSystem.Erofs/ErofsFormatDescriptor.cs" -sub(p, r"the round-trippable WORM subset", "the round-trippable offline R/W subset") -sub( - p, - r"FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \|", - "FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" - " FormatCapabilities.CanModify | FormatCapabilities.CanTest |", -) -sub( - p, - r'public string Description => "Android read-only compressed filesystem; uncompressed \+ inline inode layouts\.";', - 'public string Description => "Android read-only-on-mount filesystem; supported uncompressed/inline profile is offline R/W.";', -) -marker = " private static ErofsReader OpenReader(Stream stream) {" -value = text(p) -if marker not in value: - raise SystemExit("EROFS insertion marker missing") -methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - archive.Position = 0; - var label = OpenReader(archive).VolumeName; - archive.Position = 0; - ModifyRebuilder.Add(archive, inputs, - readEntries: stream => ReadEntries(stream), - buildImage: files => { - var writer = new ErofsWriter { VolumeName = label }; - foreach (var (name, data) in files) writer.AddFile(name, data); - return writer.Build(); - }); - } - - public void Remove(Stream archive, string[] entryNames) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - archive.Position = 0; - var label = OpenReader(archive).VolumeName; - archive.Position = 0; - ModifyRebuilder.Remove(archive, entryNames, - readEntries: stream => ReadEntries(stream), - buildImage: files => { - var writer = new ErofsWriter { VolumeName = label }; - foreach (var (name, data) in files) writer.AddFile(name, data); - return writer.Build(); - }); - } - -''' -value = value.replace(marker, methods + marker, 1) -write(p, value) - -# MSA already has functional Add/Remove against its decoded GEMDOS volume. -p = "FileSystems/FileSystem.Msa/MsaFormatDescriptor.cs" -sub( - p, - r" // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest;", - " // Existing MSA images are editable through the decoded GEMDOS volume and\n" - " // then re-encoded; physical rebuild does not make the public operation WORM.\n" - " public FormatCapabilities Capabilities =>\n" - " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" - " FormatCapabilities.CanModify | FormatCapabilities.CanTest;", - flags=re.S, -) - -# PFS0 has a real existing-container editor; its capability flag was simply stale. -p = "FileFormats/FileFormat.Pfs0/Pfs0FormatDescriptor.cs" -sub( - p, - r" // WORM, not R/W:.*? public FormatCapabilities Capabilities =>\n FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanCreate \|\n FormatCapabilities\.CanTest \| FormatCapabilities\.SupportsMultipleEntries;", - " // Existing PFS0 archives support add/replace/remove through Pfs0InPlaceModifier.\n" - " public FormatCapabilities Capabilities =>\n" - " FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |\n" - " FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries;", - flags=re.S, -) - -# Archive-model docs must agree with FormatCapabilities and the package README. -p = "docs/ARCHIVE-MODEL.md" -sub( - p, - r"`CanModify` is \*\*withheld\*\* only from \*\*read-only-by-design\*\* formats \(CramFS, SquashFS\) and\n\*\*create-only\*\* formats \(e\.g\. the checksum-record archives Sqx/Wim/Swm/Ace\) — they may still\nback the verbs with a rebuild for convenience, but they do not present themselves as editable\.", - "`CanModify` is withheld from **create-only** formats: a fresh instance can be written, but no supported edit of an existing instance exists. Read-only-on-mount filesystem formats such as CramFS, SquashFS and EROFS may still advertise `CanModify` when the workbench implements a verified offline edit/rebuild path; the native mount policy and the image-editor API are different concerns.", -) - -# EWF: expose the acquired medium as the one semantic mutable entry. -p = "FileFormats/FileFormat.Ewf/EwfFormatDescriptor.cs" -sub( - p, - r"public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable \{", - "public sealed class EwfFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable {", -) -sub( - p, - r"FormatCapabilities\.CanList \| FormatCapabilities\.CanExtract \| FormatCapabilities\.CanTest \|\n FormatCapabilities\.CanCreate \| FormatCapabilities\.SupportsMultipleEntries;", - "FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |\n" - " FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries;", -) -sub( - p, - r"var entries = new List<\(string, byte\[\], string\)> \{\n \(\"metadata\.ini\", BuildMetadata\(img\), \"stored\"\),\n \};", - 'var entries = new List<(string, byte[], string)> {\n ("metadata.ini", BuildMetadata(img), "stored"),\n };\n if (EwfMedia.TryExtract(img, out var medium))\n entries.Add(("media.raw", medium, "stored"));', -) -marker = " private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) {" -value = text(p) -if marker not in value: - raise SystemExit("EWF insertion marker missing") -methods = ''' public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - var files = inputs.Where(i => !i.IsDirectory).ToList(); - if (files.Count != 1) - throw new ArgumentException("EWF mutation accepts exactly one replacement medium.", nameof(inputs)); - var media = files[0].ReadContent(); - var rebuilt = new EwfWriter().Build(media); - archive.Position = 0; - archive.SetLength(0); - archive.Write(rebuilt); - archive.Position = 0; - } - - public void Remove(Stream archive, string[] entryNames) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - if (!entryNames.Any(n => string.Equals(n, "media.raw", StringComparison.OrdinalIgnoreCase))) - throw new NotSupportedException("EWF diagnostic sections are derived metadata and cannot be removed independently; remove media.raw to clear the acquired medium."); - var rebuilt = new EwfWriter().Build([]); - archive.Position = 0; - archive.SetLength(0); - archive.Write(rebuilt); - archive.Position = 0; - } - -''' -value = value.replace(marker, methods + marker, 1) -write(p, value) - -# Delete the one-shot machinery from the final branch. -Path(".github/workflows/rw-promotion-once.yml").unlink(missing_ok=True) -Path(".github/rw-promotion.py").unlink(missing_ok=True) From a317dbcceb752f472ecea131ede6782bd773af7b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:08:54 +0200 Subject: [PATCH 038/192] + add explicit purge capability contract --- Compression.Registry/IArchivePurgeable.cs | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Compression.Registry/IArchivePurgeable.cs diff --git a/Compression.Registry/IArchivePurgeable.cs b/Compression.Registry/IArchivePurgeable.cs new file mode 100644 index 000000000..ea1b0128c --- /dev/null +++ b/Compression.Registry/IArchivePurgeable.cs @@ -0,0 +1,25 @@ +namespace Compression.Registry; + +/// +/// Opt-in capability: all user/live entries can be removed from an existing +/// container while leaving a valid, listable empty instance. This is distinct +/// from , which preserves live entries and overwrites only +/// unused/dead bytes. +/// +public interface IArchivePurgeable { + /// + /// Removes every live non-directory entry from . + /// + /// Default implementation: descriptors that also implement + /// and + /// get a transactional staged purge through . + /// Native implementations may override this when they can empty the container + /// more efficiently. + /// + void Purge(Stream archive) { + if (this is not IArchiveFormatOperations ops || this is not IArchiveModifiable modifier) + throw new NotSupportedException( + "The default Purge requires IArchiveFormatOperations + IArchiveModifiable."); + RebuildVerb.PurgeViaModifier(archive, ops, modifier); + } +} From 11187b14d3bb96fb0f6ca10f68a91d132be7ad32 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:09:15 +0200 Subject: [PATCH 039/192] + make modifiable containers explicitly purgeable --- Compression.Registry/IArchiveModifiable.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Compression.Registry/IArchiveModifiable.cs b/Compression.Registry/IArchiveModifiable.cs index 58be8a608..323ec77ea 100644 --- a/Compression.Registry/IArchiveModifiable.cs +++ b/Compression.Registry/IArchiveModifiable.cs @@ -11,10 +11,11 @@ namespace Compression.Registry; /// /// A descriptor advertising must expose this /// interface and its supported-profile edit path must actually round-trip. Merely being able -/// to create a fresh instance is not enough. +/// to create a fresh instance is not enough. A fully modifiable container is also purgeable: +/// removing all live entries is a required subset of the remove contract. /// /// -public interface IArchiveModifiable { +public interface IArchiveModifiable : IArchivePurgeable { /// /// Adds files to an existing instance, replacing entries with the same logical path/name. /// @@ -59,4 +60,4 @@ void Remove(Stream archive, string[] entryNames) { } }); } -} \ No newline at end of file +} From 367e06a093bfc4cacb4012c0181630837dd91506 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:14:03 +0200 Subject: [PATCH 040/192] + optimize multi-entry archive compression parameters --- .../ArchiveCompressionOptimizer.cs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 Compression.Lib/ArchiveCompressionOptimizer.cs diff --git a/Compression.Lib/ArchiveCompressionOptimizer.cs b/Compression.Lib/ArchiveCompressionOptimizer.cs new file mode 100644 index 000000000..0ebbd7a21 --- /dev/null +++ b/Compression.Lib/ArchiveCompressionOptimizer.cs @@ -0,0 +1,154 @@ +using Compression.Registry; + +namespace Compression.Lib; + +/// +/// Searches a creatable archive/container's finite option schema and keeps the +/// smallest verified same-format rebuild. Unlike , +/// which optimizes one compressed stream, this optimizer works on multi-entry +/// containers (compression method/level, dictionary, solid-block size, etc.). +/// +public static class ArchiveCompressionOptimizer { + public sealed record Result(long OriginalSize, long OptimizedSize, int EntriesOptimized, + IReadOnlyDictionary Parameters, int Probes); + + public static Result Optimize( + string inputPath, + string outputPath, + IArchiveFormatOperations ops, + IArchiveCreatable creator, + IFormatOptionsSchema schema, + int maxCombinations = 256) { + ArgumentException.ThrowIfNullOrEmpty(inputPath); + ArgumentException.ThrowIfNullOrEmpty(outputPath); + ArgumentNullException.ThrowIfNull(ops); + ArgumentNullException.ThrowIfNull(creator); + ArgumentNullException.ThrowIfNull(schema); + + var axes = SearchAxes(schema).ToArray(); + var originalSize = new FileInfo(inputPath).Length; + var sourceEntries = CountLiveEntries(inputPath, ops); + var bestSize = originalSize; + var bestParameters = (IReadOnlyDictionary)new Dictionary(); + string? bestPath = null; + var probes = 0; + + try { + if (axes.Length > 0) { + long product = 1; + foreach (var axis in axes) { + product = checked(Math.Min((long)maxCombinations + 1, product * axis.Values.Count)); + if (product > maxCombinations) break; + } + + if (product <= maxCombinations) { + foreach (var combination in EnumerateCombinations(axes)) + Probe(combination); + } else { + var current = axes.ToDictionary(a => a.Key, a => a.Default, StringComparer.Ordinal); + Probe(current); + var improved = true; + while (improved && probes < maxCombinations) { + improved = false; + foreach (var axis in axes) { + foreach (var value in axis.Values) { + if (probes >= maxCombinations) break; + if (current[axis.Key] == value) continue; + var trial = new Dictionary(current, StringComparer.Ordinal) { + [axis.Key] = value, + }; + var before = bestSize; + Probe(trial); + if (bestSize < before) { + current = trial; + improved = true; + } + } + } + } + } + } + + if (bestPath == null) { + AtomicFileWriter.WriteAtomic(outputPath, output => { + using var input = File.OpenRead(inputPath); + input.CopyTo(output); + }); + return new Result(originalSize, originalSize, 0, bestParameters, probes); + } + + AtomicFileWriter.WriteAtomic(outputPath, output => { + using var best = File.OpenRead(bestPath); + best.CopyTo(output); + }); + return new Result(originalSize, bestSize, sourceEntries, bestParameters, probes); + } finally { + if (bestPath != null) TryDelete(bestPath); + } + + void Probe(IReadOnlyDictionary parameters) { + ++probes; + var candidatePath = Path.Combine(Path.GetTempPath(), "cwb_arcopt_" + Guid.NewGuid().ToString("N") + ".tmp"); + try { + using (var input = File.OpenRead(inputPath)) + using (var candidate = new FileStream(candidatePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None)) { + RebuildVerb.RebuildToStream(input, candidate, ops, creator, parameters); + candidate.Flush(flushToDisk: true); + if (candidate.Length >= bestSize) return; + bestSize = candidate.Length; + } + + if (bestPath != null) TryDelete(bestPath); + bestPath = candidatePath; + candidatePath = ""; + bestParameters = new Dictionary(parameters, StringComparer.Ordinal); + } catch (Exception ex) when (ex is InvalidOperationException or NotSupportedException or IOException or ArgumentException) { + // An individual parameter combination may be invalid for the current + // content/profile. It is a rejected probe, not an optimization failure. + } finally { + if (!string.IsNullOrEmpty(candidatePath)) TryDelete(candidatePath); + } + } + } + + private sealed record Axis(string Key, IReadOnlyList Values, string Default); + + private static IEnumerable SearchAxes(IFormatOptionsSchema schema) { + foreach (var option in schema.OptionsSchema) { + IReadOnlyList? values = option.Kind switch { + FormatOptionKind.Enum or FormatOptionKind.Integer when option.AllowedValues is { Count: > 1 } + => option.AllowedValues, + FormatOptionKind.Boolean => ["false", "true"], + _ => null, + }; + if (values is { Count: > 1 }) + yield return new Axis(option.Key, values, option.Default); + } + } + + private static IEnumerable> EnumerateCombinations(IReadOnlyList axes) { + if (axes.Count == 0) yield break; + var indices = new int[axes.Count]; + while (true) { + var result = new Dictionary(axes.Count, StringComparer.Ordinal); + for (var i = 0; i < axes.Count; ++i) result[axes[i].Key] = axes[i].Values[indices[i]]; + yield return result; + + var position = axes.Count - 1; + while (position >= 0 && ++indices[position] == axes[position].Values.Count) { + indices[position] = 0; + --position; + } + if (position < 0) yield break; + } + } + + private static int CountLiveEntries(string path, IArchiveFormatOperations ops) { + using var input = File.OpenRead(path); + return ops.List(input, null).Count(e => !e.IsDirectory); + } + + private static void TryDelete(string path) { + try { File.Delete(path); } catch { /* best effort */ } + } +} From 4aca5779836b22e15b8c341dbfee0ba54dadf872 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:19:09 +0200 Subject: [PATCH 041/192] ci: stage maintenance capability audit --- .github/maintenance-capabilities.py | 285 ++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 .github/maintenance-capabilities.py diff --git a/.github/maintenance-capabilities.py b/.github/maintenance-capabilities.py new file mode 100644 index 000000000..3b0b8354a --- /dev/null +++ b/.github/maintenance-capabilities.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +from pathlib import Path +import re + +ROOT = Path('.') + + +def read(path: str | Path) -> str: + return Path(path).read_text(encoding='utf-8') + + +def write(path: str | Path, value: str) -> None: + Path(path).write_text(value, encoding='utf-8') + + +def replace_once(path: str | Path, old: str, new: str) -> None: + value = read(path) + if old not in value: + raise SystemExit(f'{path}: required text not found:\n{old[:240]}') + write(path, value.replace(old, new, 1)) + + +def insert_before(path: str | Path, marker: str, insertion: str) -> None: + value = read(path) + if marker not in value: + raise SystemExit(f'{path}: insertion marker not found: {marker[:160]}') + write(path, value.replace(marker, insertion + marker, 1)) + + +# ── Purge: explicit, transactional capability ─────────────────────────────── +rebuild_path = Path('Compression.Registry/RebuildVerb.cs') +purge_helper = ''' /// \n /// Transactional purge for a mutable container. The modifier operates only on\n /// a staged copy; the caller's stream is replaced after the staged container\n /// re-lists successfully and every original live entry name has disappeared.\n /// \n public static void PurgeViaModifier(\n Stream archive, IArchiveFormatOperations ops, IArchiveModifiable modifier) {\n ArgumentNullException.ThrowIfNull(archive);\n ArgumentNullException.ThrowIfNull(ops);\n ArgumentNullException.ThrowIfNull(modifier);\n if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek)\n throw new ArgumentException("Purge requires a readable, writable, seekable stream.", nameof(archive));\n\n using var staged = CreateScratchStream();\n archive.Position = 0;\n archive.CopyTo(staged);\n staged.Flush();\n\n staged.Position = 0;\n var sourceNames = ops.List(staged, null)\n .Where(e => !e.IsDirectory)\n .Select(e => e.Name)\n .Distinct(StringComparer.OrdinalIgnoreCase)\n .ToArray();\n if (sourceNames.Length == 0) return;\n\n staged.Position = 0;\n modifier.Remove(staged, sourceNames);\n\n staged.Position = 0;\n var remaining = ops.List(staged, null)\n .Where(e => !e.IsDirectory)\n .Select(e => e.Name)\n .ToHashSet(StringComparer.OrdinalIgnoreCase);\n var survivors = sourceNames.Where(remaining.Contains).ToArray();\n if (survivors.Length != 0)\n throw new InvalidOperationException(\n $"Purge left {survivors.Length} original live entr{(survivors.Length == 1 ? "y" : "ies")} behind; original container retained.");\n\n archive.Position = 0;\n archive.SetLength(0);\n staged.Position = 0;\n staged.CopyTo(archive);\n archive.Flush();\n }\n\n''' +if 'public static void PurgeViaModifier(' not in read(rebuild_path): + insert_before(rebuild_path, + ' /// \n /// A writable scratch stream that is not bounded by what a byte[] can hold.', + purge_helper) + +# Purge default test now tests the explicit capability and requires an actual +# successful purge for any probe image a marked descriptor can create. +purge_test = Path('Compression.Tests/Operations/GenericPurgeRoundTripTests.cs') +v = read(purge_test) +v = v.replace('Safety net for the broad rollout of the default ', + 'Safety net for the explicit capability') +v = v.replace('// Every format using the DEFAULT IArchiveModifiable.Remove (rebuild-via-WORM):', + '// Every registered format explicitly exposing purge:') +v = v.replace('Compression.Tests.Support.CapabilityImplementers.RegisteredIdsExposing(typeof(IArchiveModifiable))', + 'Compression.Tests.Support.CapabilityImplementers.RegisteredIdsExposing(typeof(IArchivePurgeable))') +v = re.sub(r'\n\s*&& !Compression\.Tests\.Support\.CapabilityImplementers\.DeclaresOwn\(id, "Remove", typeof\(Stream\), typeof\(string\[\]\)\)', '', v) +v = v.replace('var modifiable = (IArchiveModifiable)fmtOps;', 'var purgeable = (IArchivePurgeable)fmtOps;') +v = v.replace('modifiable.Remove(ms, [.. before]);', 'purgeable.Purge(ms);') +v = v.replace('Assert.Pass($"{formatId}: purge cleanly NotSupported (no corruption).");\n return;', + 'Assert.Fail($"{formatId}: advertises purge but rejected the probe container.");\n return;') +v = v.replace('Assert.Ignore($"{formatId}: purge rebuild failed non-destructively ({ex.GetType().Name}).");\n return;', + 'Assert.Fail($"{formatId}: advertises purge but failed the probe ({ex.GetType().Name}: {ex.Message}).");\n return;') +write(purge_test, v) + +marker_test = Path('Compression.Tests/Operations/MarkerInterfaceCoverageTests.cs') +v = read(marker_test) +v = v.replace('("purge/modify", typeof(IArchiveModifiable)),', '("purge", typeof(IArchivePurgeable)),') +write(marker_test, v) + +# ── Wipe / clean: any exact extent/layout map gets a safe generic default ─── +wipe_path = Path('Compression.Registry/IWipeEmpty.cs') +v = read(wipe_path) +old_decl = ' long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true);' +new_decl = ''' long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) {\n ArgumentNullException.ThrowIfNull(image);\n if (!image.CanRead || !image.CanWrite || !image.CanSeek)\n throw new ArgumentException("Wipe requires a readable, writable, seekable stream.", nameof(image));\n\n List extents = this switch {\n IFilesystemExtentMap fs => fs.EnumerateExtents(image).ToList(),\n IArchiveLayoutMap archive => archive.EnumerateLayout(image).ToList(),\n _ => throw new NotSupportedException(\n "The default wipe requires IFilesystemExtentMap or IArchiveLayoutMap."),\n };\n\n Func? sizeLookup = null;\n if (wipeClusterTips && this is IArchiveFormatOperations ops) {\n image.Position = 0;\n var sizes = ops.List(image, null)\n .Where(e => !e.IsDirectory)\n .GroupBy(e => e.Name, StringComparer.Ordinal)\n .ToDictionary(g => g.Key, g => Math.Max(0, g.First().OriginalSize), StringComparer.Ordinal);\n sizeLookup = name => sizes.TryGetValue(name, out var size) ? size : -1;\n }\n\n image.Position = 0;\n return UnusedSpaceWiper.Wipe(image, extents, image.Length, wipeClusterTips, sizeLookup);\n }''' +if old_decl in v: + v = v.replace(old_decl, new_decl, 1) +write(wipe_path, v) + +# ── Optimize: multi-entry option-space search is a real archive capability ── +archive_ops_path = Path('Compression.Lib/ArchiveOperations.cs') +v = read(archive_ops_path) +old_fallback = ''' // ── Unsupported: fall back to copy ───────────────────────────────\n // Use temp+rename so a crash mid-copy doesn't leave a truncated target.\n AtomicFileWriter.WriteAtomic(outputPath, outFs => {\n using var inFs = File.OpenRead(inputPath);\n inFs.CopyTo(outFs);\n });\n return (originalSize, originalSize, 0);''' +new_fallback = ''' // ── Generic multi-entry archive parameter search ─────────────────\n // A creatable archive with a finite option schema can be optimized without\n // format-specific code: try method/level/dictionary/solid-size combinations,\n // verify every rebuild round-trips the exact live entry set, and keep only\n // a strictly smaller result.\n FormatRegistration.EnsureInitialized();\n var archiveOps = Compression.Registry.FormatRegistry.GetArchiveOps(format.ToString());\n if (archiveOps is Compression.Registry.IArchiveCreatable creator\n && archiveOps is Compression.Registry.IFormatOptionsSchema schema\n && schema.OptionsSchema.Any(o =>\n o.Kind == Compression.Registry.FormatOptionKind.Boolean\n || o.AllowedValues is { Count: > 1 })) {\n var best = ArchiveCompressionOptimizer.Optimize(inputPath, outputPath, archiveOps, creator, schema);\n return (best.OriginalSize, best.OptimizedSize, best.EntriesOptimized);\n }\n\n // No searchable encoding/layout parameter exists. Preserve the old behavior:\n // output is a byte-identical copy and EntriesOptimized=0, so callers do not\n // mistake a no-op for an optimization.\n AtomicFileWriter.WriteAtomic(outputPath, outFs => {\n using var inFs = File.OpenRead(inputPath);\n inFs.CopyTo(outFs);\n });\n return (originalSize, originalSize, 0);''' +if old_fallback in v: + v = v.replace(old_fallback, new_fallback, 1) +elif 'Generic multi-entry archive parameter search' not in v: + raise SystemExit('ArchiveOperations.cs: optimize fallback block not found') +write(archive_ops_path, v) + +layout_path = Path('Compression.Registry/ILayoutOptimizable.cs') +v = read(layout_path) +v = v.replace('Opt-in capability: a filesystem descriptor can analyse its current on-disk layout', + 'Opt-in capability: a filesystem, archive, or container descriptor can analyse its current layout') +v = v.replace('The descriptor can be rebuilt with a different geometry / allocation-unit choice.', + 'The descriptor can be rebuilt with a different geometry, allocation-unit, compression, dictionary, solid-block, or other layout/encoding choice.') +write(layout_path, v) + +# ── Bulk interface promotion, guarded by existing working prerequisites ───── +class_re = re.compile( + r'(public\s+(?:(?:sealed|partial|abstract)\s+)*class\s+\w+FormatDescriptor\s*:\s*)([^\{]+)(\{)', + re.S) + + +def add_interface(path: Path, interface: str, predicate) -> bool: + value = read(path) + match = class_re.search(value) + if not match: + return False + interfaces = match.group(2) + if interface in interfaces or not predicate(value, interfaces): + return False + replacement = match.group(1) + interfaces.rstrip() + ', ' + interface + ' ' + match.group(3) + value = value[:match.start()] + replacement + value[match.end():] + write(path, value) + return True + + +def has(*names): + return lambda _value, interfaces: all(name in interfaces for name in names) + +promoted = {'defrag': 0, 'optimize': 0, 'wipe': 0} +for base in (Path('FileFormats'), Path('FileSystems')): + for path in base.rglob('*FormatDescriptor.cs'): + if add_interface(path, 'IArchiveDefragmentable', has('IArchiveFormatOperations', 'IArchiveCreatable')): + promoted['defrag'] += 1 + if add_interface(path, 'ILayoutOptimizable', + lambda value, interfaces: 'IArchiveFormatOperations' in interfaces + and 'IArchiveCreatable' in interfaces + and 'IFormatOptionsSchema' in interfaces + and ('AllowedValues:' in value or 'FormatOptionKind.Boolean' in value)): + promoted['optimize'] += 1 + if add_interface(path, 'IWipeEmpty', + lambda _value, interfaces: 'IFilesystemExtentMap' in interfaces or 'IArchiveLayoutMap' in interfaces): + promoted['wipe'] += 1 + +print('Promoted:', promoted) + +# ── UI gates now use the same explicit contracts as documentation/tests ───── +main_vm = Path('Compression.UI/ViewModels/MainViewModel.cs') +v = read(main_vm) +v = v.replace('Views.MaintenanceVerb.Optimize => ops is IArchiveCreatable or IFileInternalChunkMover,', + 'Views.MaintenanceVerb.Optimize => ops is ILayoutOptimizable or IFileInternalChunkMover,') +v = v.replace('Views.MaintenanceVerb.Purge => ops is IArchiveModifiable,', + 'Views.MaintenanceVerb.Purge => ops is IArchivePurgeable,') +v = v.replace('Views.MaintenanceVerb.WipeEmpty => ops is IWipeEmpty or IFilesystemExtentMap or IArchiveLayoutMap,', + 'Views.MaintenanceVerb.WipeEmpty => ops is IWipeEmpty,') +write(main_vm, v) + +window = Path('Compression.UI/Views/DefragmentWindow.xaml.cs') +v = read(window) +v = v.replace('var isArchiveCreatable = ops is IArchiveCreatable;', + 'var isArchiveCreatable = ops is IArchiveCreatable;\n var isArchiveOptimizable = ops is ILayoutOptimizable && isArchiveCreatable;') +# Make a specifically requested Optimize verb win over a descriptor that also +# exposes defrag (common for filesystems after this promotion). +old_if = ''' if (this._defragmentable != null) {\n // FS defrag path (existing)''' +new_if = ''' if (this._requestedVerb == MaintenanceVerb.Optimize && isArchiveOptimizable) {\n this._isArchiveMode = true;\n this._isSevenZipFormat = format.ToString() == "SevenZip";\n this._archiveOps = ops;\n SupportLbl.Text = "Container optimization (verified parameter search / repack).";\n SupportLbl.Foreground = System.Windows.Media.Brushes.DarkGreen;\n RunBtn.Content = "Optimize";\n RunBtn.IsEnabled = true;\n } else if (this._defragmentable != null) {\n // FS defrag path (existing)''' +if old_if in v: + v = v.replace(old_if, new_if, 1) +v = v.replace('} else if (isArchiveLayout || isArchiveCreatable) {', + '} else if (isArchiveLayout || isArchiveOptimizable) {') +v = v.replace(' if (isArchiveCreatable) {\n SupportLbl.Text = "Archive optimization (extract + repack with optimal settings).";', + ' if (isArchiveOptimizable) {\n SupportLbl.Text = "Archive optimization (verified parameter search / optimal repack).";') +v = v.replace('var supportsWipe = ops is IWipeEmpty || ops is IFilesystemExtentMap || ops is IArchiveLayoutMap;', + 'var supportsWipe = ops is IWipeEmpty;') +v = v.replace('PurgeBtn.IsEnabled = ops is IArchiveModifiable;', 'PurgeBtn.IsEnabled = ops is IArchivePurgeable;') +v = v.replace(' /// over every entry. Distinct from', + ' /// . Distinct from') +v = v.replace(' if (ops is not IArchiveModifiable) {\n Append($"Purge not supported: {formatStr} is not modifiable.");', + ' if (ops is not IArchivePurgeable purgeable) {\n Append($"Purge not supported: {formatStr} does not expose IArchivePurgeable.");') +v = v.replace(' // Remove files first, then directories (deepest paths last avoids\n // a modifier rejecting a non-empty directory removal).\n Compression.Lib.ArchiveOperations.Remove(path, allNames);', + ' using var stream = File.Open(path, FileMode.Open, FileAccess.ReadWrite);\n purgeable.Purge(stream);') +write(window, v) + +# CLI wipe should likewise expose only the explicit marker; the map-backed +# default means no supported format loses the generic path. +cli = Path('Compression.CLI/Program.cs') +v = read(cli) +# Narrow the wipe fallback branches: they are now supplied by IWipeEmpty default. +start = v.find('var wipeCmd = new Command("wipe-empty"') +end = v.find('// ── compact', start) +if start >= 0 and end > start: + block = v[start:end] + block = block.replace(' For formats without a dedicated implementation but with an extent/layout\n map, the generic wiper zeros all gaps between live extents.\n', + ' Layout-mapped formats inherit the generic IWipeEmpty implementation, which\n zeros gaps between all live extents.\n') + # Keep runtime fallbacks for backwards compatibility; interface promotion + # makes them unreachable for shipped descriptors, so no behavior regresses. + v = v[:start] + block + v[end:] +write(cli, v) + +# ── Documentation: generate exhaustive checkmark matrices from source ─────── +id_re = re.compile(r'public\s+string\s+Id\s*=>\s*"([^"]+)"') + + +def descriptor_rows(base: Path, archive_only: bool) -> list[dict[str, object]]: + rows = [] + for path in base.rglob('*FormatDescriptor.cs'): + value = read(path) + m = class_re.search(value) + mid = id_re.search(value) + if not m or not mid: + continue + interfaces = m.group(2) + if archive_only and 'IArchiveFormatOperations' not in interfaces and 'IStreamFormatOperations' not in interfaces: + continue + row = { + 'id': mid.group(1), + 'optimize': 'ILayoutOptimizable' in interfaces or ('IStreamFormatOperations' in interfaces and 'IFormatOptionsSchema' in interfaces), + 'wipe': 'IWipeEmpty' in interfaces, + 'purge': 'IArchivePurgeable' in interfaces or 'IArchiveModifiable' in interfaces, + 'defrag': 'IArchiveDefragmentable' in interfaces, + 'shrink': 'IArchiveShrinkable' in interfaces, + 'compact': ('IArchiveDefragmentable' in interfaces or 'IArchiveShrinkable' in interfaces or 'IArchiveCreatable' in interfaces), + } + rows.append(row) + dedup = {} + for row in rows: + dedup[row['id']] = row + return sorted(dedup.values(), key=lambda r: str(r['id']).lower()) + + +def mark(value: bool) -> str: + return '✅' if value else '—' + + +def matrix(rows: list[dict[str, object]], full: bool = False) -> str: + if full: + lines = [ + '| Format | Optimize | Wipe / clean | Purge | Defrag | Shrink | Compact |', + '|---|:---:|:---:|:---:|:---:|:---:|:---:|', + ] + for r in rows: + lines.append(f"| {r['id']} | {mark(r['optimize'])} | {mark(r['wipe'])} | {mark(r['purge'])} | {mark(r['defrag'])} | {mark(r['shrink'])} | {mark(r['compact'])} |") + return '\n'.join(lines) + lines = [ + '| Format | Optimize | Wipe / clean | Purge | Defrag |', + '|---|:---:|:---:|:---:|:---:|', + ] + for r in rows: + lines.append(f"| {r['id']} | {mark(r['optimize'])} | {mark(r['wipe'])} | {mark(r['purge'])} | {mark(r['defrag'])} |") + return '\n'.join(lines) + + +def put_matrix(path: Path, rows: list[dict[str, object]]) -> None: + value = read(path) + start_marker = '' + end_marker = '' + section = f'''{start_marker}\n## Maintenance capability matrix\n\nThese columns are generated from the descriptor capability contracts. `Wipe / clean` means sanitizing unused/dead/reserved bytes while preserving the container size; removing reserve to make the file smaller is **Shrink/Compact**, not Wipe. Rebuild-backed Defrag/Optimize/Purge count only where the implementation is transactional/round-trip-verified.\n\n{matrix(rows)}\n\n{end_marker}''' + if start_marker in value and end_marker in value: + value = re.sub(re.escape(start_marker) + r'.*?' + re.escape(end_marker), section, value, flags=re.S) + else: + value = value.rstrip() + '\n\n' + section + '\n' + write(path, value) + +archive_rows = descriptor_rows(Path('FileFormats'), archive_only=True) +fs_rows = descriptor_rows(Path('FileSystems'), archive_only=False) +put_matrix(Path('Hawkynt.FileFormats.Archives/README.md'), archive_rows) +put_matrix(Path('Hawkynt.FileFormats.FileSystems/README.md'), fs_rows) + +coverage = Path('docs/OPERATION_COVERAGE.md') +v = read(coverage) +fs_section = f'''## Filesystem descriptors\n\nGenerated from descriptor capability contracts. `Wipe / clean` preserves outer size; `Shrink`/`Compact` may remove reserved/free tail space.\n\n{matrix(fs_rows, full=True)}\n\n''' +archive_section = f'''## Archive / stream descriptors\n\nUnlike the old abbreviated table, this lists every archive/stream descriptor with the same maintenance columns. Solid-container Defrag means a verified re-layout/repack; Optimize includes compression/dictionary/solid-block parameter search when exposed by the format schema.\n\n{matrix(archive_rows, full=True)}\n\n''' +fs_start = v.find('## Filesystem descriptors') +archive_start = v.find('## Archive / stream descriptors', fs_start) +notes_start = v.find('## N/A notes', archive_start) +if fs_start < 0 or archive_start < 0 or notes_start < 0: + raise SystemExit('docs/OPERATION_COVERAGE.md: expected section anchors not found') +v = v[:fs_start] + fs_section + archive_section + v[notes_start:] +# Reconcile the old no-interface wording now that purge is first-class. +v = v.replace('A dedicated **purge (empty-all)** verb has no interface yet; it is realised by\n `IArchiveModifiable.Remove` over all entries (or a fresh empty `Create`). A\n future `IArchivePurgeable` could formalise it.', + '**purge (empty-all)** is explicitly exposed by `IArchivePurgeable`; fully\n modifiable containers inherit that contract and the generic implementation stages and\n verifies the purge before replacing the original.') +write(coverage, v) + +model = Path('docs/ARCHIVE-MODEL.md') +v = read(model) +v = v.replace('`IArchiveModifiable.Remove` over all entries (or an empty `IArchiveCreatable.Create` *(no dedicated `IArchivePurgeable` yet — see Naming note)*', + '`IArchivePurgeable.Purge` (transactional default for `IArchiveModifiable`)') +v = v.replace('A dedicated **purge (empty-all)** verb has no interface yet; it is realised by\n `IArchiveModifiable.Remove` over all entries (or a fresh empty `Create`). A\n future `IArchivePurgeable` could formalise it.', + '**purge (empty-all)** is `IArchivePurgeable`. `IArchiveModifiable` inherits it;\n the default purge mutates a staged copy, verifies that every original live entry is gone,\n and only then replaces the source.') +v = v.replace('Find and apply the **best parameter set** for the data (cluster/block/inode size, geometry, alignment).', + 'Find and apply the **best parameter set** for the data (cluster/block/inode size, geometry, alignment, compression method/level, dictionary and solid-block grouping).') +write(model, v) + +# Test source comments should no longer claim defrag is filesystem-only. +defrag_test = Path('Compression.Tests/Operations/GenericDefragRoundTripTests.cs') +v = read(defrag_test) +v = v.replace('filesystem descriptor', 'archive/filesystem descriptor') +write(defrag_test, v) + +# Remove this one-shot machinery from the resulting product commit. +Path('.github/maintenance-capabilities.py').unlink(missing_ok=True) +Path('.github/workflows/maintenance-capabilities-once.yml').unlink(missing_ok=True) From 375dc3be552849e194242f2e9e5f3769d5a2b4a5 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:19:28 +0200 Subject: [PATCH 042/192] ci: run maintenance capability audit --- .../maintenance-capabilities-once.yml | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/maintenance-capabilities-once.yml diff --git a/.github/workflows/maintenance-capabilities-once.yml b/.github/workflows/maintenance-capabilities-once.yml new file mode 100644 index 000000000..ca65b8b75 --- /dev/null +++ b/.github/workflows/maintenance-capabilities-once.yml @@ -0,0 +1,48 @@ +name: Maintenance capability audit one-shot + +on: + push: + branches: + - feat/filesystem-rw-gaps + +permissions: + contents: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + promote: + if: github.event.head_commit.message == 'ci: run maintenance capability audit' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: feat/filesystem-rw-gaps + + - name: Apply capability promotions and generate matrices + run: python .github/maintenance-capabilities.py + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: 'preview' + + - name: Clone PNGCrushCS sibling + run: git clone --depth 1 https://github.com/Hawkynt/PNGCrushCS.git "$GITHUB_WORKSPACE/../PNGCrushCS" + + - name: Build test project + run: dotnet build Compression.Tests/Compression.Tests.csproj -c Release + + - name: Run maintenance contract tests + run: dotnet test Compression.Tests/Compression.Tests.csproj -c Release --no-build --filter "FullyQualifiedName~MarkerInterfaceCoverageTests|FullyQualifiedName~WriteCapabilityHonestyTests" + + - name: Commit promoted capabilities + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "+ expose and promote maintenance capabilities" + git push origin HEAD:feat/filesystem-rw-gaps From c00cac71420c23b76988e07ef2c45e83fccc6b5e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:21:19 +0200 Subject: [PATCH 043/192] ci: simplify maintenance capability audit runner --- .github/workflows/maintenance-capabilities-once.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maintenance-capabilities-once.yml b/.github/workflows/maintenance-capabilities-once.yml index ca65b8b75..32850f589 100644 --- a/.github/workflows/maintenance-capabilities-once.yml +++ b/.github/workflows/maintenance-capabilities-once.yml @@ -13,7 +13,6 @@ env: jobs: promote: - if: github.event.head_commit.message == 'ci: run maintenance capability audit' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -36,7 +35,7 @@ jobs: run: dotnet build Compression.Tests/Compression.Tests.csproj -c Release - name: Run maintenance contract tests - run: dotnet test Compression.Tests/Compression.Tests.csproj -c Release --no-build --filter "FullyQualifiedName~MarkerInterfaceCoverageTests|FullyQualifiedName~WriteCapabilityHonestyTests" + run: dotnet test Compression.Tests/Compression.Tests.csproj -c Release --no-build --filter 'FullyQualifiedName~MarkerInterfaceCoverageTests|FullyQualifiedName~WriteCapabilityHonestyTests' - name: Commit promoted capabilities run: | From 5286bef9a781d1c98f657240b7a6c5e78fc3259b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 28 Aug 2026 12:39:05 +0200 Subject: [PATCH 044/192] ci: stage live rebuild progress and cancellation UX --- .github/maintenance-ux.py | 941 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 941 insertions(+) create mode 100644 .github/maintenance-ux.py diff --git a/.github/maintenance-ux.py b/.github/maintenance-ux.py new file mode 100644 index 000000000..f9f83ea5b --- /dev/null +++ b/.github/maintenance-ux.py @@ -0,0 +1,941 @@ +from pathlib import Path + + +def read(path: str) -> str: + return Path(path).read_text(encoding='utf-8') + + +def write(path: str, content: str) -> None: + Path(path).write_text(content, encoding='utf-8') + + +# ── Generic rebuild engine: real progress + cancellation + staged commit ───── +write('Compression.Registry/RebuildVerb.cs', r'''namespace Compression.Registry; + +/// +/// Generic, round-trip-verified extract → re-create engine shared by maintenance +/// verbs. Rebuilds are staged, verified, progress-reporting, and cancellable; +/// the caller's original stream is not touched until the staged target has been +/// built successfully and cancellation is no longer accepted. +/// +public static class RebuildVerb { + + /// + /// Extracts every live entry, re-creates the container in , + /// verifies the exact live-name multiset, and reports block-map/read/write-head + /// progress suitable for the maintenance UI. + /// + public static int RebuildToStream( + Stream input, + Stream output, + IArchiveFormatOperations ops, + IArchiveCreatable creator, + IReadOnlyDictionary? formatSpecific = null, + IReadOnlySet? syntheticNames = null, + Action? onProgress = null, + CancellationToken cancellationToken = default) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(ops); + ArgumentNullException.ThrowIfNull(creator); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("Rebuild input must be readable and seekable.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("Rebuild output must be writable and seekable.", nameof(output)); + + cancellationToken.ThrowIfCancellationRequested(); + input.Position = 0; + var sourceEntries = ops.List(input, null); + var sourceNames = LiveNameList(sourceEntries); + var sourceFileCount = sourceNames.Count; + var sourceLength = Math.Max(1L, input.Length); + var liveEntries = sourceEntries + .Where(e => !e.IsDirectory && (syntheticNames == null || !syntheticNames.Contains(e.Name))) + .ToArray(); + var totalLogical = Math.Max(1L, liveEntries.Sum(e => Math.Max(0L, e.OriginalSize))); + var sourceLayout = BuildSourceLayout(input, ops, sourceEntries); + + onProgress?.Invoke(new DefragProgressEvent( + "scanning", 0, 0, -1, sourceLength, sourceLayout, + $"Scanning {sourceFileCount:N0} live entr{(sourceFileCount == 1 ? "y" : "ies")} before staged rebuild")); + + var tmpDir = Path.Combine(Path.GetTempPath(), "cwb_rebuild_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(tmpDir); + try { + // Extract entry-by-entry rather than through the opaque bulk Extract call. + // This makes large WORM/re-layout passes cancellable and gives the UI an + // honest moving read head while bytes are consumed from the source. + long logicalDone = 0; + for (var i = 0; i < sourceEntries.Count; i++) { + cancellationToken.ThrowIfCancellationRequested(); + var entry = sourceEntries[i]; + var target = SafeExtractPath(tmpDir, entry.Name); + if (entry.IsDirectory) { + Directory.CreateDirectory(target); + continue; + } + if (syntheticNames != null && syntheticNames.Contains(entry.Name)) + continue; + + var parent = Path.GetDirectoryName(target); + if (!string.IsNullOrEmpty(parent)) Directory.CreateDirectory(parent); + + var entrySize = Math.Max(0L, entry.OriginalSize); + onProgress?.Invoke(new DefragProgressEvent( + "reading", 0.45 * logicalDone / totalLogical, + ScaleOffset(logicalDone, totalLogical, sourceLength), -1, + sourceLength, HighlightEntry(sourceLayout, entry.Name), + $"Reading {i + 1:N0}/{sourceEntries.Count:N0}: {entry.Name}")); + + input.Position = 0; + using var src = ops.OpenEntry(input, entry.Name, null); + using var dst = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 64 * 1024, + FileOptions.SequentialScan); + var buffer = new byte[64 * 1024]; + long entryDone = 0; + var lastReport = Environment.TickCount64; + while (true) { + cancellationToken.ThrowIfCancellationRequested(); + var read = src.Read(buffer, 0, buffer.Length); + if (read <= 0) break; + dst.Write(buffer, 0, read); + entryDone += read; + var now = Environment.TickCount64; + if (now - lastReport >= 50) { + lastReport = now; + var effective = logicalDone + Math.Min(entrySize > 0 ? entrySize : entryDone, entryDone); + var fraction = 0.45 * Math.Clamp((double)effective / totalLogical, 0, 1); + onProgress?.Invoke(new DefragProgressEvent( + "reading", fraction, + ScaleOffset(effective, totalLogical, sourceLength), -1, + sourceLength, null, + $"Reading {i + 1:N0}/{sourceEntries.Count:N0}: {entry.Name} ({entryDone:N0} bytes)")); + } + } + dst.Flush(flushToDisk: true); + logicalDone += entrySize > 0 ? entrySize : entryDone; + } + + cancellationToken.ThrowIfCancellationRequested(); + + var inputs = new List(); + foreach (var dir in Directory.GetDirectories(tmpDir, "*", SearchOption.AllDirectories)) { + var rel = Path.GetRelativePath(tmpDir, dir).Replace('\\', '/'); + inputs.Add(new ArchiveInputInfo("", rel + "/", true)); + } + foreach (var file in Directory.GetFiles(tmpDir, "*", SearchOption.AllDirectories)) { + var rel = Path.GetRelativePath(tmpDir, file).Replace('\\', '/'); + inputs.Add(new ArchiveInputInfo(file, rel, false)); + } + + var visualSize = Math.Max(sourceLength, totalLogical); + var targetLayout = BuildWeightedLayout(liveEntries, visualSize); + onProgress?.Invoke(new DefragProgressEvent( + "writing", 0.45, -1, 0, visualSize, targetLayout, + "Building staged target — original container is still unchanged")); + + var options = new FormatCreateOptions { FormatSpecific = formatSpecific }; + output.Position = 0; + output.SetLength(0); + using (var progressOutput = new ProgressWriteStream(output, cancellationToken, maxPosition => { + var fraction = 0.45 + 0.45 * Math.Clamp((double)maxPosition / sourceLength, 0, 1); + onProgress?.Invoke(new DefragProgressEvent( + "writing", fraction, -1, maxPosition, visualSize, null, + $"Writing staged target: {maxPosition:N0} bytes")); + })) { + creator.Create(progressOutput, inputs, options); + progressOutput.Flush(); + } + + cancellationToken.ThrowIfCancellationRequested(); + onProgress?.Invoke(new DefragProgressEvent( + "verifying", 0.92, -1, Math.Max(0, output.Position), Math.Max(1, output.Length), null, + "Verifying rebuilt container before commit")); + + output.Position = 0; + List rebuiltNames; + try { + rebuiltNames = LiveNameList(ops.List(output, null)); + } catch (Exception ex) { + throw new InvalidOperationException( + $"Rebuilt image could not be listed back ({ex.GetType().Name}: {ex.Message}); refusing a lossy rebuild.", ex); + } + if (!rebuiltNames.SequenceEqual(sourceNames, StringComparer.Ordinal)) + throw new InvalidOperationException( + $"Rebuild changed the entry set ({sourceFileCount} → {rebuiltNames.Count}); refusing a non-identity-preserving rebuild."); + + cancellationToken.ThrowIfCancellationRequested(); + var finalLength = Math.Max(1L, output.Length); + onProgress?.Invoke(new DefragProgressEvent( + "staged", 0.98, -1, Math.Max(0, output.Length - 1), finalLength, + BuildWeightedLayout(liveEntries, finalLength), + "Staged rebuild verified; ready to commit")); + output.Position = 0; + return sourceFileCount; + } finally { + try { Directory.Delete(tmpDir, true); } catch { /* best effort */ } + } + } + + /// + /// Rebuilds into a scratch file, verifies it, and only then replaces the + /// caller-supplied stream. Cancellation is honoured until commit starts; + /// once commit begins it runs to completion so a cancellation cannot leave + /// the original half-overwritten. + /// + public static void RebuildInPlace( + Stream archive, + IArchiveFormatOperations ops, + IArchiveCreatable creator, + IReadOnlyDictionary? formatSpecific = null, + Action? onProgress = null, + CancellationToken cancellationToken = default) { + ArgumentNullException.ThrowIfNull(archive); + using var rebuilt = CreateScratchStream(); + RebuildToStream(archive, rebuilt, ops, creator, formatSpecific, + onProgress: onProgress, cancellationToken: cancellationToken); + + // This is the point of no return. Do not inspect cancellation again after + // announcing commit: callers can disable the Cancel button for this phase. + onProgress?.Invoke(new DefragProgressEvent( + "committing", 0.99, -1, 0, Math.Max(1, rebuilt.Length), null, + "Committing verified staged target — cancellation is no longer safe")); + + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + + var finalLength = Math.Max(1L, archive.Length); + archive.Position = 0; + var entries = ops.List(archive, null); + onProgress?.Invoke(new DefragProgressEvent( + "complete", 1, -1, -1, finalLength, + BuildWeightedLayout(entries.Where(e => !e.IsDirectory).ToArray(), finalLength), + "Rebuild committed successfully")); + } + + /// + /// Rebuild-based edit used by the generic modifier. The mutation and rebuilt + /// validation happen off to the side; the original is overwritten only after + /// a valid staged result exists. + /// + public static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, + IArchiveCreatable creator, Action mutate) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(mutate); + var tmpDir = Path.Combine(Path.GetTempPath(), "cwb_edit_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(tmpDir); + try { + archive.Position = 0; + ops.Extract(archive, tmpDir, null, null); + mutate(tmpDir); + + var inputs = new List(); + foreach (var dir in Directory.GetDirectories(tmpDir, "*", SearchOption.AllDirectories)) { + var rel = Path.GetRelativePath(tmpDir, dir).Replace('\\', '/'); + inputs.Add(new ArchiveInputInfo("", rel + "/", true)); + } + foreach (var file in Directory.GetFiles(tmpDir, "*", SearchOption.AllDirectories)) { + var rel = Path.GetRelativePath(tmpDir, file).Replace('\\', '/'); + inputs.Add(new ArchiveInputInfo(file, rel, false)); + } + + using var rebuilt = CreateScratchStream(); + creator.Create(rebuilt, inputs, new FormatCreateOptions()); + rebuilt.Position = 0; + _ = ops.List(rebuilt, null); + + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + } finally { + try { Directory.Delete(tmpDir, true); } catch { /* best effort */ } + } + } + + /// + /// Transactional purge for a mutable container. The modifier operates on a + /// staged copy and the caller's stream is replaced only after the result lists + /// successfully with every original live entry gone. + /// + public static void PurgeViaModifier(Stream archive, IArchiveFormatOperations ops, IArchiveModifiable modifier) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(ops); + ArgumentNullException.ThrowIfNull(modifier); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Purge requires a readable, writable, seekable stream.", nameof(archive)); + + using var staged = CreateScratchStream(); + archive.Position = 0; + archive.CopyTo(staged); + staged.Flush(); + + staged.Position = 0; + var sourceNames = ops.List(staged, null) + .Where(e => !e.IsDirectory) + .Select(e => e.Name) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (sourceNames.Length == 0) return; + + staged.Position = 0; + modifier.Remove(staged, sourceNames); + staged.Position = 0; + var remaining = ops.List(staged, null) + .Where(e => !e.IsDirectory) + .Select(e => e.Name) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var survivors = sourceNames.Where(remaining.Contains).ToArray(); + if (survivors.Length != 0) + throw new InvalidOperationException( + $"Purge left {survivors.Length} original live entr{(survivors.Length == 1 ? "y" : "ies")} behind; original container retained."); + + archive.Position = 0; + archive.SetLength(0); + staged.Position = 0; + staged.CopyTo(archive); + archive.Flush(); + } + + private static List LiveNameList(IEnumerable entries) + => entries.Where(e => !e.IsDirectory).Select(e => e.Name) + .OrderBy(n => n, StringComparer.Ordinal).ToList(); + + private static IReadOnlyList BuildSourceLayout( + Stream input, IArchiveFormatOperations ops, IReadOnlyList entries) { + try { + input.Position = 0; + var extents = ops switch { + IFilesystemExtentMap fs => fs.EnumerateExtents(input).ToArray(), + IArchiveLayoutMap archive => archive.EnumerateLayout(input).ToArray(), + _ => [], + }; + if (extents.Length > 0) return extents; + } catch { + // Visualization is best-effort; the actual rebuild remains fully verified. + } + return BuildWeightedLayout(entries.Where(e => !e.IsDirectory).ToArray(), Math.Max(1, input.Length)); + } + + private static IReadOnlyList BuildWeightedLayout( + IReadOnlyList entries, long totalSize) { + totalSize = Math.Max(1, totalSize); + if (entries.Count == 0) + return [new DefragBlockInfo(0, totalSize, DefragBlockKind.Free)]; + + var weights = entries.Select(e => Math.Max(1L, e.OriginalSize)).ToArray(); + var totalWeight = Math.Max(1L, weights.Sum()); + var result = new List(entries.Count); + long cumulative = 0; + long cursor = 0; + for (var i = 0; i < entries.Count; i++) { + cumulative += weights[i]; + var end = i == entries.Count - 1 + ? totalSize + : (long)((double)cumulative / totalWeight * totalSize); + end = Math.Clamp(end, cursor, totalSize); + var length = end - cursor; + if (length > 0) + result.Add(new DefragBlockInfo(cursor, length, DefragBlockKind.Used, + entries[i].Name, Classify(entries[i].Method))); + cursor = end; + } + if (cursor < totalSize) + result.Add(new DefragBlockInfo(cursor, totalSize - cursor, DefragBlockKind.Free)); + return result; + } + + private static IReadOnlyList HighlightEntry( + IReadOnlyList source, string entryName) { + var changed = false; + var result = new DefragBlockInfo[source.Count]; + for (var i = 0; i < source.Count; i++) { + var block = source[i]; + if (block.FileName != null && string.Equals(block.FileName, entryName, StringComparison.Ordinal)) { + result[i] = block with { Kind = DefragBlockKind.InProgress }; + changed = true; + } else { + result[i] = block; + } + } + return changed ? result : source; + } + + private static DefragBlockClass Classify(string? method) { + var value = (method ?? "").ToUpperInvariant(); + if (value.Contains("STORE") || value.Contains("COPY") || value == "NONE" || value.Length == 0) + return DefragBlockClass.Frozen; + if (value.Contains("LZMA") || value.Contains("PPMD") || value.Contains("BZIP")) + return DefragBlockClass.Hot; + if (value.Contains("ZSTD") || value.Contains("LZ4")) + return DefragBlockClass.Cold; + return DefragBlockClass.Normal; + } + + private static long ScaleOffset(long done, long total, long imageSize) + => total <= 0 ? 0 : Math.Clamp((long)((double)done / total * imageSize), 0, Math.Max(0, imageSize - 1)); + + private static string SafeExtractPath(string root, string archiveName) { + var normalized = archiveName.Replace('/', Path.DirectorySeparatorChar).TrimStart(Path.DirectorySeparatorChar); + var rootFull = Path.GetFullPath(root) + Path.DirectorySeparatorChar; + var candidate = Path.GetFullPath(Path.Combine(root, normalized)); + if (!candidate.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase) && + !string.Equals(candidate, rootFull.TrimEnd(Path.DirectorySeparatorChar), StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Entry path escapes rebuild staging directory: {archiveName}"); + return candidate; + } + + /// A writable scratch stream not bounded by byte[] / MemoryStream size. + internal static FileStream CreateScratchStream() + => new(Path.Combine(Path.GetTempPath(), "cwb_rebuild_" + Guid.NewGuid().ToString("N") + ".tmp"), + FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.DeleteOnClose); + + /// + /// Write-through wrapper used solely to expose actual target-byte progress and + /// make long encoder writes cancellable without giving ownership of the target + /// stream to the wrapper. + /// + private sealed class ProgressWriteStream( + Stream inner, CancellationToken cancellationToken, Action report) : Stream { + private long _maxPosition; + private long _lastReportTick; + + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => inner.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) { + cancellationToken.ThrowIfCancellationRequested(); + inner.Write(buffer, offset, count); + Report(); + } + + public override void Write(ReadOnlySpan buffer) { + cancellationToken.ThrowIfCancellationRequested(); + inner.Write(buffer); + Report(); + } + + public override void WriteByte(byte value) { + cancellationToken.ThrowIfCancellationRequested(); + inner.WriteByte(value); + Report(); + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, + CancellationToken cancellationTokenFromCaller = default) { + cancellationToken.ThrowIfCancellationRequested(); + cancellationTokenFromCaller.ThrowIfCancellationRequested(); + await inner.WriteAsync(buffer, cancellationTokenFromCaller).ConfigureAwait(false); + Report(); + } + + private void Report() { + _maxPosition = Math.Max(_maxPosition, inner.Position); + var now = Environment.TickCount64; + if (now - _lastReportTick < 50) return; + _lastReportTick = now; + report(_maxPosition); + } + + protected override void Dispose(bool disposing) { + if (disposing) { + try { inner.Flush(); } catch { } + report(Math.Max(_maxPosition, inner.CanSeek ? inner.Position : _maxPosition)); + } + base.Dispose(disposing); + } + } +} +''') + + +# ── Defrag options/default implementation: cancellation reaches rebuild ────── +defrag_options = read('Compression.Registry/DefragOptions.cs') +needle = ''' public Action? OnProgress { get; init; }\n''' +addition = ''' public Action? OnProgress { get; init; }\n\n /// \n /// Cooperative cancellation for long maintenance operations. Generic staged\n /// rebuilds honour it while reading and writing and never commit a cancelled\n /// target. Native in-place movers may honour it at their next safe move boundary.\n /// \n public CancellationToken CancellationToken { get; init; }\n''' +if needle in defrag_options and 'public CancellationToken CancellationToken' not in defrag_options: + defrag_options = defrag_options.replace(needle, addition, 1) +write('Compression.Registry/DefragOptions.cs', defrag_options) + +write('Compression.Registry/IArchiveDefragmentable.cs', r'''namespace Compression.Registry; + +/// +/// Opt-in capability for physical or logical re-layout. Native mutable filesystems +/// may move extents in place; WORM/archive containers can satisfy the same verb by +/// building a verified staged target and committing it after completion. +/// +public interface IArchiveDefragmentable { + /// Defragments using the format's default consolidate-at-start strategy. + void Defragment(Stream archive) { + if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) + throw new NotSupportedException( + "The default Defragment requires IArchiveFormatOperations + IArchiveCreatable."); + RebuildVerb.RebuildInPlace(archive, ops, creator); + } + + /// + /// Rewrites according to . The default implementation + /// uses the progress-reporting/cancellable staged rebuild for descriptors that + /// rely on the interface default, while preserving a descriptor's own native + /// parameterless implementation when it has one. + /// + void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException( + $"This descriptor only supports DefragMode.ConsolidateAtStart; got {options.Mode}."); + + // If the concrete descriptor has a public native parameterless mover, retain + // its semantics. Generic promoted descriptors have no such method and use the + // staged rebuild below, gaining smooth progress + safe cancellation for free. + var native = this.GetType().GetMethod(nameof(Defragment), [typeof(Stream)]); + if (native != null && native.DeclaringType != typeof(IArchiveDefragmentable)) { + options.CancellationToken.ThrowIfCancellationRequested(); + this.Defragment(archive); + return; + } + + if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) { + this.Defragment(archive); + return; + } + RebuildVerb.RebuildInPlace(archive, ops, creator, + onProgress: options.OnProgress, cancellationToken: options.CancellationToken); + } +} +''') + + +# ── 7z regrouping: cancellable phase-level progress ────────────────────────── +solid = read('FileFormats/FileFormat.SevenZip/SolidBlockOptimizer.cs') +if 'public sealed record DetailedProgress' not in solid: + solid = solid.replace( +''' public sealed class TrialResult {\n public required string StrategyName { get; init; }\n public required long OutputSize { get; init; }\n public required TimeSpan Elapsed { get; init; }\n }\n''', +''' public sealed class TrialResult {\n public required string StrategyName { get; init; }\n public required long OutputSize { get; init; }\n public required TimeSpan Elapsed { get; init; }\n }\n\n /// Detailed progress for the block-map UI during extraction/regrouping.\n public sealed record DetailedProgress(\n string Phase, int Current, int Total, string? Name, long BytesDone, long BytesTotal);\n''') +solid = solid.replace( +''' public static OptimizeResult Optimize(Stream archive, int maxTrials = 5, ProgressCallback? onProgress = null) {''', +''' public static OptimizeResult Optimize(Stream archive, int maxTrials = 5, ProgressCallback? onProgress = null,\n Action? onDetailedProgress = null, CancellationToken cancellationToken = default) {''') +solid = solid.replace( +''' // Step 1: Extract all entries from the input archive\n archive.Position = 0;\n var reader = new SevenZipReader(archive, leaveOpen: true);\n var entries = new List<(string Name, byte[] Data, SevenZipEntry Meta)>();\n for (var i = 0; i < reader.Entries.Count; i++) {\n var e = reader.Entries[i];\n if (e.IsDirectory) continue;\n var data = reader.Extract(i);\n entries.Add((e.Name, data, e));\n }''', +''' // Step 1: Extract all entries from the input archive. This is an actual\n // progress source for the block-map read head rather than an indeterminate spinner.\n archive.Position = 0;\n var reader = new SevenZipReader(archive, leaveOpen: true);\n var fileEntries = reader.Entries.Where(e => !e.IsDirectory).ToArray();\n var totalBytes = Math.Max(1L, fileEntries.Sum(e => Math.Max(0L, e.Size)));\n long extractedBytes = 0;\n var entries = new List<(string Name, byte[] Data, SevenZipEntry Meta)>();\n for (var i = 0; i < reader.Entries.Count; i++) {\n cancellationToken.ThrowIfCancellationRequested();\n var e = reader.Entries[i];\n if (e.IsDirectory) continue;\n onDetailedProgress?.Invoke(new DetailedProgress(\n "extracting", entries.Count, fileEntries.Length, e.Name, extractedBytes, totalBytes));\n var data = reader.Extract(i);\n extractedBytes += data.LongLength;\n entries.Add((e.Name, data, e));\n onDetailedProgress?.Invoke(new DetailedProgress(\n "extracting", entries.Count, fileEntries.Length, e.Name, extractedBytes, totalBytes));\n }''') +solid = solid.replace( +''' for (var i = 0; i < strategies.Count; i++) {\n var (name, grouper) = strategies[i];\n onProgress?.Invoke(i, strategies.Count, name);\n\n var sw = System.Diagnostics.Stopwatch.StartNew();\n try {\n var groups = grouper(entries);\n var output = BuildArchive(entries, groups);''', +''' for (var i = 0; i < strategies.Count; i++) {\n cancellationToken.ThrowIfCancellationRequested();\n var (name, grouper) = strategies[i];\n onProgress?.Invoke(i, strategies.Count, name);\n onDetailedProgress?.Invoke(new DetailedProgress(\n "strategy", i, strategies.Count, name, i, strategies.Count));\n\n var sw = System.Diagnostics.Stopwatch.StartNew();\n try {\n var groups = grouper(entries);\n var output = BuildArchive(entries, groups, cancellationToken,\n (current, total, entryName) => onDetailedProgress?.Invoke(new DetailedProgress(\n "building", current, total, entryName, current, total)));''') +solid = solid.replace( +''' private static byte[] BuildArchive(\n IReadOnlyList<(string Name, byte[] Data, SevenZipEntry Meta)> entries,\n IReadOnlyList groups) {''', +''' private static byte[] BuildArchive(\n IReadOnlyList<(string Name, byte[] Data, SevenZipEntry Meta)> entries,\n IReadOnlyList groups, CancellationToken cancellationToken,\n Action? onProgress) {''') +solid = solid.replace( +''' foreach (var group in groups)\n foreach (var idx in group) {\n var (name, data, meta) = entries[idx];''', +''' foreach (var group in groups)\n foreach (var idx in group) {\n cancellationToken.ThrowIfCancellationRequested();\n var (name, data, meta) = entries[idx];\n onProgress?.Invoke(addOrder, entries.Count, name);''') +solid = solid.replace( +''' entryIndexMap[idx] = addOrder++;\n }''', +''' entryIndexMap[idx] = addOrder++;\n onProgress?.Invoke(addOrder, entries.Count, name);\n }''', 1) +solid = solid.replace( +''' writer.FinishWithBlocks(blockDescs);''', +''' cancellationToken.ThrowIfCancellationRequested();\n writer.FinishWithBlocks(blockDescs);\n cancellationToken.ThrowIfCancellationRequested();''') +write('FileFormats/FileFormat.SevenZip/SolidBlockOptimizer.cs', solid) + + +# ── Maintenance window: block-map is always the progress surface ───────────── +xaml = read('Compression.UI/Views/DefragmentWindow.xaml') +xaml = xaml.replace( +''' ''', +''' ''') +xaml = xaml.replace( +'''