diff --git a/Compression.Tests/Wacz/WaczWriterTests.cs b/Compression.Tests/Wacz/WaczWriterTests.cs new file mode 100644 index 000000000..e5a364b86 --- /dev/null +++ b/Compression.Tests/Wacz/WaczWriterTests.cs @@ -0,0 +1,95 @@ +using System.Text; +using Compression.Registry; +using FileFormat.Wacz; +using FileFormat.Zip; + +namespace Compression.Tests.Wacz; + +[TestFixture] +public sealed class WaczWriterTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Create_WritesValidContainerThatReaderLists() { + ArchiveInputInfo[] inputs = [ + ArchiveInputInfo.InMemory("datapackage.json", "{\"profile\":\"data-package\",\"resources\":[],\"wacz_version\":\"1.1.1\"}"u8.ToArray()), + ArchiveInputInfo.InMemory("archive/data.warc.gz", new byte[] { 0x1F, 0x8B, 0x08, 0x00 }), + ArchiveInputInfo.InMemory("pages/pages.jsonl", "{\"format\":\"json-pages-1.0\"}\n{\"url\":\"https://example.test\",\"ts\":\"2026-01-01T00:00:00Z\"}\n"u8.ToArray()), + ArchiveInputInfo.InMemory("indexes/index.cdxj", "com,example)/ 20260101000000 {\"url\":\"https://example.test\"}\n"u8.ToArray()), + ]; + + using var output = new MemoryStream(); + var descriptor = new WaczFormatDescriptor(); + descriptor.Create(output, inputs, new FormatCreateOptions()); + + output.Position = 0; + var entries = descriptor.List(output, null).Select(entry => entry.Name).ToArray(); + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(entries, Does.Contain("datapackage.json")); + Assert.That(entries, Does.Contain("archive/data.warc.gz")); + Assert.That(entries, Does.Contain("pages/pages.jsonl")); + Assert.That(entries, Does.Contain("indexes/index.cdxj")); + }); + + output.Position = 0; + using var zip = new ZipReader(output, leaveOpen: true); + var warc = zip.Entries.Single(entry => entry.FileName == "archive/data.warc.gz"); + Assert.That(warc.CompressionMethod, Is.EqualTo(ZipCompressionMethod.Store), + "already-gzipped WARC members should not be deflated a second time"); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Create_GenericFiles_PreservesPayloadAndSynthesizesWebArchiveResources() { + var payload = "generic archive payload\n"u8.ToArray(); + ArchiveInputInfo[] inputs = [ArchiveInputInfo.InMemory("docs/readme.txt", payload)]; + using var output = new MemoryStream(); + var descriptor = new WaczFormatDescriptor(); + + descriptor.Create(output, inputs, new FormatCreateOptions()); + + output.Position = 0; + using (var zip = new ZipReader(output, leaveOpen: true)) { + var names = zip.Entries.Select(entry => entry.FileName).ToArray(); + Assert.Multiple(() => { + Assert.That(names, Does.Contain("docs/readme.txt")); + Assert.That(names, Does.Contain("archive/data.warc")); + Assert.That(names, Does.Contain("indexes/index.cdxj")); + Assert.That(names, Does.Contain("pages/pages.jsonl")); + Assert.That(names, Does.Contain("datapackage.json")); + }); + + var warcEntry = zip.Entries.Single(entry => entry.FileName == "archive/data.warc"); + Assert.That(warcEntry.CompressionMethod, Is.EqualTo(ZipCompressionMethod.Store)); + + var manifestEntry = zip.Entries.Single(entry => entry.FileName == "datapackage.json"); + var manifest = Encoding.UTF8.GetString(zip.ExtractEntry(manifestEntry)); + Assert.Multiple(() => { + Assert.That(manifest, Does.Contain("\"profile\": \"data-package\"")); + Assert.That(manifest, Does.Contain("\"wacz_version\": \"1.1.1\"")); + Assert.That(manifest, Does.Contain("\"path\": \"docs/readme.txt\"")); + Assert.That(manifest, Does.Contain("sha256:")); + }); + } + + output.Position = 0; + Assert.That(descriptor.ExtractEntryToMemory(output, "docs/readme.txt", null), Is.EqualTo(payload).AsCollection); + } + + [Test, Category("HappyPath")] + public void Create_SuppliedWarcWithoutManifest_GeneratesManifest() { + ArchiveInputInfo[] inputs = [ + ArchiveInputInfo.InMemory("archive/data.warc", "WARC/1.0\r\nWARC-Type: resource\r\nWARC-Record-ID: \r\nWARC-Date: 2026-01-01T00:00:00Z\r\nContent-Length: 0\r\n\r\n\r\n\r\n"u8.ToArray()), + ]; + using var output = new MemoryStream(); + + new WaczFormatDescriptor().Create(output, inputs, new FormatCreateOptions()); + + output.Position = 0; + using var zip = new ZipReader(output, leaveOpen: true); + var manifest = zip.Entries.Single(entry => entry.FileName == "datapackage.json"); + var text = Encoding.UTF8.GetString(zip.ExtractEntry(manifest)); + Assert.Multiple(() => { + Assert.That(text, Does.Contain("\"profile\": \"data-package\"")); + Assert.That(text, Does.Contain("\"path\": \"archive/data.warc\"")); + }); + } +} diff --git a/FileFormats/FileFormat.Wacz/FileFormat.Wacz.csproj b/FileFormats/FileFormat.Wacz/FileFormat.Wacz.csproj index 70ab15c1d..c59c1afd8 100644 --- a/FileFormats/FileFormat.Wacz/FileFormat.Wacz.csproj +++ b/FileFormats/FileFormat.Wacz/FileFormat.Wacz.csproj @@ -6,6 +6,7 @@ + diff --git a/FileFormats/FileFormat.Wacz/WaczCreator.cs b/FileFormats/FileFormat.Wacz/WaczCreator.cs new file mode 100644 index 000000000..9caf7a913 --- /dev/null +++ b/FileFormats/FileFormat.Wacz/WaczCreator.cs @@ -0,0 +1,211 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Compression.Registry; +using FileFormat.Warc; +using FileFormat.Zip; + +namespace FileFormat.Wacz; + +/// Creates WACZ 1.x ZIP containers from caller-supplied web-archive resources. +internal static class WaczCreator { + private const string SyntheticDate = "1970-01-01T00:00:00Z"; + private const string SyntheticTimestamp = "19700101000000"; + private const string SyntheticWarcName = "archive/data.warc"; + private const string SyntheticIndexName = "indexes/index.cdxj"; + private const string SyntheticPagesName = "pages/pages.jsonl"; + + public static void Create(Stream output, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + + var files = new List<(string Name, byte[] Data)>(); + var names = new HashSet(StringComparer.Ordinal); + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var name = NormalizeName(input.ArchiveName); + if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) + continue; + if (!names.Add(name)) + throw new InvalidDataException($"WACZ input contains duplicate archive path '{name}'."); + files.Add((name, input.ReadContent())); + } + + var hasWarc = files.Any(file => IsWarcPath(file.Name)); + if (!hasWarc) { + // Archive conversion supplies an arbitrary file tree, not WARC-specific inputs. + // Preserve those files verbatim as legal root/custom WACZ resources and also + // wrap them into a deterministic WARC so the resulting package remains a web + // archive rather than merely a ZIP with a .wacz extension. + var payloads = files + .Where(file => !string.Equals(file.Name, "datapackage.json", StringComparison.Ordinal)) + .ToArray(); + + RemoveFile(files, names, "datapackage.json"); + AddGeneratedFile(files, names, SyntheticWarcName, BuildSyntheticWarc(payloads, out var captures)); + AddGeneratedFile(files, names, SyntheticIndexName, BuildSyntheticIndex(captures)); + AddGeneratedFile(files, names, SyntheticPagesName, BuildSyntheticPages(captures)); + AddGeneratedFile(files, names, "datapackage.json", BuildManifest(files)); + } else if (!names.Contains("datapackage.json")) { + // A caller that already supplied WARC/index/page resources can omit the manifest; + // generate its mandatory fixity inventory from the exact bytes we will store. + AddGeneratedFile(files, names, "datapackage.json", BuildManifest(files)); + } + + files.Sort((a, b) => StringComparer.Ordinal.Compare(a.Name, b.Name)); + + using var zip = new ZipWriter(output, leaveOpen: true); + foreach (var (name, data) in files) { + // WACZ 1.1.1 says WARC files SHOULD be ZIP-stored for range access and + // already-compressed files MUST NOT be compressed a second time. + var method = name.StartsWith("archive/", StringComparison.Ordinal) + || name.EndsWith(".gz", StringComparison.OrdinalIgnoreCase) + ? ZipCompressionMethod.Store + : ZipCompressionMethod.Deflate; + zip.AddEntry(name, data, method); + } + } + + private static byte[] BuildSyntheticWarc( + IReadOnlyList<(string Name, byte[] Data)> payloads, + out List captures) { + captures = []; + using var combined = new MemoryStream(); + + if (payloads.Count == 0) { + payloads = [("empty", Array.Empty())]; + } + + foreach (var (name, data) in payloads) { + var targetUrl = ToSyntheticUrl(name); + var digest = Convert.ToHexStringLower(SHA256.HashData(data)); + var entry = new WarcEntry { + Type = "resource", + TargetUri = targetUrl, + RecordId = $"", + Date = SyntheticDate, + ContentType = "application/octet-stream", + ContentLength = data.Length, + }; + + using var recordStream = new MemoryStream(); + var writer = new WarcWriter(); + writer.AddRecord(entry, data); + writer.WriteTo(recordStream); + var record = recordStream.ToArray(); + var offset = combined.Position; + combined.Write(record); + captures.Add(new SyntheticCapture(name, targetUrl, digest, offset, record.Length, data.Length)); + } + + return combined.ToArray(); + } + + private static byte[] BuildSyntheticIndex(IEnumerable captures) { + var lines = new StringBuilder(); + foreach (var capture in captures.OrderBy(capture => capture.Url, StringComparer.Ordinal)) { + var fields = JsonSerializer.Serialize(new { + offset = capture.Offset.ToString(System.Globalization.CultureInfo.InvariantCulture), + length = capture.RecordLength.ToString(System.Globalization.CultureInfo.InvariantCulture), + mime = "application/octet-stream", + status = "-", + filename = "data.warc", + url = capture.Url, + digest = "sha256:" + capture.Digest, + }); + lines.Append(ToSurtKey(capture.Url)).Append(' ') + .Append(SyntheticTimestamp).Append(' ') + .Append(fields).Append('\n'); + } + return Encoding.UTF8.GetBytes(lines.ToString()); + } + + private static byte[] BuildSyntheticPages(IEnumerable captures) { + var lines = new StringBuilder(); + lines.AppendLine(JsonSerializer.Serialize(new { format = "json-pages-1.0", id = "pages", title = "Converted files" })); + foreach (var capture in captures.OrderBy(capture => capture.Url, StringComparer.Ordinal)) + lines.AppendLine(JsonSerializer.Serialize(new { + id = capture.Digest[..Math.Min(12, capture.Digest.Length)], + url = capture.Url, + ts = SyntheticDate, + title = capture.Name, + size = capture.PayloadLength, + })); + return Encoding.UTF8.GetBytes(lines.ToString()); + } + + private static byte[] BuildManifest(IEnumerable<(string Name, byte[] Data)> files) { + var resources = files + .Where(file => !string.Equals(file.Name, "datapackage.json", StringComparison.Ordinal)) + .OrderBy(file => file.Name, StringComparer.Ordinal) + .Select(file => new { + name = Path.GetFileName(file.Name), + path = file.Name, + hash = "sha256:" + Convert.ToHexStringLower(SHA256.HashData(file.Data)), + bytes = file.Data.Length, + }) + .ToArray(); + + var json = JsonSerializer.Serialize(new { + profile = "data-package", + wacz_version = "1.1.1", + title = "CompressionWorkbench converted archive", + created = SyntheticDate, + software = "CompressionWorkbench", + resources, + }, new JsonSerializerOptions { WriteIndented = true }); + return Encoding.UTF8.GetBytes(json + "\n"); + } + + private static bool IsWarcPath(string name) + => name.StartsWith("archive/", StringComparison.Ordinal) + && (name.EndsWith(".warc", StringComparison.OrdinalIgnoreCase) + || name.EndsWith(".warc.gz", StringComparison.OrdinalIgnoreCase)); + + private static void AddGeneratedFile( + List<(string Name, byte[] Data)> files, + HashSet names, + string name, + byte[] data) { + if (!names.Add(name)) + throw new InvalidDataException($"WACZ input conflicts with generated required path '{name}'."); + files.Add((name, data)); + } + + private static void RemoveFile(List<(string Name, byte[] Data)> files, HashSet names, string name) { + files.RemoveAll(file => string.Equals(file.Name, name, StringComparison.Ordinal)); + names.Remove(name); + } + + private static string ToSyntheticUrl(string name) + => "https://compression-workbench.invalid/" + string.Join('/', + name.Split('/').Select(Uri.EscapeDataString)); + + private static string ToSurtKey(string url) { + var uri = new Uri(url, UriKind.Absolute); + var hostParts = uri.Host.Split('.'); + Array.Reverse(hostParts); + return string.Join(',', hostParts) + ")" + uri.AbsolutePath; + } + + private static string NormalizeName(string name) { + if (string.IsNullOrWhiteSpace(name)) + throw new InvalidDataException("WACZ entries require a non-empty archive path."); + var normalized = name.Replace('\\', '/').TrimStart('/'); + if (normalized.Length == 0 || normalized.EndsWith('/')) + throw new InvalidDataException($"WACZ file path '{name}' is invalid."); + foreach (var component in normalized.Split('/')) + if (component is "" or "." or "..") + throw new InvalidDataException($"WACZ file path '{name}' contains an unsafe path component."); + return normalized; + } + + private sealed record SyntheticCapture( + string Name, + string Url, + string Digest, + long Offset, + int RecordLength, + int PayloadLength); +} diff --git a/FileFormats/FileFormat.Wacz/WaczFormatDescriptor.cs b/FileFormats/FileFormat.Wacz/WaczFormatDescriptor.cs index 1982e178c..5717e9608 100644 --- a/FileFormats/FileFormat.Wacz/WaczFormatDescriptor.cs +++ b/FileFormats/FileFormat.Wacz/WaczFormatDescriptor.cs @@ -35,7 +35,7 @@ namespace FileFormat.Wacz; /// about (title, version, software, page count, archive count). /// /// -public sealed class WaczFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveLayoutMap { +public sealed class WaczFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveLayoutMap { /// public IEnumerable EnumerateLayout(Stream archive) => ZipLayoutMap.Enumerate(archive); @@ -51,7 +51,7 @@ public sealed class WaczFormatDescriptor : IFormatDescriptor, IArchiveFormatOper /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; @@ -167,6 +167,10 @@ public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? pas return memoryStream.ToArray(); } + /// + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) + => WaczCreator.Create(output, inputs); + /// /// Throws unless the ZIP root looks like a WACZ /// (must contain datapackage.json and an archive/ directory). diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 6b7a34cac..ab3157b83 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -3222,7 +3222,7 @@ Implements `IFormatDescriptor`, `IStreamFormatOperations`. #### `CscStream` -CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size \| uint24 csc_blocksize \| uint24 raw_blocksize \| uint32 actual_size +CSC: Context Stream Compression by Fu Siyuan. Format: 10-byte big-endian property header + 4-byte uncompressed size, then range-coded LZ77. Header layout: uint32 dict_size | uint24 csc_blocksize | uint24 raw_blocksize | uint32 actual_size | Member | Signature | Summary | | --- | --- | --- | @@ -4820,7 +4820,7 @@ DEFLATE (RFC 1951) decoder for the PEtite dialect: block type 1 carries the dyna #### `PetiteUnpacker` -Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000\|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments) +Container format of a PEtite-packed Win32 PE, reconstructed from the on-disk layout and the entry stub of the samples themselves. A PEtite image keeps the original section virtual addresses. The packed bytes live in one oversized section mapped at the first original section's RVA; a second section holds the untouched resources; the last section (the one the entry point falls into) holds the loader stub. Right behind the stub code sits a block table that drives unpacking:a record whose first dword has bit 31 set is a descending `rep movsd` — `{0x80000000|dwordCount, sourceEndRva, destinationEndRva}`, 12 bytes — which lifts the packed bytes out of the way of the image that is about to be written over them;any other record is `{sourceRva, decompressedSize, destinationRva, unused}`, 16 bytes, and expands one original section in place. A zero length marks an original section without initialised data and is skipped; a zero source ends the table.The compressed streams are DEFLATE (RFC 1951) with one deviation: the stub has no fixed-Huffman tables, so block type `1` selects the dynamic Huffman tables that standard DEFLATE assigns to type `2`, and types 2 and 3 are rejected. Everything else — LSB-first bit order, the 14-bit HLIT/HDIST/HCLEN header, the code-length alphabet, the length/distance base and extra-bit tables — matches RFC 1951 byte for byte; those five tables are stored verbatim at the head of the stub section and were read from there.Code blocks are additionally stored with relative branch targets converted to absolute ones: scanning forward, every `E8`/`E9` and every `0F 80..0F 8F` has the block offset of its opcode added to the following dword, and the scan then skips the whole instruction. Reversing it subtracts the same offset again. References: `https://www.rfc-editor.org/rfc/rfc1951` — DEFLATE compressed data format`https://www.un4seen.com/petite/` — PEtite (Ian Luck / Un4seen Developments) | Member | Signature | Summary | | --- | --- | --- | @@ -9003,8 +9003,8 @@ WORM writer for the NumPy NPY array serialization format (NEP 1). Emits a v1 fil | Member | Signature | Summary | | --- | --- | --- | | `DefaultDtype` | `const string DefaultDtype` | Default dtype string used when no explicit type is supplied. | -| `Write` | `static void Write(Stream output, ReadOnlySpan payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. | -| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. | +| `Write` | `static void Write(Stream output, ReadOnlySpan payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Writes an NPY file from `payload` with the supplied dtype/shape header. When `shape` is null, a 1-D shape matching the payload's element count is inferred from the dtype's item-size. | +| `Write` | `static void Write(Stream output, byte[] payload, string dtype = "\|u1", string shape = null, bool fortranOrder = false)` | Convenience: writes an NPY file from a byte array. See span overload for parameter docs. | #### `NpzFormatDescriptor` @@ -11909,8 +11909,8 @@ Reader and writer for the Microsoft SZDD / COMPRESS.EXE file format. SZDD uses a | --- | --- | --- | | `CompressQBasic` | `static byte[] CompressQBasic(ReadOnlySpan data)` | Compresses `data` in the older "SZ " (QBasic) COMPRESS variant and returns the result. The body is the same LZSS stream as SZDD, wrapped in the 12-byte "SZ " header (8-byte magic + little-endian u32 uncompressed length). Round-trips through `Decompress`. | | `CompressQBasic` | `static void CompressQBasic(Stream input, Stream output)` | Stream overload of `CompressQBasic`. | -| `Compress` | `static byte[] Compress(ReadOnlySpan data, char missingChar = _)` | Compresses `data` in SZDD format and returns the result as a new byte array. | -| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = _)` | Compresses `input` in SZDD format and writes the result to `output`. | +| `Compress` | `static byte[] Compress(ReadOnlySpan data, char missingChar = '_')` | Compresses `data` in SZDD format and returns the result as a new byte array. | +| `Compress` | `static void Compress(Stream input, Stream output, char missingChar = '_')` | Compresses `input` in SZDD format and writes the result to `output`. | | `Decompress` | `static byte[] Decompress(ReadOnlySpan data)` | Decompresses an SZDD-encoded byte array and returns the raw data. | | `Decompress` | `static void Decompress(Stream input, Stream output)` | Decompresses an SZDD-encoded stream and writes the raw data to `output`. | | `GetMissingChar` | `static char GetMissingChar(Stream input)` | Returns the "missing character" stored in the SZDD header — the last character of the original filename extension before it was replaced with `'_'`. | @@ -13144,7 +13144,7 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati Descriptor for the WACZ (Web Archive Collection Zipped) format — a ZIP container that wraps one or more WARC files together with a Frictionless-Data manifest, page index and optional resource bundles. References: `https://specs.webrecorder.net/wacz/1.1.1/` — the WACZ 1.1.1 specification (Webrecorder)`https://webrecorder.net` — Webrecorder, the format's author and reference tooling (py-wacz, ReplayWeb.page) -Implements `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IFormatDescriptor`. | Member | Signature | Summary | | --- | --- | --- | @@ -13161,6 +13161,7 @@ Implements `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | | `Methods` | `IReadOnlyList Methods { get; }` | | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | | `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | | `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Native in-memory single-entry extraction routed through the bounded `OpenEntry`. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index af7ac0e76..b2bf80568 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -921,7 +921,7 @@ Reader for Intel HEX records (`:LLAAAATT[DD…]CC`), the long-standing flash-pro #### `SRecordReader` -Reader for Motorola S-Record files (`Stnn[aaaa\|aaaaaa\|aaaaaaaa]dd…cc`). Recognised types: S0 header, S1/S2/S3 data (16/24/32-bit address), S5/S6 record counts (informational), S7/S8/S9 termination (32/24/16-bit start addr). +Reader for Motorola S-Record files (`Stnn[aaaa|aaaaaa|aaaaaaaa]dd…cc`). Recognised types: S0 header, S1/S2/S3 data (16/24/32-bit address), S5/S6 record counts (informational), S7/S8/S9 termination (32/24/16-bit start addr). | Member | Signature | Summary | | --- | --- | --- | @@ -2820,7 +2820,7 @@ Random-access in-place modifier for BBC Micro Acorn DFS `.ssd` images. The DFS c | Member | Signature | Summary | | --- | --- | --- | -| `AddFile` | `static void AddFile(Stream image, string name, byte[] data, char directory = $, uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | Adds a file to an existing single-sided DFS image. Caller is responsible for ensuring the name does not already exist (use `RemoveFile` first for replace-by-name semantics). The file is placed in the lowest contiguous gap large enough to hold it. | +| `AddFile` | `static void AddFile(Stream image, string name, byte[] data, char directory = '$', uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | Adds a file to an existing single-sided DFS image. Caller is responsible for ensuring the name does not already exist (use `RemoveFile` first for replace-by-name semantics). The file is placed in the lowest contiguous gap large enough to hold it. | | `RemoveFile` | `static bool RemoveFile(Stream image, string name, bool wipeData = true)` | Removes a named file from the image. Returns true if found and removed. When `wipeData` is true, the data sectors are zeroed. | #### `BbcReader` @@ -2855,7 +2855,7 @@ Builds a fresh BBC Micro Acorn DFS `.ssd` single-sided disk image from scratch ( | `SectorSize` | `const int SectorSize` | | | `SectorsPerTrack` | `const int SectorsPerTrack` | | | `TotalSectors40` | `const int TotalSectors40` | | -| `AddFile` | `void AddFile(string name, byte[] data, char directory = $, uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | | +| `AddFile` | `void AddFile(string name, byte[] data, char directory = '$', uint loadAddr = 6400, uint execAddr = 6400, bool locked = false)` | | | `Build` | `byte[] Build(string diskTitle = "WORMDISK", int bootOption = 0)` | Builds the complete 40-track SSD image (100 000 bytes). | ### Namespace `FileSystem.BcacheFs` @@ -3232,7 +3232,7 @@ From-scratch writer for the Commodore nibble container the `CbmNibbleReader` con | `AddFile` | `void AddFile(string name, byte[] data)` | Adds a file to the flat directory. Commodore names are PETSCII and at most 16 characters; longer names are truncated. The default file type is PRG. | | `Build` | `byte[] Build()` | Builds the G64 GCR nibble image holding all added files. | | `DecodeToD64` | `static byte[] DecodeToD64(NibbleImage image)` | Reconstructs a standard 174 848-byte D64 image from the GCR tracks of a nibble image previously parsed by `CbmNibbleReader`. Each track is rescanned for sync marks and its header/data blocks GCR-decoded back into the correct sector slots. | -| `SetDisk` | `void SetDisk(string name, char id1 = 0, char id2 = 0)` | Sets the on-disk volume name (PETSCII, ≤16 chars) and the 2-byte disk id. | +| `SetDisk` | `void SetDisk(string name, char id1 = '0', char id2 = '0')` | Sets the on-disk volume name (PETSCII, ≤16 chars) and the 2-byte disk id. | | `WriteTo` | `void WriteTo(Stream output)` | Writes the G64 image to `output`. | #### `G64FormatDescriptor` @@ -6152,7 +6152,7 @@ Implements `IDisposable`. #### `JfsWriter` -Writes a minimal IBM Journaled File System (JFS1) aggregate image with a single allocation group, one fileset, and an inline dtree root directory. Byte layout matches the on-disk structures in `linux/fs/jfs` and the `jfsutils` reference (mkfs.jfs / fsck.jfs); validated by exit-zero from `fsck.jfs -n -f -v`. All integer fields are little-endian. `pxd_t` is packed as `len_addr = (len & 0xFFFFFF) \| ((addr >> 32) << 24)`, `addr2 = addr & 0xFFFFFFFF`. Dtree slot names are UCS-2 (UTF-16 LE). Round-trips through `JfsReader`. Aggregate inode table (block 11..14, IXSIZE=16 KB) holds the AGGR_RESERVED_I (0), AGGREGATE_I (1, → AIM), BMAP_I (2, → block-allocation map), LOG_I (3), BADBLOCK_I (4) and FILESYSTEM_I (16, → fileset AIM) metadata inodes. The fileset inode table at blocks 29..32 holds FILESET_RSVD_I (0), FILESET_EXT_I (1), ROOT_I (2, dtroot inline), ACL_I (3) and user file inodes (4+). +Writes a minimal IBM Journaled File System (JFS1) aggregate image with a single allocation group, one fileset, and an inline dtree root directory. Byte layout matches the on-disk structures in `linux/fs/jfs` and the `jfsutils` reference (mkfs.jfs / fsck.jfs); validated by exit-zero from `fsck.jfs -n -f -v`. All integer fields are little-endian. `pxd_t` is packed as `len_addr = (len & 0xFFFFFF) | ((addr >> 32) << 24)`, `addr2 = addr & 0xFFFFFFFF`. Dtree slot names are UCS-2 (UTF-16 LE). Round-trips through `JfsReader`. Aggregate inode table (block 11..14, IXSIZE=16 KB) holds the AGGR_RESERVED_I (0), AGGREGATE_I (1, → AIM), BMAP_I (2, → block-allocation map), LOG_I (3), BADBLOCK_I (4) and FILESYSTEM_I (16, → fileset AIM) metadata inodes. The fileset inode table at blocks 29..32 holds FILESET_RSVD_I (0), FILESET_EXT_I (1), ROOT_I (2, dtroot inline), ACL_I (3) and user file inodes (4+). | Member | Signature | Summary | | --- | --- | --- | @@ -7535,7 +7535,7 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati #### `Ocfs2InPlaceModifier` -True in-place R/W modifier for OCFS2 (Oracle Cluster Filesystem 2) images produced by `Ocfs2Writer`. Performs O(touched bytes) random-access I/O against the image: only the global bitmap data block, the root directory dinode (inline dirents), the affected file dinode block, and the file's data blocks are read or written. No whole-image read or rewrite. Layout (matches `Ocfs2Writer`'s single-node geometry): 4 KB blocks = 4 KB clusters; one dinode per block.Superblock dinode at block 2; global bitmap dinode at block 3; bitmap data at block 4 (1 bit per cluster, LSB-first, bit=1 means used).Root directory dinode at block 5 (INODE01) with inline dirents in id2 after the 8-byte ocfs2_inline_data header (id2 + 8), each entry `inode(8) \| rec_len(2) \| name_len(1) \| file_type(1) \| name[]`.User files start at block 8: each gets one dinode block, plus contiguous data clusters whose run is held in a single extent record.Scope (MVP, single-node only): root-directory mutations only. Sub-directory mutation, DLM/heartbeat lockdown, multi-node cluster semantics, and root-directory B-tree splits (extent-backed root) are out of scope and throw `NotSupportedException` if encountered. +True in-place R/W modifier for OCFS2 (Oracle Cluster Filesystem 2) images produced by `Ocfs2Writer`. Performs O(touched bytes) random-access I/O against the image: only the global bitmap data block, the root directory dinode (inline dirents), the affected file dinode block, and the file's data blocks are read or written. No whole-image read or rewrite. Layout (matches `Ocfs2Writer`'s single-node geometry): 4 KB blocks = 4 KB clusters; one dinode per block.Superblock dinode at block 2; global bitmap dinode at block 3; bitmap data at block 4 (1 bit per cluster, LSB-first, bit=1 means used).Root directory dinode at block 5 (INODE01) with inline dirents in id2 after the 8-byte ocfs2_inline_data header (id2 + 8), each entry `inode(8) | rec_len(2) | name_len(1) | file_type(1) | name[]`.User files start at block 8: each gets one dinode block, plus contiguous data clusters whose run is held in a single extent record.Scope (MVP, single-node only): root-directory mutations only. Sub-directory mutation, DLM/heartbeat lockdown, multi-node cluster semantics, and root-directory B-tree splits (extent-backed root) are out of scope and throw `NotSupportedException` if encountered. | Member | Signature | Summary | | --- | --- | --- | @@ -10429,7 +10429,7 @@ Builds a fresh ZX Spectrum `.scl` TR-DOS archive from scratch (WORM). | --- | --- | --- | | `ZxSclWriter` | `ZxSclWriter()` | | | `MaxEntries` | `const int MaxEntries` | TR-DOS hard cap: headers are stored in a single 256-entry directory-like table. | -| `AddFile` | `void AddFile(string name, byte[] data, char fileType = C, ushort param1 = 32768, ushort param2 = 0)` | | +| `AddFile` | `void AddFile(string name, byte[] data, char fileType = 'C', ushort param1 = 32768, ushort param2 = 0)` | | | `Build` | `byte[] Build()` | |