+ create validated WACZ web archives

Compression.Tests/Wacz/WaczWriterTests.cs

@@ -0,0 +1,95 @@
+using System.Text;
+using Compression.Registry;
+using FileFormat.Wacz;
+using FileFormat.Zip;
+
+namespace Compression.Tests.Wacz;
+
+[TestFixture]
+public sealed class WaczWriterTests {
+  [Test, Category("HappyPath"), Category("RoundTrip")]
+  public void Create_WritesValidContainerThatReaderLists() {
+    ArchiveInputInfo[] inputs = [
+      ArchiveInputInfo.InMemory("datapackage.json", "{\"profile\":\"data-package\",\"resources\":[],\"wacz_version\":\"1.1.1\"}"u8.ToArray()),
+      ArchiveInputInfo.InMemory("archive/data.warc.gz", new byte[] { 0x1F, 0x8B, 0x08, 0x00 }),
+      ArchiveInputInfo.InMemory("pages/pages.jsonl", "{\"format\":\"json-pages-1.0\"}\n{\"url\":\"https://example.test\",\"ts\":\"2026-01-01T00:00:00Z\"}\n"u8.ToArray()),
+      ArchiveInputInfo.InMemory("indexes/index.cdxj", "com,example)/ 20260101000000 {\"url\":\"https://example.test\"}\n"u8.ToArray()),
+    ];
+
+    using var output = new MemoryStream();
+    var descriptor = new WaczFormatDescriptor();
+    descriptor.Create(output, inputs, new FormatCreateOptions());
+
+    output.Position = 0;
+    var entries = descriptor.List(output, null).Select(entry => entry.Name).ToArray();
+    Assert.Multiple(() => {
+      Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True);
+      Assert.That(entries, Does.Contain("datapackage.json"));
+      Assert.That(entries, Does.Contain("archive/data.warc.gz"));
+      Assert.That(entries, Does.Contain("pages/pages.jsonl"));
+      Assert.That(entries, Does.Contain("indexes/index.cdxj"));
+    });
+
+    output.Position = 0;
+    using var zip = new ZipReader(output, leaveOpen: true);
+    var warc = zip.Entries.Single(entry => entry.FileName == "archive/data.warc.gz");
+    Assert.That(warc.CompressionMethod, Is.EqualTo(ZipCompressionMethod.Store),
+      "already-gzipped WARC members should not be deflated a second time");
+  }
+
+  [Test, Category("HappyPath"), Category("RoundTrip")]
+  public void Create_GenericFiles_PreservesPayloadAndSynthesizesWebArchiveResources() {
+    var payload = "generic archive payload\n"u8.ToArray();
+    ArchiveInputInfo[] inputs = [ArchiveInputInfo.InMemory("docs/readme.txt", payload)];
+    using var output = new MemoryStream();
+    var descriptor = new WaczFormatDescriptor();
+
+    descriptor.Create(output, inputs, new FormatCreateOptions());
+
+    output.Position = 0;
+    using (var zip = new ZipReader(output, leaveOpen: true)) {
+      var names = zip.Entries.Select(entry => entry.FileName).ToArray();
+      Assert.Multiple(() => {
+        Assert.That(names, Does.Contain("docs/readme.txt"));
+        Assert.That(names, Does.Contain("archive/data.warc"));
+        Assert.That(names, Does.Contain("indexes/index.cdxj"));
+        Assert.That(names, Does.Contain("pages/pages.jsonl"));
+        Assert.That(names, Does.Contain("datapackage.json"));
+      });
+
+      var warcEntry = zip.Entries.Single(entry => entry.FileName == "archive/data.warc");
+      Assert.That(warcEntry.CompressionMethod, Is.EqualTo(ZipCompressionMethod.Store));
+
+      var manifestEntry = zip.Entries.Single(entry => entry.FileName == "datapackage.json");
+      var manifest = Encoding.UTF8.GetString(zip.ExtractEntry(manifestEntry));
+      Assert.Multiple(() => {
+        Assert.That(manifest, Does.Contain("\"profile\": \"data-package\""));
+        Assert.That(manifest, Does.Contain("\"wacz_version\": \"1.1.1\""));
+        Assert.That(manifest, Does.Contain("\"path\": \"docs/readme.txt\""));
+        Assert.That(manifest, Does.Contain("sha256:"));
+      });
+    }
+
+    output.Position = 0;
+    Assert.That(descriptor.ExtractEntryToMemory(output, "docs/readme.txt", null), Is.EqualTo(payload).AsCollection);
+  }
+
+  [Test, Category("HappyPath")]
+  public void Create_SuppliedWarcWithoutManifest_GeneratesManifest() {
+    ArchiveInputInfo[] inputs = [
+      ArchiveInputInfo.InMemory("archive/data.warc", "WARC/1.0\r\nWARC-Type: resource\r\nWARC-Record-ID: <urn:test>\r\nWARC-Date: 2026-01-01T00:00:00Z\r\nContent-Length: 0\r\n\r\n\r\n\r\n"u8.ToArray()),
+    ];
+    using var output = new MemoryStream();
+
+    new WaczFormatDescriptor().Create(output, inputs, new FormatCreateOptions());
+
+    output.Position = 0;
+    using var zip = new ZipReader(output, leaveOpen: true);
+    var manifest = zip.Entries.Single(entry => entry.FileName == "datapackage.json");
+    var text = Encoding.UTF8.GetString(zip.ExtractEntry(manifest));
+    Assert.Multiple(() => {
+      Assert.That(text, Does.Contain("\"profile\": \"data-package\""));
+      Assert.That(text, Does.Contain("\"path\": \"archive/data.warc\""));
+    });
+  }
+}

FileFormats/FileFormat.Wacz/FileFormat.Wacz.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Compression.Registry\Compression.Registry.csproj" />
+    <ProjectReference Include="..\FileFormat.Warc\FileFormat.Warc.csproj" />
     <ProjectReference Include="..\FileFormat.Zip\FileFormat.Zip.csproj" />
   </ItemGroup>
 

FileFormats/FileFormat.Wacz/WaczCreator.cs

@@ -0,0 +1,211 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using Compression.Registry;
+using FileFormat.Warc;
+using FileFormat.Zip;
+
+namespace FileFormat.Wacz;
+
+/// <summary>Creates WACZ 1.x ZIP containers from caller-supplied web-archive resources.</summary>
+internal static class WaczCreator {
+  private const string SyntheticDate = "1970-01-01T00:00:00Z";
+  private const string SyntheticTimestamp = "19700101000000";
+  private const string SyntheticWarcName = "archive/data.warc";
+  private const string SyntheticIndexName = "indexes/index.cdxj";
+  private const string SyntheticPagesName = "pages/pages.jsonl";
+
+  public static void Create(Stream output, IReadOnlyList<ArchiveInputInfo> inputs) {
+    ArgumentNullException.ThrowIfNull(output);
+    ArgumentNullException.ThrowIfNull(inputs);
+
+    var files = new List<(string Name, byte[] Data)>();
+    var names = new HashSet<string>(StringComparer.Ordinal);
+    foreach (var input in inputs) {
+      if (input.IsDirectory)
+        continue;
+      var name = NormalizeName(input.ArchiveName);
+      if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase))
+        continue;
+      if (!names.Add(name))
+        throw new InvalidDataException($"WACZ input contains duplicate archive path '{name}'.");
+      files.Add((name, input.ReadContent()));
+    }
+
+    var hasWarc = files.Any(file => IsWarcPath(file.Name));
+    if (!hasWarc) {
+      // Archive conversion supplies an arbitrary file tree, not WARC-specific inputs.
+      // Preserve those files verbatim as legal root/custom WACZ resources and also
+      // wrap them into a deterministic WARC so the resulting package remains a web
+      // archive rather than merely a ZIP with a .wacz extension.
+      var payloads = files
+        .Where(file => !string.Equals(file.Name, "datapackage.json", StringComparison.Ordinal))
+        .ToArray();
+
+      RemoveFile(files, names, "datapackage.json");
+      AddGeneratedFile(files, names, SyntheticWarcName, BuildSyntheticWarc(payloads, out var captures));
+      AddGeneratedFile(files, names, SyntheticIndexName, BuildSyntheticIndex(captures));
+      AddGeneratedFile(files, names, SyntheticPagesName, BuildSyntheticPages(captures));
+      AddGeneratedFile(files, names, "datapackage.json", BuildManifest(files));
+    } else if (!names.Contains("datapackage.json")) {
+      // A caller that already supplied WARC/index/page resources can omit the manifest;
+      // generate its mandatory fixity inventory from the exact bytes we will store.
+      AddGeneratedFile(files, names, "datapackage.json", BuildManifest(files));
+    }
+
+    files.Sort((a, b) => StringComparer.Ordinal.Compare(a.Name, b.Name));
+
+    using var zip = new ZipWriter(output, leaveOpen: true);
+    foreach (var (name, data) in files) {
+      // WACZ 1.1.1 says WARC files SHOULD be ZIP-stored for range access and
+      // already-compressed files MUST NOT be compressed a second time.
+      var method = name.StartsWith("archive/", StringComparison.Ordinal)
+                   || name.EndsWith(".gz", StringComparison.OrdinalIgnoreCase)
+        ? ZipCompressionMethod.Store
+        : ZipCompressionMethod.Deflate;
+      zip.AddEntry(name, data, method);
+    }
+  }
+
+  private static byte[] BuildSyntheticWarc(
+      IReadOnlyList<(string Name, byte[] Data)> payloads,
+      out List<SyntheticCapture> captures) {
+    captures = [];
+    using var combined = new MemoryStream();
+
+    if (payloads.Count == 0) {
+      payloads = [("empty", Array.Empty<byte>())];
+    }
+
+    foreach (var (name, data) in payloads) {
+      var targetUrl = ToSyntheticUrl(name);
+      var digest = Convert.ToHexStringLower(SHA256.HashData(data));
+      var entry = new WarcEntry {
+        Type = "resource",
+        TargetUri = targetUrl,
+        RecordId = $"<urn:sha256:{digest}>",
+        Date = SyntheticDate,
+        ContentType = "application/octet-stream",
+        ContentLength = data.Length,
+      };
+
+      using var recordStream = new MemoryStream();
+      var writer = new WarcWriter();
+      writer.AddRecord(entry, data);
+      writer.WriteTo(recordStream);
+      var record = recordStream.ToArray();
+      var offset = combined.Position;
+      combined.Write(record);
+      captures.Add(new SyntheticCapture(name, targetUrl, digest, offset, record.Length, data.Length));
+    }
+
+    return combined.ToArray();
+  }
+
+  private static byte[] BuildSyntheticIndex(IEnumerable<SyntheticCapture> captures) {
+    var lines = new StringBuilder();
+    foreach (var capture in captures.OrderBy(capture => capture.Url, StringComparer.Ordinal)) {
+      var fields = JsonSerializer.Serialize(new {
+        offset = capture.Offset.ToString(System.Globalization.CultureInfo.InvariantCulture),
+        length = capture.RecordLength.ToString(System.Globalization.CultureInfo.InvariantCulture),
+        mime = "application/octet-stream",
+        status = "-",
+        filename = "data.warc",
+        url = capture.Url,
+        digest = "sha256:" + capture.Digest,
+      });
+      lines.Append(ToSurtKey(capture.Url)).Append(' ')
+        .Append(SyntheticTimestamp).Append(' ')
+        .Append(fields).Append('\n');
+    }
+    return Encoding.UTF8.GetBytes(lines.ToString());
+  }
+
+  private static byte[] BuildSyntheticPages(IEnumerable<SyntheticCapture> captures) {
+    var lines = new StringBuilder();
+    lines.AppendLine(JsonSerializer.Serialize(new { format = "json-pages-1.0", id = "pages", title = "Converted files" }));
+    foreach (var capture in captures.OrderBy(capture => capture.Url, StringComparer.Ordinal))
+      lines.AppendLine(JsonSerializer.Serialize(new {
+        id = capture.Digest[..Math.Min(12, capture.Digest.Length)],
+        url = capture.Url,
+        ts = SyntheticDate,
+        title = capture.Name,
+        size = capture.PayloadLength,
+      }));
+    return Encoding.UTF8.GetBytes(lines.ToString());
+  }
+
+  private static byte[] BuildManifest(IEnumerable<(string Name, byte[] Data)> files) {
+    var resources = files
+      .Where(file => !string.Equals(file.Name, "datapackage.json", StringComparison.Ordinal))
+      .OrderBy(file => file.Name, StringComparer.Ordinal)
+      .Select(file => new {
+        name = Path.GetFileName(file.Name),
+        path = file.Name,
+        hash = "sha256:" + Convert.ToHexStringLower(SHA256.HashData(file.Data)),
+        bytes = file.Data.Length,
+      })
+      .ToArray();
+
+    var json = JsonSerializer.Serialize(new {
+      profile = "data-package",
+      wacz_version = "1.1.1",
+      title = "CompressionWorkbench converted archive",
+      created = SyntheticDate,
+      software = "CompressionWorkbench",
+      resources,
+    }, new JsonSerializerOptions { WriteIndented = true });
+    return Encoding.UTF8.GetBytes(json + "\n");
+  }
+
+  private static bool IsWarcPath(string name)
+    => name.StartsWith("archive/", StringComparison.Ordinal)
+       && (name.EndsWith(".warc", StringComparison.OrdinalIgnoreCase)
+           || name.EndsWith(".warc.gz", StringComparison.OrdinalIgnoreCase));
+
+  private static void AddGeneratedFile(
+      List<(string Name, byte[] Data)> files,
+      HashSet<string> names,
+      string name,
+      byte[] data) {
+    if (!names.Add(name))
+      throw new InvalidDataException($"WACZ input conflicts with generated required path '{name}'.");
+    files.Add((name, data));
+  }
+
+  private static void RemoveFile(List<(string Name, byte[] Data)> files, HashSet<string> names, string name) {
+    files.RemoveAll(file => string.Equals(file.Name, name, StringComparison.Ordinal));
+    names.Remove(name);
+  }
+
+  private static string ToSyntheticUrl(string name)
+    => "https://compression-workbench.invalid/" + string.Join('/',
+      name.Split('/').Select(Uri.EscapeDataString));
+
+  private static string ToSurtKey(string url) {
+    var uri = new Uri(url, UriKind.Absolute);
+    var hostParts = uri.Host.Split('.');
+    Array.Reverse(hostParts);
+    return string.Join(',', hostParts) + ")" + uri.AbsolutePath;
+  }
+
+  private static string NormalizeName(string name) {
+    if (string.IsNullOrWhiteSpace(name))
+      throw new InvalidDataException("WACZ entries require a non-empty archive path.");
+    var normalized = name.Replace('\\', '/').TrimStart('/');
+    if (normalized.Length == 0 || normalized.EndsWith('/'))
+      throw new InvalidDataException($"WACZ file path '{name}' is invalid.");
+    foreach (var component in normalized.Split('/'))
+      if (component is "" or "." or "..")
+        throw new InvalidDataException($"WACZ file path '{name}' contains an unsafe path component.");
+    return normalized;
+  }
+
+  private sealed record SyntheticCapture(
+    string Name,
+    string Url,
+    string Digest,
+    long Offset,
+    int RecordLength,
+    int PayloadLength);
+}

FileFormats/FileFormat.Wacz/WaczFormatDescriptor.cs

@@ -35,7 +35,7 @@ namespace FileFormat.Wacz;
 /// about (title, version, software, page count, archive count).
 /// </para>
 /// </remarks>
-public sealed class WaczFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveLayoutMap {
+public sealed class WaczFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveLayoutMap {
 
   /// <inheritdoc />
   public IEnumerable<DefragBlockInfo> EnumerateLayout(Stream archive) => ZipLayoutMap.Enumerate(archive);
@@ -51,7 +51,7 @@ public sealed class WaczFormatDescriptor : IFormatDescriptor, IArchiveFormatOper
 
   /// <inheritdoc/>
   public FormatCapabilities Capabilities =>
-    FormatCapabilities.CanList | FormatCapabilities.CanExtract |
+    FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |
     FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries |
     FormatCapabilities.SupportsDirectories;
 
@@ -167,6 +167,10 @@ public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? pas
     return memoryStream.ToArray();
   }
 
+  /// <inheritdoc/>
+  public void Create(Stream output, IReadOnlyList<ArchiveInputInfo> inputs, FormatCreateOptions options)
+    => WaczCreator.Create(output, inputs);
+
   /// <summary>
   /// Throws <see cref="InvalidDataException"/> unless the ZIP root looks like a WACZ
   /// (must contain <c>datapackage.json</c> and an <c>archive/</c> directory).