diff --git a/.github/cli-purge-fix.py b/.github/cli-purge-fix.py new file mode 100644 index 000000000..857a90aff --- /dev/null +++ b/.github/cli-purge-fix.py @@ -0,0 +1,9 @@ +from pathlib import Path +p = Path('Compression.CLI/Program.cs') +t = p.read_text() +old = ''' var before = ops.List(File.OpenRead(archive.FullName), null).Count(e => !e.IsDirectory);''' +new = ''' int before;\n using (var source = File.OpenRead(archive.FullName))\n before = ops.List(source, null).Count(e => !e.IsDirectory);''' +if old not in t: + raise SystemExit('generated purge read handle anchor not found') +p.write_text(t.replace(old, new, 1)) +Path('.github/cli-purge-fix.py').unlink() diff --git a/.github/coverage-matrix.py b/.github/coverage-matrix.py new file mode 100644 index 000000000..0b187dcfc --- /dev/null +++ b/.github/coverage-matrix.py @@ -0,0 +1,94 @@ +from pathlib import Path +import re + + +def scan(root): + rows=[] + for path in sorted(Path(root).glob('**/*FormatDescriptor.cs')): + text=path.read_text(errors='ignore') + cm=re.search(r'public\s+(?:sealed\s+)?class\s+\w+FormatDescriptor\s*:\s*(.*?)\{',text,re.S) + mid=re.search(r'public\s+string\s+Id\s*=>\s*"([^"]+)"',text) + if not cm or not mid: continue + interfaces=set(re.findall(r'\bI[A-Za-z0-9_]+\b',cm.group(1))) + tunable='FormatOptionKind.Boolean' in text or re.search(r'AllowedValues\s*:\s*\[[^\]]*,',text,re.S) + optimize=('ILayoutOptimizable' in interfaces or + ('IFormatOptionsSchema' in interfaces and tunable and + ('IArchiveCreatable' in interfaces or 'IStreamFormatOperations' in interfaces))) + row={ + 'id':mid.group(1), 'ifaces':interfaces, + 'defrag':'IArchiveDefragmentable' in interfaces, + 'shrink':'IArchiveShrinkable' in interfaces, + 'wipe':'IWipeEmpty' in interfaces, + 'purge':'IArchivePurgeable' in interfaces or 'IArchiveModifiable' in interfaces, + 'optimize':optimize, + 'meta':'IFileInternalLayoutMap' in interfaces or 'IFileInternalChunkMover' in interfaces, + } + row['compact']=row['defrag'] or row['shrink'] or row['optimize'] + rows.append(row) + return rows + + +def mark(v): return '✅' if v else '—' + +def fs_section(rows): + lines=[ + '## Filesystem descriptors', '', + 'Generated from the descriptor capability interfaces in this tree. **Compact** is the composite ' + 'defrag → optimize → shrink action and is available when at least one of those primitives is executable. ' + 'A checkmark may represent a native in-place operation or a verified offline rebuild; mounted-driver R/W ' + 'is tracked separately by the filesystem-driver readiness model.', '', + '| Format | Optimize | Wipe / clean | Purge | Defrag | Shrink | Compact |', + '| --- | :---: | :---: | :---: | :---: | :---: | :---: |', + ] + for r in sorted(rows,key=lambda r:r['id'].lower()): + lines.append(f"| {r['id']} | {mark(r['optimize'])} | {mark(r['wipe'])} | {mark(r['purge'])} | {mark(r['defrag'])} | {mark(r['shrink'])} | {mark(r['compact'])} |") + return '\n'.join(lines) + + +def archive_section(rows): + visible=[r for r in rows if any(r[k] for k in ('optimize','wipe','purge','defrag','shrink','compact','meta'))] + lines=[ + '## Archive / stream descriptors with at least one operation', '', + 'Archive **defrag** includes verified repack/relayout (including solid-stream regrouping where the format supports it). ' + '**Optimize** includes layout tuning and finite compression/dictionary/solid-block parameter search; candidates only win ' + 'after round-trip verification.', '', + '| Format | Optimize | Wipe / clean | Purge | Defrag | Shrink | Compact | Meta reorder |', + '| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: |', + ] + for r in sorted(visible,key=lambda r:r['id'].lower()): + lines.append(f"| {r['id']} | {mark(r['optimize'])} | {mark(r['wipe'])} | {mark(r['purge'])} | {mark(r['defrag'])} | {mark(r['shrink'])} | {mark(r['compact'])} | {mark(r['meta'])} |") + return '\n'.join(lines) + +p=Path('docs/OPERATION_COVERAGE.md') +t=p.read_text() +# Canonical verb definitions and defaults. +t=re.sub(r'\| Purge\s*\| `IArchiveModifiable\.Remove`-all / empty `Create`\s*\|[^\n]*\|', + '| Purge | `IArchivePurgeable` | Erase all live user data, leaving a valid empty container/image; system metadata may be recreated as required by the format. |',t) +t=t.replace('`Remove(all)` is the **purge** verb.', '`IArchivePurgeable.Purge` is the **purge** verb; `IArchiveModifiable` inherits it because full modification includes removing all live user files.') +t=t.replace('A filesystem descriptor therefore gains shrink / defrag / purge by simply declaring\nthe interface', 'A filesystem descriptor therefore gains shrink / defrag / purge by declaring\nthe corresponding interface') +t=t.replace('> **`CanModify` is advertised when the format is a mutable container with a working modify\n> path.** It is withheld only from **read-only-by-design** formats (CramFS, SquashFS) and\n> **create-only** formats — even though a rebuild could synthesise a modified copy, those do\n> not present themselves as editable.', +'''**`CanModify` means the public API can edit an existing instance and produce a verified valid result.**\nThe physical strategy may be in-place, copy-on-write, relayout/repack, or verified rebuild. Native OS mount\nimmutability (for example CramFS/SquashFS/EROFS) is not the same thing as offline image-editor capability.\nMounted-driver write readiness is tracked separately and remains fail-closed until its durability model is proven.''') +t=t.replace('- **CramFS**, **SquashFS** — compressed *read-only* filesystems by design; not presented as editable.\n','') +t=t.replace('- A handful of niche/append-shift formats (MSA per-track RLE; Wrapster/PFS0 header-at-start;\n OVA manifest-over-all-members; MFS-1 bespoke catalog) keep the rebuild-backed verb without\n advertising R/W.\n','- Formats whose correct edit necessarily rewrites headers, manifests, tracks, or whole images still advertise R/W when the verified rebuild is their supported existing-instance mutation strategy.\n') +# Broaden optimize definition to match the runtime implementation. +t=re.sub(r'\| Optimize\s*\| `ILayoutOptimizable`\s*\|[^\n]*\|', + '| Optimize | `ILayoutOptimizable` / tunable `IFormatOptionsSchema` | Search executable layout/compression parameters and keep the smallest/best verified result. |',t) + +fs=scan('FileSystems') +arc=scan('FileFormats') +start=t.index('## Filesystem descriptors') +na=t.index('## N/A notes',start) +t=t[:start]+fs_section(fs)+'\n\n'+archive_section(arc)+'\n\n'+t[na:] + +# Replace stale totals with source-derived advertised counts so the document and interfaces cannot drift in this pass. +allrows=fs+arc +counts={k:sum(1 for r in allrows if r[k]) for k in ('defrag','wipe','purge','shrink','optimize','meta')} +t=re.sub(r'\| Defragment\s*\| \d+ \|',f"| Defragment | {counts['defrag']} |",t) +t=re.sub(r'\| Wipe\s*\| \d+ \|',f"| Wipe | {counts['wipe']} |",t) +t=re.sub(r'\| Purge\s*\| \d+ \|',f"| Purge | {counts['purge']} |",t) +t=re.sub(r'\| Shrink\s*\| \d+\s*\|',f"| Shrink | {counts['shrink']} |",t) +t=re.sub(r'\| Optimize \(layout\)\| \d+\s*\|',f"| Optimize | {counts['optimize']} |",t) +t=re.sub(r'\| Metadata-reorder\s*\| \d+\s*\|',f"| Metadata-reorder | {counts['meta']} |",t) +t=t.replace('(Counts are `GetArchiveOps(id) is IXxx` over the registered descriptors — i.e.\nwhat the UI/CLI actually gate on.', '(Counts above are regenerated from explicit descriptor capability interfaces in this tree. Runtime marker/flag consistency is enforced by CI; the UI/CLI gate on the same capability contracts.') +p.write_text(t) +Path('.github/coverage-matrix.py').unlink() diff --git a/.github/merge-readiness.py b/.github/merge-readiness.py new file mode 100644 index 000000000..66d81324f --- /dev/null +++ b/.github/merge-readiness.py @@ -0,0 +1,66 @@ +from pathlib import Path + + +def replace(path, old, new, count=1): + p = Path(path) + text = p.read_text() + if old not in text: + raise SystemExit(f'expected text not found in {path}: {old[:120]!r}') + p.write_text(text.replace(old, new, count)) + +# ReFS: wire the already-implemented offline editor into the public descriptor. +replace('FileSystems/FileSystem.Refs/RefsFormatDescriptor.cs', +''' IFormatDescriptor,\n IArchiveFormatOperations,\n IFilesystemExtentMap,''', +''' IFormatDescriptor,\n IArchiveFormatOperations,\n IArchiveModifiable,\n IFilesystemExtentMap,''') +replace('FileSystems/FileSystem.Refs/RefsFormatDescriptor.cs', +''' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |\n FormatCapabilities.SupportsMultipleEntries;''', +''' FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |\n FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries;''') +replace('FileSystems/FileSystem.Refs/RefsFormatDescriptor.cs', +''' public string Description => "Microsoft ReFS 3.x volume image with native read-only driver projection, namespace/allocation parsing, and offline filesystem-metadata placement support.";''', +''' public string Description => "Microsoft ReFS 3.x volume image with native read-only driver projection, namespace/allocation parsing, offline existing-file replacement/removal, and filesystem-metadata placement support.";''') +anchor = '''\n private static List ListDiagnosticSurface(Stream stream) {''' +insert = '''\n public void Add(Stream archive, IReadOnlyList inputs)\n => RefsOfflineModifier.Add(archive, inputs);\n\n public void Remove(Stream archive, string[] entryNames)\n => RefsOfflineModifier.Remove(archive, entryNames);\n''' +replace('FileSystems/FileSystem.Refs/RefsFormatDescriptor.cs', anchor, insert + anchor) + +# Maintenance marker tests: purge is its own explicit capability. +replace('Compression.Tests/Operations/MarkerInterfaceCoverageTests.cs', +''' ("purge/modify", typeof(IArchiveModifiable)),''', +''' ("purge", typeof(IArchivePurgeable)),\n ("modify", typeof(IArchiveModifiable)),''') + +# Generic purge test must exercise the Purge contract, not approximate it with Remove(all). +p = Path('Compression.Tests/Operations/GenericPurgeRoundTripTests.cs') +t = p.read_text() +t = t.replace('''/// Safety net for the broad rollout of the default \n/// (verified extract → edit → re-create rebuild). For every filesystem descriptor\n/// using the DEFAULT Remove, this builds a small image, purges it (Remove every\n/// entry), and asserts the result is a valid, listable, empty container — the\n/// purge verb. The rebuild only commits a result that re-lists, so a writer\n/// limitation surfaces as a clean throw (original untouched) rather than corruption.''', +'''/// Safety net for the explicit capability. For every\n/// purgeable descriptor that can create a representative probe, this builds an image,\n/// invokes Purge, and asserts the user's live files are gone while the result remains\n/// listable. A descriptor that advertises purge must not silently fail or corrupt.''') +t = t.replace(''' private static IEnumerable ModifiableDefaultIds() =>\n Compression.Tests.Support.CapabilityImplementers.RegisteredIdsExposing(typeof(IArchiveModifiable))\n .Where(id => FormatRegistry.GetArchiveOps(id) is IArchiveCreatable\n && Enum.TryParse(id, out _)\n && !Compression.Tests.Support.CapabilityImplementers.DeclaresOwn(id, "Remove", typeof(Stream), typeof(string[])));\n\n [TestCaseSource(nameof(ModifiableDefaultIds))]''', +''' private static IEnumerable PurgeableIds() =>\n Compression.Tests.Support.CapabilityImplementers.RegisteredIdsExposing(typeof(IArchivePurgeable))\n .Where(id => FormatRegistry.GetArchiveOps(id) is IArchiveCreatable\n && Enum.TryParse(id, out _));\n\n [TestCaseSource(nameof(PurgeableIds))]''') +t = t.replace(''' var modifiable = (IArchiveModifiable)fmtOps;\n try {\n modifiable.Remove(ms, [.. before]);\n } catch (NotSupportedException) {\n Assert.Pass($"{formatId}: purge cleanly NotSupported (no corruption).");\n return;\n } catch (Exception ex) {\n Assert.Ignore($"{formatId}: purge rebuild failed non-destructively ({ex.GetType().Name}).");\n return;\n }''', +''' var purgeable = (IArchivePurgeable)fmtOps;\n Assert.DoesNotThrow(() => purgeable.Purge(ms),\n $"{formatId}: advertises IArchivePurgeable but purge failed for its own representative image.");''') +p.write_text(t) + +# Add a root-level purge verb to the CLI. It is deliberately destructive and requires --yes. +program = Path('Compression.CLI/Program.cs') +t = program.read_text() +insert_after = '''replaceCmd.SetAction((ParseResult ctx) => {\n var archive = ctx.GetValue(replaceArchiveArg)!;\n var name = ctx.GetValue(replaceNameArg)!;\n var file = ctx.GetValue(replaceFileArg)!;\n if (!archive.Exists) { Console.Error.WriteLine($"File not found: {archive.FullName}"); return 1; }\n if (!file.Exists) { Console.Error.WriteLine($"File not found: {file.FullName}"); return 1; }\n\n var opts = new CompressionOptions {\n Method = MethodSpec.Parse(ctx.GetValue(methodOpt)),\n Level = ctx.GetValue(levelOpt),\n Password = ctx.GetValue(passwordOpt),\n };\n\n Console.Write($"Replacing '{name}' in {archive.Name}...");\n var sw = Stopwatch.StartNew();\n ArchiveOperations.Replace(archive.FullName, name, file.FullName, opts);\n sw.Stop();\n Console.WriteLine($" done ({sw.ElapsedMilliseconds}ms)");\n return 0;\n});\n''' +purge_block = '''\n// ── purge ────────────────────────────────────────────────────────────\n\nvar purgeArchiveArg = new Argument("archive") { Description = "Archive or filesystem image to empty" };\nvar purgeYesOpt = new Option("--yes", "-y") { Description = "Confirm destructive purge without prompting" };\nvar purgeCmd = new Command("purge", """\n Remove all live user entries while leaving a valid empty container/image.\n This is different from 'wipe': purge removes content; wipe preserves content\n and sanitizes only unused/dead bytes. The format must advertise IArchivePurgeable.\n\n Examples:\n cwb purge disk.d64 --yes\n cwb purge archive.zip --yes\n """) { purgeArchiveArg, purgeYesOpt };\npurgeCmd.SetAction((ParseResult ctx) => {\n var archive = ctx.GetValue(purgeArchiveArg)!;\n if (!archive.Exists) { Console.Error.WriteLine($"File not found: {archive.FullName}"); return 1; }\n if (!ctx.GetValue(purgeYesOpt)) {\n Console.Error.WriteLine("Purge is destructive. Re-run with --yes to remove all live user entries.");\n return 2;\n }\n\n try {\n FormatRegistration.EnsureInitialized();\n var formatId = FormatDetector.Detect(archive.FullName).ToString();\n var ops = FormatRegistry.GetArchiveOps(formatId);\n if (ops is not IArchivePurgeable purgeable) {\n Console.Error.WriteLine($"Format {formatId} does not advertise purge support.");\n return 1;\n }\n\n var before = ops.List(File.OpenRead(archive.FullName), null).Count(e => !e.IsDirectory);\n Console.Write($"Purging {archive.Name} ({formatId}, {before} live file(s))...");\n var sw = Stopwatch.StartNew();\n using (var stream = File.Open(archive.FullName, FileMode.Open, FileAccess.ReadWrite, FileShare.None))\n purgeable.Purge(stream);\n sw.Stop();\n using var verify = File.OpenRead(archive.FullName);\n var after = ops.List(verify, null).Count(e => !e.IsDirectory);\n Console.WriteLine($" done ({sw.ElapsedMilliseconds}ms; {before} -> {after} live file(s))");\n return 0;\n } catch (Exception ex) {\n Console.Error.WriteLine($"Purge failed: {ex.Message}");\n return 1;\n }\n});\n''' +if insert_after not in t: + raise SystemExit('replace command anchor not found in Program.cs') +t = t.replace(insert_after, insert_after + purge_block, 1) +t = t.replace(''' cwb wipe-empty disk.img Zero all unused space in image''', +''' cwb purge disk.img --yes Remove all live user entries\n cwb wipe-empty disk.img Zero all unused space in image''', 1) +t = t.replace(''' listCmd, extractCmd, createCmd, testCmd, addCmd, removeCmd, replaceCmd, infoCmd,''', +''' listCmd, extractCmd, createCmd, testCmd, addCmd, removeCmd, replaceCmd, purgeCmd, infoCmd,''', 1) +program.write_text(t) + +# Correct stale operation-model prose without touching generated tables. +coverage = Path('docs/OPERATION_COVERAGE.md') +t = coverage.read_text() +t = t.replace('''No archive descriptor currently implements a dedicated `IArchivePurgeable` interface; purge is represented by `IArchiveModifiable.Remove` over all live entries.''', +'''Purge is an explicit `IArchivePurgeable` capability. `IArchiveModifiable` inherits it because removing all live user entries is a required subset of full modification; generic purge is staged and verified before commit.''') +t = t.replace('''CramFS and SquashFS remain read-only/WORM because their on-disk formats are immutable by design.''', +'''CramFS, SquashFS and EROFS remain read-only when mounted by their native operating-system drivers, but CompressionWorkbench exposes verified offline rebuild-backed modification for supported profiles. Mounted-driver R/W is a separate capability and remains fail-closed where the native format is immutable or crash-consistent mutation is not implemented.''') +coverage.write_text(t) + +# Remove the one-shot machinery in the commit it creates. +Path('.github/merge-readiness.py').unlink() +Path('.github/workflows/merge-readiness-once.yml').unlink() diff --git a/.github/modify-consistency.py b/.github/modify-consistency.py new file mode 100644 index 000000000..45487557a --- /dev/null +++ b/.github/modify-consistency.py @@ -0,0 +1,43 @@ +from pathlib import Path +import re + +# Normalize descriptor flags to the established public contract: +# IArchiveModifiable means an existing instance can be edited through the API, +# regardless of whether the physical implementation patches in place or stages a +# verified rebuild. +for path in list(Path('FileSystems').glob('**/*FormatDescriptor.cs')) + list(Path('FileFormats').glob('**/*FormatDescriptor.cs')): + text = path.read_text(errors='ignore') + # Only touch classes that explicitly implement the capability. + head = text[:text.find('{', text.find('class ')) + 1] if 'class ' in text else '' + if 'IArchiveModifiable' not in head: + continue + m = re.search(r'public\s+FormatCapabilities\s+Capabilities\s*=>\s*(.*?);', text, re.S) + if not m: + continue + expr = m.group(1) + if 'FormatCapabilities.CanModify' in expr: + continue + replacement = m.group(0)[:-1].rstrip() + ' | FormatCapabilities.CanModify;' + text = text[:m.start()] + replacement + text[m.end():] + # Remove obsolete comments that equate rebuild-backed mutation with WORM. + text = re.sub( + r'\s*// WORM, not R/W: Add/Remove rebuild the whole image \(read-all -> re-create\),\n' + r'\s*// so the verb works via rebuild but nothing is modified in place\. CanModify\n' + r'\s*// must not be advertised\. See Compression\.Registry/FormatCapabilities\.cs\.\n', + '\n // Existing-instance mutation may be implemented by a verified rebuild; that still satisfies CanModify.\n', + text, + count=1) + path.write_text(text) + +# Permanent runtime consistency test: no hidden modifier and no false CanModify flag. +p = Path('Compression.Tests/Operations/MarkerInterfaceCoverageTests.cs') +t = p.read_text() +anchor = ''' [TestCaseSource(nameof(Markers))]\n public void RegistryDrivenSourceIsSupersetOfReflection(Type marker) {''' +method = ''' [Test]\n public void CanModifyFlagAndRuntimeModifierStayInSync() {\n Compression.Lib.FormatRegistration.EnsureInitialized();\n var problems = new List();\n foreach (var descriptor in FormatRegistry.All) {\n var ops = FormatRegistry.GetArchiveOps(descriptor.Id);\n var flag = descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify);\n var runtime = ops is IArchiveModifiable;\n if (flag != runtime)\n problems.Add($"{descriptor.Id}: CanModify={flag}, IArchiveModifiable={runtime}");\n }\n Assert.That(problems, Is.Empty,\n "Existing-instance mutation capability drift:\\n " + string.Join("\\n ", problems));\n }\n\n''' +if method not in t: + if anchor not in t: + raise SystemExit('marker consistency insertion anchor not found') + t = t.replace(anchor, method + anchor, 1) +p.write_text(t) + +Path('.github/modify-consistency.py').unlink() diff --git a/.github/optimizer-wire.py b/.github/optimizer-wire.py new file mode 100644 index 000000000..b8327922e --- /dev/null +++ b/.github/optimizer-wire.py @@ -0,0 +1,12 @@ +from pathlib import Path + +p = Path('Compression.Lib/ArchiveOperations.cs') +t = p.read_text() +old = ''' // ── Unsupported: fall back to copy ───────────────────────────────\n // Use temp+rename so a crash mid-copy doesn't leave a truncated target.\n AtomicFileWriter.WriteAtomic(outputPath, outFs => {\n using var inFs = File.OpenRead(inputPath);\n inFs.CopyTo(outFs);\n });\n return (originalSize, originalSize, 0);''' +new = ''' // ── Generic multi-entry optimizer ─────────────────────────────────\n // Any creatable/listable container that publishes a finite tunable schema\n // can participate. Every candidate is rebuilt and exact-name/data verified\n // by ArchiveCompressionOptimizer/RebuildVerb before it can win.\n FormatRegistration.EnsureInitialized();\n var archiveOps = Compression.Registry.FormatRegistry.GetArchiveOps(format.ToString());\n if (archiveOps is Compression.Registry.IArchiveCreatable creator && archiveOps is Compression.Registry.IFormatOptionsSchema schema\n && schema.OptionsSchema.Any(option =>\n option.Kind == Compression.Registry.FormatOptionKind.Boolean || option.AllowedValues is { Count: > 1 })) {\n var optimized = ArchiveCompressionOptimizer.Optimize(\n inputPath, outputPath, archiveOps, creator, schema);\n return (optimized.OriginalSize, optimized.OptimizedSize, optimized.EntriesOptimized);\n }\n\n // ── No honest optimization surface: preserve the input byte-for-byte ───\n // Use temp+rename so a crash mid-copy doesn't leave a truncated target.\n AtomicFileWriter.WriteAtomic(outputPath, outFs => {\n using var inFs = File.OpenRead(inputPath);\n inFs.CopyTo(outFs);\n });\n return (originalSize, originalSize, 0);''' +if old not in t: + raise SystemExit('ArchiveOperations optimize fallback anchor not found') +p.write_text(t.replace(old, new, 1)) + +# The helper exists only to transform this branch; never retain it in the PR. +Path('.github/optimizer-wire.py').unlink() diff --git a/.github/readme-capabilities.py b/.github/readme-capabilities.py new file mode 100644 index 000000000..a99900605 --- /dev/null +++ b/.github/readme-capabilities.py @@ -0,0 +1,95 @@ +from pathlib import Path +import re + + +def patch_states(): + p = Path('Hawkynt.FileFormats.FileSystems/README.md') + t = p.read_text() + replacements = { + '| [Apple DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | WORM | Apple disk-image container |': + '| [Apple DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | R/W | Apple disk-image container; verified rebuild-backed modification |', + '| [Expert Witness Format](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | R | EWF/EnCase forensic images |': + '| [Expert Witness Format](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | R/W ⚠️ | EWF/EnCase forensic images; logical `media.raw` mutation/repack profile |', + '| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | R | Firmware volume / FFS-oriented inspection |': + '| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | R/W ⚠️ | Fixed-size firmware-volume create/add/replace/remove profile |', + '| [Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | R | Flattened Device Tree property traversal |': + '| [Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | R/W | Flattened Device Tree hierarchy-preserving rebuild/edit |', + '| [ReFS](https://en.wikipedia.org/wiki/ReFS) | R ⚠️ | Header/boot-sector oriented subset |': + '| [ReFS](https://en.wikipedia.org/wiki/ReFS) | R/W ⚠️ | Native metadata read plus offline existing-file replace/remove and empty-directory removal; new-name insertion remains gated |', + '| [EROFS](https://en.wikipedia.org/wiki/EROFS) | WORM | Enhanced read-only filesystem images |': + '| [EROFS](https://en.wikipedia.org/wiki/EROFS) | R/W ⚠️ | Native-mounted read-only format; supported FLAT_PLAIN/FLAT_INLINE images are mutable offline by verified rebuild |', + } + for old, new in replacements.items(): + if old in t: + t = t.replace(old, new, 1) + p.write_text(t) + + +def descriptors(root): + rows = [] + for path in sorted(Path(root).glob('**/*FormatDescriptor.cs')): + text = path.read_text(errors='ignore') + m = re.search(r'public\s+(?:sealed\s+)?class\s+\w+FormatDescriptor\s*:\s*(.*?)(?:\{|\n\s*public\s)', text, re.S) + if not m: + continue + decl = m.group(1) + mid = re.search(r'public\s+string\s+Id\s*=>\s*"([^"]+)"', text) + if not mid: + continue + id_ = mid.group(1) + interfaces = set(re.findall(r'\bI[A-Za-z0-9_]+\b', decl)) + # The class declaration may span until a later member in unusual formatting; + # source-level interface tokens are still the authoritative advertised markers. + rows.append((id_, interfaces, text)) + return rows + + +def matrix(rows, archive=False): + out = [ + '', + '## Maintenance capability matrix', + '', + 'A checkmark means the descriptor explicitly exposes the corresponding runtime capability. ' + 'Profile-specific limitations still apply and unsafe/unknown layouts must fail closed. ' + 'For immutable-on-mount filesystems, a checkmark may represent verified offline rebuild-backed maintenance.', + '', + '| Format | Optimize | Wipe / clean | Purge | Defrag |', + '| --- | :---: | :---: | :---: | :---: |', + ] + for id_, interfaces, text in sorted(rows, key=lambda r: r[0].lower()): + optimize = 'ILayoutOptimizable' in interfaces + if archive and 'IStreamFormatOperations' in interfaces and 'IFormatOptionsSchema' in interfaces: + optimize = True + wipe = 'IWipeEmpty' in interfaces + purge = 'IArchivePurgeable' in interfaces or 'IArchiveModifiable' in interfaces + defrag = 'IArchiveDefragmentable' in interfaces + mark = lambda v: '✅' if v else '—' + out.append(f'| `{id_}` | {mark(optimize)} | {mark(wipe)} | {mark(purge)} | {mark(defrag)} |') + out += [ + '', + 'The matrix is generated from descriptor interfaces during the merge-readiness audit; ' + 'do not add a checkmark without an executable capability and round-trip/fail-closed coverage.', + '', + ] + return '\n'.join(out) + + +def install_matrix(readme, rows, archive=False): + p = Path(readme) + t = p.read_text() + block = matrix(rows, archive) + pattern = re.compile(r'.*?', re.S) + if pattern.search(t): + t = pattern.sub(block, t, count=1) + else: + anchor = '\n## 🚀 Quick start\n' + if anchor not in t: + raise SystemExit(f'quick-start anchor not found in {readme}') + t = t.replace(anchor, '\n' + block + '\n' + anchor, 1) + p.write_text(t) + + +patch_states() +install_matrix('Hawkynt.FileFormats.FileSystems/README.md', descriptors('FileSystems')) +install_matrix('Hawkynt.FileFormats.Archives/README.md', descriptors('FileFormats'), archive=True) +Path('.github/readme-capabilities.py').unlink() diff --git a/.github/workflows/branch-screenshots.yml b/.github/workflows/branch-screenshots.yml new file mode 100644 index 000000000..63da9fca2 --- /dev/null +++ b/.github/workflows/branch-screenshots.yml @@ -0,0 +1,97 @@ +name: Branch screenshots + +on: + push: + branches-ignore: + - main + - master + paths-ignore: + - README.md + - docs/screenshots/** + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: branch-screenshots-${{ github.ref }} + cancel-in-progress: true + +# Match the repository's other workflows while GitHub finishes the Node 24 +# transition for JavaScript actions. +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + capture: + # The workflow commits refreshed images back to the branch. The actor guard + # is a second loop breaker in addition to the [skip ci] commit marker. + if: github.actor != 'github-actions[bot]' + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: 'preview' + + - name: Build screenshot host + # Documentation capture needs the normal build output, not the Release + # single-file packaging target which rewrites/removes that output tree. + run: dotnet build Compression.UI/Compression.UI.csproj --configuration Release --nologo -p:EnableSingleFileBundle=false + + - name: Capture deterministic UI screenshots + shell: pwsh + run: | + Remove-Item docs/screenshots -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force docs/screenshots | Out-Null + + dotnet run --project Compression.UI/Compression.UI.csproj --configuration Release --no-build -p:EnableSingleFileBundle=false -- --screenshots docs/screenshots + if ($LASTEXITCODE -ne 0) { + $diagnostic = "docs/screenshots/screenshot-error.txt" + if (Test-Path $diagnostic) { + Write-Host "--- screenshot diagnostic ---" + Get-Content $diagnostic + Write-Host "--- end diagnostic ---" + } + throw "Compression.UI screenshot mode exited with $LASTEXITCODE" + } + + $expected = @( + "archive-browser.png", + "analysis.png", + "maintenance.png" + ) + foreach ($name in $expected) { + $path = Join-Path "docs/screenshots" $name + if (!(Test-Path $path)) { throw "Missing screenshot: $path" } + if ((Get-Item $path).Length -lt 1024) { throw "Screenshot is suspiciously small: $path" } + } + + - name: Update README screenshot section + run: python .github/workflows/scripts/update-readme-screenshots.py + + - name: Upload screenshot artifact + uses: actions/upload-artifact@v4 + with: + name: branch-screenshots + path: docs/screenshots/*.png + retention-days: 14 + + - name: Commit refreshed screenshots + shell: bash + run: | + if [ -z "$(git status --porcelain -- README.md docs/screenshots)" ]; then + echo "Screenshots and README are already current." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add README.md docs/screenshots + git commit -m "docs: refresh branch screenshots [skip ci]" + git push origin "HEAD:${GITHUB_REF_NAME}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8fdb8d11..d35ec4700 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,6 @@ on: workflow_call: {} workflow_dispatch: {} -# Pushing three times to a branch used to queue three full matrices and run all of them. A newer -# pull-request run supersedes the older one; a push to main is not cancelled, because main wants a -# recorded result for every commit rather than only for the newest. -concurrency: - group: ci-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - # One run per pull request at a time. Pushing again while checks are running # supersedes them, and without this the old run keeps a runner busy: a handful # of quick force-pushes left a dozen doomed runs queued ahead of the one that @@ -284,4 +277,4 @@ jobs: - uses: actions/upload-artifact@v4 with: name: coverage-report - path: ./coverage/report + path: ./coverage/report \ No newline at end of file diff --git a/.github/workflows/merge-readiness-once.yml b/.github/workflows/merge-readiness-once.yml new file mode 100644 index 000000000..163018eb8 --- /dev/null +++ b/.github/workflows/merge-readiness-once.yml @@ -0,0 +1,49 @@ +name: Merge readiness one-shot + +on: + push: + branches: + - feat/filesystem-rw-gaps + +permissions: + contents: write + +concurrency: + group: merge-readiness-${{ github.ref }} + cancel-in-progress: true + +jobs: + prepare: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: feat/filesystem-rw-gaps + fetch-depth: 0 + - name: Apply merge-readiness fixes + run: | + python .github/merge-readiness.py + python .github/optimizer-wire.py + python .github/cli-purge-fix.py + python .github/modify-consistency.py + python .github/readme-capabilities.py + python .github/coverage-matrix.py + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + dotnet-quality: preview + - name: Clone optional sibling dependency + run: git clone --depth 1 https://github.com/Hawkynt/PNGCrushCS.git ../PNGCrushCS || true + - name: Build tests + run: dotnet build Compression.Tests/Compression.Tests.csproj -c Release + - name: Run maintenance and driver safety tests + run: >- + dotnet test Compression.Tests/Compression.Tests.csproj -c Release --no-build + --filter "FullyQualifiedName~MarkerInterfaceCoverageTests|FullyQualifiedName~GenericPurgeRoundTripTests|FullyQualifiedName~FilesystemRwPromotionRoundTripTests|FullyQualifiedName~FilesystemDriver|FullyQualifiedName~Btrfs|FullyQualifiedName~Zfs" + - name: Commit generated fixes + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "fix: complete mutable maintenance surfaces" + git push origin HEAD:feat/filesystem-rw-gaps diff --git a/.github/workflows/scripts/update-readme-screenshots.py b/.github/workflows/scripts/update-readme-screenshots.py new file mode 100644 index 000000000..e6d228543 --- /dev/null +++ b/.github/workflows/scripts/update-readme-screenshots.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +from pathlib import Path + +START = "" +END = "" +ANCHOR = "\n## Vision\n" + +BLOCK = f"""{START} +## UI snapshots + +These screenshots are generated from the current branch by the real WPF application on every non-main push. They are committed back to the branch so the README shows the UI that branch actually builds, rather than a manually curated image from some older revision. + +| Archive browser | Binary analysis | Maintenance | +| :--: | :--: | :--: | +| [![Archive browser](docs/screenshots/archive-browser.png)](docs/screenshots/archive-browser.png) | [![Binary analysis](docs/screenshots/analysis.png)](docs/screenshots/analysis.png) | [![Maintenance](docs/screenshots/maintenance.png)](docs/screenshots/maintenance.png) | + +{END}""" + + +def update_readme(path: Path) -> bool: + text = path.read_text(encoding="utf-8") + + start_count = text.count(START) + end_count = text.count(END) + if start_count != end_count or start_count > 1: + raise RuntimeError( + f"Malformed screenshot markers: {START}={start_count}, {END}={end_count}" + ) + + if start_count == 1: + start = text.index(START) + end = text.index(END, start) + len(END) + updated = text[:start] + BLOCK + text[end:] + else: + if ANCHOR not in text: + raise RuntimeError("README anchor '## Vision' was not found") + updated = text.replace(ANCHOR, f"\n{BLOCK}\n\n## Vision\n", 1) + + if updated == text: + return False + + path.write_text(updated, encoding="utf-8", newline="\n") + return True + + +def main() -> None: + changed = update_readme(Path("README.md")) + print("README screenshot section updated." if changed else "README screenshot section already current.") + + +if __name__ == "__main__": + main() diff --git a/Compression.CLI/Compression.CLI.csproj b/Compression.CLI/Compression.CLI.csproj index b591550fc..9c1dd41d6 100644 --- a/Compression.CLI/Compression.CLI.csproj +++ b/Compression.CLI/Compression.CLI.csproj @@ -33,6 +33,7 @@ + diff --git a/Compression.Core/Layout/DefragPlannerExecutor.cs b/Compression.Core/Layout/DefragPlannerExecutor.cs index 75908e66c..4c351d3d5 100644 --- a/Compression.Core/Layout/DefragPlannerExecutor.cs +++ b/Compression.Core/Layout/DefragPlannerExecutor.cs @@ -17,7 +17,8 @@ public static class DefragPlannerExecutor { /// calling and /// for each move. /// Emits a per move so the UI can animate - /// read/write head positions in real time. + /// read/write head positions in real time. Cancellation is checked between + /// safe move units; already-completed in-place moves are intentionally not rolled back. /// public static void Execute( Stream archive, @@ -28,6 +29,7 @@ public static void Execute( Action? reinitAfterMove = null, IFilesystemMetadataMover? metadataMover = null) { + options.CancellationToken.ThrowIfCancellationRequested(); var metadataNames = metadataMover?.RelocatableMetadata ?? (IReadOnlySet)new HashSet(); bool IsMetadata(string owner) => metadataNames.Contains(owner); @@ -65,6 +67,10 @@ public static void Execute( using var staging = parks ? new DefragStagingBuffer(options.StagingMemoryBudgetBytes) : null; for (var i = 0; i < moves.Count; i++) { + // This is the native/in-place cancellation boundary. A previous move may + // already be durable; stopping here preserves validity rather than trying + // to reverse arbitrary filesystem pointer updates. + options.CancellationToken.ThrowIfCancellationRequested(); var move = moves[i]; var what = move.Staging switch { DefragStaging.Park => "Holding", @@ -117,6 +123,7 @@ public static void Execute( reinitAfterMove?.Invoke(); } + options.CancellationToken.ThrowIfCancellationRequested(); if (relink != null) { var live = new HashSet(relink.BlocksInUseAfterMoves); if (metadataNames.Count > 0) { @@ -129,8 +136,10 @@ public static void Execute( } } - foreach (var (owner, oldBlocks, newBlocks) in relink.Owners()) + foreach (var (owner, oldBlocks, newBlocks) in relink.Owners()) { + options.CancellationToken.ThrowIfCancellationRequested(); mover.UpdateAllocationScattered(archive, owner, oldBlocks, newBlocks, live); + } } } diff --git a/Compression.Core/README.md b/Compression.Core/README.md index 977e66dfc..238b40c26 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -204,7 +204,7 @@ Use the concrete version you intend to consume; this document does not predict a -Every public and protected member of all 537 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). +Every public and protected member of all 573 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 6db8bbfff..510b2640a 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -5391,7 +5391,7 @@ Shared execution loop for planner-driven defragmentation. Runs an ordered list o | Member | Signature | Summary | | --- | --- | --- | -| `Execute` | `static void Execute(Stream archive, DefragOptions options, IFilesystemBlockMover mover, IReadOnlyList moves, long imageSize, Action reinitAfterMove = null, IFilesystemMetadataMover metadataMover = null)` | Executes the supplied `moves` against `archive`, calling `MoveExtent` and `UpdateAllocationAfterMove` for each move. Emits a `DefragProgressEvent` per move so the UI can animate read/write head positions in real time. | +| `Execute` | `static void Execute(Stream archive, DefragOptions options, IFilesystemBlockMover mover, IReadOnlyList moves, long imageSize, Action reinitAfterMove = null, IFilesystemMetadataMover metadataMover = null)` | Executes the supplied `moves` against `archive`, calling `MoveExtent` and `UpdateAllocationAfterMove` for each move. Emits a `DefragProgressEvent` per move so the UI can animate read/write head positions in real time. Cancellation is checked between safe move units; already-completed in-place moves are intentionally not rolled back. | #### `DefragStaging` @@ -6068,7 +6068,7 @@ Run-Length Encoding (RLE) transform. Encodes runs of identical bytes as (count, ### Namespace `Compression.Registry` -[`AlgorithmFamily`](#algorithmfamily) · [`ArchiveEntryInfo`](#archiveentryinfo) · [`ArchiveInputInfo`](#archiveinputinfo) · [`ArchiveShrinker`](#archiveshrinker) · [`AudioPseudoArchive`](#audiopseudoarchive) · [`AudioPseudoArchive.Entry`](#audiopseudoarchiveentry) · [`BuildingBlockRegistry`](#buildingblockregistry) · [`CompoundTarDescriptor`](#compoundtardescriptor) · [`DefragBlockClass`](#defragblockclass) · [`DefragBlockInfo`](#defragblockinfo) · [`DefragBlockKind`](#defragblockkind) · [`DefragContentGuard`](#defragcontentguard) · [`DefragMode`](#defragmode) · [`DefragOptions`](#defragoptions) · [`DefragProgressEvent`](#defragprogressevent) · [`DefragRebuilder`](#defragrebuilder) · [`EntropyDetector`](#entropydetector) · [`FatDirStamp`](#fatdirstamp) · [`FilesystemSchemaPresets`](#filesystemschemapresets) · [`FormatCapabilities`](#formatcapabilities) · [`FormatCategory`](#formatcategory) · [`FormatCreateOptions`](#formatcreateoptions) · [`FormatHealth`](#formathealth) · [`FormatHelpers`](#formathelpers) · [`FormatMethodInfo`](#formatmethodinfo) · [`FormatOptionDescriptor`](#formatoptiondescriptor) · [`FormatOptionKind`](#formatoptionkind) · [`FormatRegistry`](#formatregistry) · [`IArchiveCreatable`](#iarchivecreatable) · [`IArchiveDefragmentable`](#iarchivedefragmentable) · [`IArchiveFormatOperations`](#iarchiveformatoperations) · [`IArchiveInMemoryExtract`](#iarchiveinmemoryextract) · [`IArchiveLayoutMap`](#iarchivelayoutmap) · [`IArchiveModifiable`](#iarchivemodifiable) · [`IArchiveShrinkable`](#iarchiveshrinkable) · [`IArchiveWriteConstraints`](#iarchivewriteconstraints) · [`IAsyncArchiveOperations`](#iasyncarchiveoperations) · [`IBuildingBlock`](#ibuildingblock) · [`IFileInternalChunkMover`](#ifileinternalchunkmover) · [`IFileInternalLayoutMap`](#ifileinternallayoutmap) · [`IFilesystemBlockMover`](#ifilesystemblockmover) · [`IFilesystemExtentMap`](#ifilesystemextentmap) · [`IFilesystemMetadataMover`](#ifilesystemmetadatamover) · [`IFormatDescriptor`](#iformatdescriptor) · [`IFormatOptionsSchema`](#iformatoptionsschema) · [`IFormatValidator`](#iformatvalidator) · [`ILayoutOptimizable`](#ilayoutoptimizable) · [`IPartitionEditable`](#ipartitioneditable) · [`IStreamFormatOperations`](#istreamformatoperations) · [`IWipeEmpty`](#iwipeempty) · [`InnerFsDetector`](#innerfsdetector) · [`IssueSeverity`](#issueseverity) · [`LayoutAnalysis`](#layoutanalysis) · [`LayoutPatch`](#layoutpatch) · [`LayoutProfile`](#layoutprofile) · [`LayoutRebuildOptions`](#layoutrebuildoptions) · [`LayoutReclaim`](#layoutreclaim) · [`MagicSignature`](#magicsignature) · [`MediaProfile`](#mediaprofile) · [`MediaProfileLookup`](#mediaprofilelookup) · [`MetadataPlacementProfile`](#metadataplacementprofile) · [`MetadataPlacementRule`](#metadataplacementrule) · [`MetadataZone`](#metadatazone) · [`MethodNameParser`](#methodnameparser) · [`ModifyRebuilder`](#modifyrebuilder) · [`PlacementZone`](#placementzone) · [`RebuildVerb`](#rebuildverb) · [`SymlinkResolver`](#symlinkresolver) · [`UnusedSpaceWiper`](#unusedspacewiper) · [`ValidationIssue`](#validationissue) · [`ValidationLevel`](#validationlevel) · [`ValidationResult`](#validationresult) +[`AlgorithmFamily`](#algorithmfamily) · [`ArchiveEntryInfo`](#archiveentryinfo) · [`ArchiveInputInfo`](#archiveinputinfo) · [`ArchiveShrinker`](#archiveshrinker) · [`AudioPseudoArchive`](#audiopseudoarchive) · [`AudioPseudoArchive.Entry`](#audiopseudoarchiveentry) · [`BlockDeviceGeometry`](#blockdevicegeometry) · [`BlockDeviceStream`](#blockdevicestream) · [`BuildingBlockRegistry`](#buildingblockregistry) · [`CompoundTarDescriptor`](#compoundtardescriptor) · [`DefragBlockClass`](#defragblockclass) · [`DefragBlockInfo`](#defragblockinfo) · [`DefragBlockKind`](#defragblockkind) · [`DefragContentGuard`](#defragcontentguard) · [`DefragMode`](#defragmode) · [`DefragOptions`](#defragoptions) · [`DefragProgressEvent`](#defragprogressevent) · [`DefragRebuilder`](#defragrebuilder) · [`EntropyDetector`](#entropydetector) · [`FatDirStamp`](#fatdirstamp) · [`FilesystemDirectoryEntry`](#filesystemdirectoryentry) · [`FilesystemDriverBindingKind`](#filesystemdriverbindingkind) · [`FilesystemDriverCapabilities`](#filesystemdrivercapabilities) · [`FilesystemDriverCoverage`](#filesystemdrivercoverage) · [`FilesystemDriverDerivation`](#filesystemdriverderivation) · [`FilesystemDriverProfile`](#filesystemdriverprofile) · [`FilesystemDriverReadinessLayer`](#filesystemdriverreadinesslayer) · [`FilesystemDriverReadinessReport`](#filesystemdriverreadinessreport) · [`FilesystemDriverTarget`](#filesystemdrivertarget) · [`FilesystemMetadataPatch`](#filesystemmetadatapatch) · [`FilesystemMutationModel`](#filesystemmutationmodel) · [`FilesystemNodeId`](#filesystemnodeid) · [`FilesystemNodeInfo`](#filesystemnodeinfo) · [`FilesystemNodeKind`](#filesystemnodekind) · [`FilesystemOpenOptions`](#filesystemopenoptions) · [`FilesystemSchemaPresets`](#filesystemschemapresets) · [`FilesystemSnapshotDirectoryEntry`](#filesystemsnapshotdirectoryentry) · [`FilesystemSnapshotNode`](#filesystemsnapshotnode) · [`FormatCapabilities`](#formatcapabilities) · [`FormatCategory`](#formatcategory) · [`FormatCreateOptions`](#formatcreateoptions) · [`FormatHealth`](#formathealth) · [`FormatHelpers`](#formathelpers) · [`FormatMethodInfo`](#formatmethodinfo) · [`FormatOptionDescriptor`](#formatoptiondescriptor) · [`FormatOptionKind`](#formatoptionkind) · [`FormatRegistry`](#formatregistry) · [`IArchiveCreatable`](#iarchivecreatable) · [`IArchiveDefragmentable`](#iarchivedefragmentable) · [`IArchiveFormatOperations`](#iarchiveformatoperations) · [`IArchiveInMemoryExtract`](#iarchiveinmemoryextract) · [`IArchiveLayoutMap`](#iarchivelayoutmap) · [`IArchiveModifiable`](#iarchivemodifiable) · [`IArchivePurgeable`](#iarchivepurgeable) · [`IArchiveShrinkable`](#iarchiveshrinkable) · [`IArchiveWriteConstraints`](#iarchivewriteconstraints) · [`IAsyncArchiveOperations`](#iasyncarchiveoperations) · [`IBlockDeviceFilesystemDriverProvider`](#iblockdevicefilesystemdriverprovider) · [`IBlockDeviceProvider`](#iblockdeviceprovider) · [`IBuildingBlock`](#ibuildingblock) · [`IFileInternalChunkMover`](#ifileinternalchunkmover) · [`IFileInternalLayoutMap`](#ifileinternallayoutmap) · [`IFilesystemBlockMover`](#ifilesystemblockmover) · [`IFilesystemDriverAdapter`](#ifilesystemdriveradapter) · [`IFilesystemDriverProvider`](#ifilesystemdriverprovider) · [`IFilesystemDriverReadinessProvider`](#ifilesystemdriverreadinessprovider) · [`IFilesystemExtentMap`](#ifilesystemextentmap) · [`IFilesystemFileHandle`](#ifilesystemfilehandle) · [`IFilesystemMetadataMover`](#ifilesystemmetadatamover) · [`IFilesystemSession`](#ifilesystemsession) · [`IFilesystemTransaction`](#ifilesystemtransaction) · [`IFormatDescriptor`](#iformatdescriptor) · [`IFormatOptionsSchema`](#iformatoptionsschema) · [`IFormatValidator`](#iformatvalidator) · [`ILayoutOptimizable`](#ilayoutoptimizable) · [`IPartitionEditable`](#ipartitioneditable) · [`IRandomAccessBlockDevice`](#irandomaccessblockdevice) · [`IRandomAccessBlockDeviceProvider`](#irandomaccessblockdeviceprovider) · [`IRawTrackDevice`](#irawtrackdevice) · [`IRawTrackDeviceProvider`](#irawtrackdeviceprovider) · [`IStreamFormatOperations`](#istreamformatoperations) · [`IWipeEmpty`](#iwipeempty) · [`InnerFsDetector`](#innerfsdetector) · [`IssueSeverity`](#issueseverity) · [`LayoutAnalysis`](#layoutanalysis) · [`LayoutPatch`](#layoutpatch) · [`LayoutProfile`](#layoutprofile) · [`LayoutRebuildOptions`](#layoutrebuildoptions) · [`LayoutReclaim`](#layoutreclaim) · [`MagicSignature`](#magicsignature) · [`MediaProfile`](#mediaprofile) · [`MediaProfileLookup`](#mediaprofilelookup) · [`MetadataPlacementProfile`](#metadataplacementprofile) · [`MetadataPlacementRule`](#metadataplacementrule) · [`MetadataZone`](#metadatazone) · [`MethodNameParser`](#methodnameparser) · [`ModifyRebuilder`](#modifyrebuilder) · [`PlacementZone`](#placementzone) · [`RawTrackInfo`](#rawtrackinfo) · [`ReadOnlyFilesystemSnapshotSession`](#readonlyfilesystemsnapshotsession) · [`RebuildVerb`](#rebuildverb) · [`SpoolingReadOnlyFileHandle`](#spoolingreadonlyfilehandle) · [`StreamBlockDevice`](#streamblockdevice) · [`SymlinkResolver`](#symlinkresolver) · [`UnusedSpaceWiper`](#unusedspacewiper) · [`ValidationIssue`](#validationissue) · [`ValidationLevel`](#validationlevel) · [`ValidationResult`](#validationresult) #### `AlgorithmFamily` @@ -6160,6 +6160,46 @@ One surfaced pseudo-archive entry with its display `Kind` and codec `Method`. Th | `Lazy` | `static Entry Lazy(string name, string kind, Func factory, long declaredSize, string method = "stored")` | Builds a lazy entry: `factory` produces the payload only when the entry is extracted, and `declaredSize` is the exact byte count the factory will yield (used for listing without invoking the factory). The produced bytes are cached on first materialisation so a second extraction reuses them. | | `Materialize` | `byte[] Materialize()` | Returns the payload, invoking and caching the factory on first access for a lazy entry. | +#### `BlockDeviceGeometry` + +Geometry of a sector/block-addressable device exposed beneath a filesystem. Container formats such as VHD/QCOW2/EWF should eventually implement this layer; FAT/ext/ReFS drivers then consume block devices rather than knowing how their outer container stores bytes. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `BlockDeviceGeometry` | `BlockDeviceGeometry(int LogicalBlockSize, long BlockCount, int PhysicalBlockSize = 0, bool SupportsTrim = false)` | Geometry of a sector/block-addressable device exposed beneath a filesystem. Container formats such as VHD/QCOW2/EWF should eventually implement this layer; FAT/ext/ReFS drivers then consume block devices rather than knowing how their outer container stores bytes. | +| `BlockCount` | `long BlockCount { get; init; }` | | +| `Length` | `long Length { get; }` | | +| `LogicalBlockSize` | `int LogicalBlockSize { get; init; }` | | +| `PhysicalBlockSize` | `int PhysicalBlockSize { get; init; }` | | +| `SupportsTrim` | `bool SupportsTrim { get; init; }` | | + +#### `BlockDeviceStream` + +Seekable byte-stream view over a block device. Legacy filesystem parsers can therefore run on VHD/QCOW2/GCR-backed logical disks before they are rewritten to issue block requests directly. Unaligned writes use read-modify-write of only the touched edge blocks; unrelated blocks are never rewritten. + +Inherits `Stream`. Implements `IAsyncDisposable`, `IDisposable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `BlockDeviceStream` | `BlockDeviceStream(IRandomAccessBlockDevice device, bool leaveOpen = true)` | | +| `CanRead` | `override bool CanRead { get; }` | | +| `CanSeek` | `override bool CanSeek { get; }` | | +| `CanWrite` | `override bool CanWrite { get; }` | | +| `Length` | `override long Length { get; }` | | +| `Position` | `override long Position { get; set; }` | | +| `Dispose` | `protected override void Dispose(bool disposing)` | | +| `Flush` | `override void Flush()` | | +| `ReadByte` | `override int ReadByte()` | | +| `Read` | `override int Read(Span buffer)` | | +| `Read` | `override int Read(byte[] buffer, int offset, int count)` | | +| `Seek` | `override long Seek(long offset, SeekOrigin origin)` | | +| `SetLength` | `override void SetLength(long value)` | | +| `WriteByte` | `override void WriteByte(byte value)` | | +| `Write` | `override void Write(ReadOnlySpan buffer)` | | +| `Write` | `override void Write(byte[] buffer, int offset, int count)` | | + #### `BuildingBlockRegistry` Central registry for compression building blocks (algorithm primitives). Populated at startup via source-generated code, similar to `FormatRegistry`. @@ -6198,25 +6238,25 @@ Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`, #### `DefragBlockClass` -Heuristic classification of a file's "thermal" zone based on its modification time. Drives layout placement: hot at start, normal in the middle, frozen near the end. Used by the live-progress block map for tile coloring. +Heuristic classification used for the maintenance block-map colors. Filesystem defraggers commonly map this to hot/cold placement; archive rebuilds may map it to storage/compression classes so a staged target remains visually informative. | Value | Numeric | Summary | | --- | --- | --- | -| `Hot` | `0` | File modified recently (top quartile) — placed near start. | -| `Normal` | `1` | File modified normally — placed in the middle. | -| `Cold` | `2` | File modified a while ago — placed near end. | -| `Frozen` | `3` | File hasn't been touched in a long time (bottom quartile) — placed at end. | -| `Directory` | `4` | Directory metadata (folder contents, B-tree dir node, etc.) — rendered gold to make placement visible. | +| `Hot` | `0` | Hot / heavy-processing class. | +| `Normal` | `1` | Normal class. | +| `Cold` | `2` | Cold / alternative-processing class. | +| `Frozen` | `3` | Frozen / stored-verbatim class. | +| `Directory` | `4` | Directory or structural metadata class. | #### `DefragBlockInfo` -One contiguous region of an image's address space, as seen by the live-progress block map. +One contiguous region in the address-space currently visualized by the maintenance block map. Implements `IEquatable`. | Member | Signature | Summary | | --- | --- | --- | -| `DefragBlockInfo` | `DefragBlockInfo(long Offset, long Length, DefragBlockKind Kind, string FileName = null, DefragBlockClass? Classification = null)` | One contiguous region of an image's address space, as seen by the live-progress block map. | +| `DefragBlockInfo` | `DefragBlockInfo(long Offset, long Length, DefragBlockKind Kind, string FileName = null, DefragBlockClass? Classification = null)` | One contiguous region in the address-space currently visualized by the maintenance block map. | | `Classification` | `DefragBlockClass? Classification { get; init; }` | | | `FileName` | `string FileName { get; init; }` | | | `Kind` | `DefragBlockKind Kind { get; init; }` | | @@ -6225,15 +6265,15 @@ Implements `IEquatable`. #### `DefragBlockKind` -What kind of bytes a contiguous region holds. Used by the live-progress block map to color-code regions as defrag proceeds. +What kind of bytes a contiguous region holds. Used by the live-progress block map to color-code regions as maintenance proceeds. | Value | Numeric | Summary | | --- | --- | --- | | `Free` | `0` | Free space — not allocated to any file. | | `Used` | `1` | Allocated to a file (see `FileName`). | | `Bad` | `2` | Marked bad / quarantined (FAT-style "BAD" cluster, or post-fsck flag). | -| `MetadataReserved` | `3` | Reserved for filesystem metadata (boot sectors, superblock, MFT, FAT, bitmap, root directory). | -| `InProgress` | `4` | Currently being read or written by the in-progress defrag operation. | +| `MetadataReserved` | `3` | Reserved for filesystem/container metadata. | +| `InProgress` | `4` | Currently being read, moved, compressed, grouped, or written. | #### `DefragContentGuard` @@ -6264,6 +6304,7 @@ Implements `IEquatable`. | --- | --- | --- | | `DefragOptions` | `DefragOptions()` | | | `Alignment` | `long Alignment { get; init; }` | Round each target offset up to this byte alignment (1 for byte-tight, 2048 for ISO 9660 sectors, 512 for FAT12/16, …). Default: 1. | +| `CancellationToken` | `CancellationToken CancellationToken { get; init; }` | Cooperative cancellation for long maintenance operations. Generic staged rebuilds honour it while reading and writing and never commit a cancelled target. Native in-place movers may honour it at their next safe move boundary. | | `HoleAt` | `long HoleAt { get; init; }` | Byte offset where the carved hole should start. -1 (default) = auto-pick (carve at the end, immediately after the last live extent). Ignored except in `CarveHole`. | | `HoleSize` | `long HoleSize { get; init; }` | Size in bytes of the hole to carve. Required for `CarveHole`; ignored otherwise. | | `ImageEnd` | `long ImageEnd { get; init; }` | Byte offset just past the last sector available for live data. -1 = auto-detect from the image's physical size. Required for `ConsolidateAtEnd` — must be set explicitly or auto-detected. | @@ -6272,27 +6313,27 @@ Implements `IEquatable`. | `MetadataPlacement` | `MetadataPlacementProfile MetadataPlacement { get; init; }` | Optional metadata placement profile for file-internal optimizers. When non-null, optimizers that support `IFileInternalChunkMover` use these rules to decide where metadata chunks land relative to the primary data payload. When null, each optimizer uses its format-specific default placement. | | `MetadataZonePlacement` | `MetadataZone MetadataZonePlacement { get; init; }` | Controls where filesystem metadata and directory extents are placed during defragmentation. Default: `Unchanged` (metadata stays where it is). Only affects planner-driven defragmentation of filesystem images; ignored for archive optimization and file-internal layout. | | `Mode` | `DefragMode Mode { get; init; }` | Defragmentation strategy. Default: `ConsolidateAtStart`. | -| `OnProgress` | `Action OnProgress { get; init; }` | Optional progress callback. When non-null, the defragmenter emits at least three events: a "scanning" event with the pre-defrag block map, periodic "writing" events with read/write offsets during the rebuild, and a "complete" event with the post-defrag block map. UI consumers can render a live tile chart from these events. | +| `OnProgress` | `Action OnProgress { get; init; }` | Optional progress callback. When non-null, the defragmenter emits snapshots and incremental read/write-head updates that can drive the maintenance block map. Staged archive rebuilds use the same contract as native block movers. | | `Origin` | `long Origin { get; init; }` | Byte offset of the first sector available for live data (e.g. 16 * 2048 for ISO 9660 to leave the volume descriptor space alone, 0 for raw FAT). Default: 0. | | `Profile` | `LayoutProfile Profile { get; init; }` | Layout profile for planner-driven defragmentation. Controls whether the defragmenter performs full zone-based rearrangement (`Performance`) or per-file consolidation only (`Quick`). Default: `Performance`. | | `StagingMemoryBudgetBytes` | `long StagingMemoryBudgetBytes { get; set; }` | Bytes a defragmentation may hold in memory while rearranging a volume that has nowhere of its own to park a run. | #### `DefragProgressEvent` -Snapshot of an image's block layout at a moment in time. Emitted by `Defragment` implementations through DefragOptions.OnProgress at scan start, periodically during writes, and at completion. +Snapshot emitted by defrag/re-layout/rebuild maintenance operations. Native in-place movers normally report one physical image address-space. Transactional WORM/archive rebuilds may report the source read head and staged target write head in their respective byte-spaces, projected onto the same chart. In that mode the two head offsets are progress visualization and do not assert that identical numerical offsets refer to the same physical bytes. Implements `IEquatable`. | Member | Signature | Summary | | --- | --- | --- | -| `DefragProgressEvent` | `DefragProgressEvent(string Phase, double Fraction, long CurrentReadOffset, long CurrentWriteOffset, long ImageSize, IReadOnlyList BlockMap, string Status = null)` | Snapshot of an image's block layout at a moment in time. Emitted by `Defragment` implementations through DefragOptions.OnProgress at scan start, periodically during writes, and at completion. | -| `BlockMap` | `IReadOnlyList BlockMap { get; init; }` | Block-map snapshot, present at scan start + completion. Null during incremental updates. | -| `CurrentReadOffset` | `long CurrentReadOffset { get; init; }` | Byte offset currently being read; -1 if not reading. | -| `CurrentWriteOffset` | `long CurrentWriteOffset { get; init; }` | Byte offset currently being written; -1 if not writing. | +| `DefragProgressEvent` | `DefragProgressEvent(string Phase, double Fraction, long CurrentReadOffset, long CurrentWriteOffset, long ImageSize, IReadOnlyList BlockMap, string Status = null)` | Snapshot emitted by defrag/re-layout/rebuild maintenance operations. Native in-place movers normally report one physical image address-space. Transactional WORM/archive rebuilds may report the source read head and staged target write head in their respective byte-spaces, projected onto the same chart. In that mode the two head offsets are progress visualization and do not assert that identical numerical offsets refer to the same physical bytes. | +| `BlockMap` | `IReadOnlyList BlockMap { get; init; }` | Optional block-map snapshot. Null incremental events retain the previous map and only move heads/progress, keeping redraw cost low on large archives. | +| `CurrentReadOffset` | `long CurrentReadOffset { get; init; }` | Current source/read offset; -1 when not reading. | +| `CurrentWriteOffset` | `long CurrentWriteOffset { get; init; }` | Current destination/write offset; -1 when not writing. | | `Fraction` | `double Fraction { get; init; }` | 0..1 fraction of work done. -1 = indeterminate. | -| `ImageSize` | `long ImageSize { get; init; }` | Total image size in bytes (helpful for tile binning). | -| `Phase` | `string Phase { get; init; }` | Progress phase identifier ("scanning" / "writing" / "complete" / "error"). | -| `Status` | `string Status { get; init; }` | Optional human-readable status (e.g. "moving extent 23 of 87"). | +| `ImageSize` | `long ImageSize { get; init; }` | Address-space size used for visualization/binning. For staged rebuilds this is a display scale large enough to project the source and target progress. | +| `Phase` | `string Phase { get; init; }` | Progress phase identifier. Common values are `scanning`, `reading`, `writing`, `verifying`, `staged`, `committing`, `complete`, and `error`. | +| `Status` | `string Status { get; init; }` | Optional human-readable phase/status text. | #### `DefragRebuilder` @@ -6322,6 +6363,226 @@ Small helpers for writing FAT directory metadata (creation/modification timestam | `Parse` | `static DateTime Parse(string iso)` | Parses an ISO-8601 date/time string for a create-option; returns `default(DateTime)` (treated as "unset") when blank or unparsable. | | `WriteVolumeLabel` | `static void WriteVolumeLabel(byte[] img, int entryOffset, string label)` | Writes an 11-byte volume-label directory entry (attribute 0x08, no cluster, zero size) at `entryOffset`. The label is upper-cased and space-padded/truncated to 11 bytes, matching the FAT short-name field. | +#### `FilesystemDirectoryEntry` + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemDirectoryEntry` | `FilesystemDirectoryEntry(string Name, FilesystemNodeId NodeId, FilesystemNodeKind Kind)` | | +| `Kind` | `FilesystemNodeKind Kind { get; init; }` | | +| `Name` | `string Name { get; init; }` | | +| `NodeId` | `FilesystemNodeId NodeId { get; init; }` | | + +#### `FilesystemDriverBindingKind` + +Structural binding used to reach the common filesystem-driver contract. This deliberately says nothing about the exact image profile: probing an image can still refuse a damaged/unsupported feature set. + +| Value | Numeric | Summary | +| --- | --- | --- | +| `None` | `0` | | +| `ArchiveProjection` | `1` | | +| `SidecarNative` | `2` | | +| `DescriptorNative` | `3` | | + +#### `FilesystemDriverCapabilities` + +| Value | Numeric | Summary | +| --- | --- | --- | +| `None` | `0` | | +| `EnumerateDirectories` | `1` | | +| `ReadData` | `2` | | +| `RandomAccess` | `4` | | +| `StableNodeIds` | `8` | | +| `WriteData` | `16` | | +| `Truncate` | `32` | | +| `CreateFile` | `64` | | +| `DeleteFile` | `128` | | +| `CreateDirectory` | `256` | | +| `RemoveDirectory` | `512` | | +| `Rename` | `1024` | | +| `HardLinks` | `2048` | | +| `SymbolicLinks` | `4096` | | +| `SetMetadata` | `8192` | | +| `SparseFiles` | `16384` | | +| `Flush` | `32768` | | +| `Transactions` | `65536` | | +| `CaseSensitiveNames` | `131072` | | +| `CasePreservingNames` | `262144` | | + +#### `FilesystemDriverCoverage` + +Machine-readable repository coverage for one FileSystem.* descriptor. It answers whether the implementation has a path to an IFilesystemSession and which lower-level primitives are already available for finishing a native read/write driver. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemDriverCoverage` | `FilesystemDriverCoverage(string FormatId, string DisplayName, FilesystemDriverBindingKind Binding, bool HasArchiveProjection, bool HasArchiveMutation, bool HasExtentMap, bool HasBlockMover, bool HasBlockDeviceProvider, bool HasNativeReadinessProvider)` | Machine-readable repository coverage for one FileSystem.* descriptor. It answers whether the implementation has a path to an IFilesystemSession and which lower-level primitives are already available for finishing a native read/write driver. | +| `Binding` | `FilesystemDriverBindingKind Binding { get; init; }` | | +| `DisplayName` | `string DisplayName { get; init; }` | | +| `FormatId` | `string FormatId { get; init; }` | | +| `HasArchiveMutation` | `bool HasArchiveMutation { get; init; }` | | +| `HasArchiveProjection` | `bool HasArchiveProjection { get; init; }` | | +| `HasBlockDeviceProvider` | `bool HasBlockDeviceProvider { get; init; }` | | +| `HasBlockMover` | `bool HasBlockMover { get; init; }` | | +| `HasDriverPath` | `bool HasDriverPath { get; }` | | +| `HasExtentMap` | `bool HasExtentMap { get; init; }` | | +| `HasNativeReadinessProvider` | `bool HasNativeReadinessProvider { get; init; }` | | +| `IsNative` | `bool IsNative { get; }` | | + +#### `FilesystemDriverDerivation` + +Common entry point for filesystem frontends. Native filesystem providers are always preferred. A descriptor that only exposes the normalized archive listing/open-entry surface still gets a real read-only filesystem session: hierarchy is reconstructed, node ids remain stable for the lifetime of the mount, symlinks are represented, and file handles use positional reads. The fallback is deliberately read-only. It never turns archive-level rebuild/Add/Remove support into mounted write support. This makes every filesystem parser usable by FUSE/Dokany/WinFsp-style frontends immediately, while leaving a precise upgrade path to native allocation and mutation code. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Assess` | `static FilesystemDriverReadinessReport Assess(IFormatDescriptor descriptor, Stream image, FilesystemDriverTarget target, string password = null)` | | +| `Open` | `static IFilesystemSession Open(IFormatDescriptor descriptor, Stream image, FilesystemOpenOptions options, string password = null)` | | +| `Probe` | `static FilesystemDriverProfile Probe(IFormatDescriptor descriptor, Stream image, string password = null)` | | + +#### `FilesystemDriverProfile` + +Per-image probe result. Capabilities are not assumed from the format name: an EROFS flat profile, a compressed EROFS profile, a damaged FAT image, or a ReFS version with an unsupported metadata feature can have different safe operations even though they share one descriptor. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemDriverProfile` | `FilesystemDriverProfile(string FormatId, string ProfileName, FilesystemDriverCapabilities Capabilities, FilesystemMutationModel MutationModel, bool CanMount, bool CanMountWritable, IReadOnlyList Limitations)` | Per-image probe result. Capabilities are not assumed from the format name: an EROFS flat profile, a compressed EROFS profile, a damaged FAT image, or a ReFS version with an unsupported metadata feature can have different safe operations even though they share one descriptor. | +| `CanMountWritable` | `bool CanMountWritable { get; init; }` | | +| `CanMount` | `bool CanMount { get; init; }` | | +| `Capabilities` | `FilesystemDriverCapabilities Capabilities { get; init; }` | | +| `FormatId` | `string FormatId { get; init; }` | | +| `Limitations` | `IReadOnlyList Limitations { get; init; }` | | +| `MutationModel` | `FilesystemMutationModel MutationModel { get; init; }` | | +| `ProfileName` | `string ProfileName { get; init; }` | | + +#### `FilesystemDriverReadinessLayer` + +| Value | Numeric | Summary | +| --- | --- | --- | +| `None` | `0` | | +| `ImageValidation` | `1` | | +| `Namespace` | `2` | | +| `SessionStableNodeIds` | `4` | | +| `NativeStableNodeIds` | `8` | | +| `ReadData` | `16` | | +| `RandomAccessRead` | `32` | | +| `AllocationMap` | `64` | | +| `WriteData` | `128` | | +| `Truncate` | `256` | | +| `NamespaceMutation` | `512` | | +| `MetadataMutation` | `1024` | | +| `Links` | `2048` | | +| `Flush` | `4096` | | +| `DurabilityModel` | `8192` | | +| `Recovery` | `16384` | | +| `Concurrency` | `32768` | | +| `ValidationCorpus` | `65536` | | + +#### `FilesystemDriverReadinessReport` + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemDriverReadinessReport` | `FilesystemDriverReadinessReport(string FormatId, FilesystemDriverTarget Target, FilesystemDriverReadinessLayer AvailableLayers, FilesystemDriverReadinessLayer RequiredLayers, bool Derivable, bool UsesNativeProvider, IReadOnlyList Blockers)` | | +| `AvailableLayers` | `FilesystemDriverReadinessLayer AvailableLayers { get; init; }` | | +| `Blockers` | `IReadOnlyList Blockers { get; init; }` | | +| `Derivable` | `bool Derivable { get; init; }` | | +| `FormatId` | `string FormatId { get; init; }` | | +| `RequiredLayers` | `FilesystemDriverReadinessLayer RequiredLayers { get; init; }` | | +| `Target` | `FilesystemDriverTarget Target { get; init; }` | | +| `UsesNativeProvider` | `bool UsesNativeProvider { get; init; }` | | + +#### `FilesystemDriverTarget` + +| Value | Numeric | Summary | +| --- | --- | --- | +| `ReadOnly` | `0` | | +| `ReadWrite` | `1` | | + +#### `FilesystemMetadataPatch` + +Optional metadata changes; null means leave the field unchanged. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemMetadataPatch` | `FilesystemMetadataPatch(DateTimeOffset? Created = null, DateTimeOffset? Modified = null, DateTimeOffset? Accessed = null, ulong? NativeAttributes = null)` | Optional metadata changes; null means leave the field unchanged. | +| `Accessed` | `DateTimeOffset? Accessed { get; init; }` | | +| `Created` | `DateTimeOffset? Created { get; init; }` | | +| `Modified` | `DateTimeOffset? Modified { get; init; }` | | +| `NativeAttributes` | `ulong? NativeAttributes { get; init; }` | | + +#### `FilesystemMutationModel` + +Describes how namespace/data writes become durable on this exact on-disk profile. This is intentionally separate from `CanModify`: archive-level Add/Remove may legitimately rebuild a whole image, while a writable mounted filesystem driver needs bounded, handle-safe mutations. + +| Value | Numeric | Summary | +| --- | --- | --- | +| `None` | `0` | | +| `Direct` | `1` | | +| `Journaled` | `2` | | +| `CopyOnWrite` | `3` | | +| `LogStructured` | `4` | | +| `WholeImageRebuild` | `5` | | + +#### `FilesystemNodeId` + +Stable, path-independent identity of a filesystem object. A real driver must not use a pathname as identity: rename/unlink can change names while open handles keep referring to the same inode/object. Providers map their native inode, file-reference, object-id or directory-slot identity into these two opaque 64-bit words. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemNodeId` | `FilesystemNodeId(ulong Value, ulong Generation = 0)` | Stable, path-independent identity of a filesystem object. A real driver must not use a pathname as identity: rename/unlink can change names while open handles keep referring to the same inode/object. Providers map their native inode, file-reference, object-id or directory-slot identity into these two opaque 64-bit words. | +| `Generation` | `ulong Generation { get; init; }` | | +| `Value` | `ulong Value { get; init; }` | | + +#### `FilesystemNodeInfo` + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemNodeInfo` | `FilesystemNodeInfo(FilesystemNodeId NodeId, FilesystemNodeKind Kind, long Size, long AllocatedSize, uint LinkCount = 1, ulong NativeAttributes = 0, DateTimeOffset? Created = null, DateTimeOffset? Modified = null, DateTimeOffset? Accessed = null, DateTimeOffset? Changed = null)` | | +| `Accessed` | `DateTimeOffset? Accessed { get; init; }` | | +| `AllocatedSize` | `long AllocatedSize { get; init; }` | | +| `Changed` | `DateTimeOffset? Changed { get; init; }` | | +| `Created` | `DateTimeOffset? Created { get; init; }` | | +| `Kind` | `FilesystemNodeKind Kind { get; init; }` | | +| `LinkCount` | `uint LinkCount { get; init; }` | | +| `Modified` | `DateTimeOffset? Modified { get; init; }` | | +| `NativeAttributes` | `ulong NativeAttributes { get; init; }` | | +| `NodeId` | `FilesystemNodeId NodeId { get; init; }` | | +| `Size` | `long Size { get; init; }` | | + +#### `FilesystemNodeKind` + +| Value | Numeric | Summary | +| --- | --- | --- | +| `Unknown` | `0` | | +| `RegularFile` | `1` | | +| `Directory` | `2` | | +| `SymbolicLink` | `3` | | +| `BlockDevice` | `4` | | +| `CharacterDevice` | `5` | | +| `Fifo` | `6` | | +| `Socket` | `7` | | + +#### `FilesystemOpenOptions` + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemOpenOptions` | `FilesystemOpenOptions(bool ReadOnly = true, bool LeaveOpen = true)` | | +| `LeaveOpen` | `bool LeaveOpen { get; init; }` | | +| `ReadOnly` | `bool ReadOnly { get; init; }` | | + #### `FilesystemSchemaPresets` Reusable `FormatOptionDescriptor` building blocks shared by every filesystem that exposes tunable layout parameters through `IFormatOptionsSchema`. Cluster/block size and volume size are near-universal across cluster-based filesystems, so they live here rather than being re-declared in each descriptor. Filesystem-specific knobs (MFT record size, inode size, FAT type, …) are declared by the individual descriptor. @@ -6335,23 +6596,58 @@ Reusable `FormatOptionDescriptor` building blocks shared by every filesystem tha | `PowerOfTwoSize` | `static FormatOptionDescriptor PowerOfTwoSize(string key, string displayName, int min, int max, string defaultLabel, string description)` | Generic power-of-two size dropdown for any byte-valued knob (inode size, MFT record, …). | | `VolumeLabel` | `static FormatOptionDescriptor VolumeLabel(int maxChars = 11)` | Standard volume-label text field. | +#### `FilesystemSnapshotDirectoryEntry` + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemSnapshotDirectoryEntry` | `FilesystemSnapshotDirectoryEntry(FilesystemNodeId ParentNodeId, string Name, FilesystemNodeId NodeId)` | | +| `Name` | `string Name { get; init; }` | | +| `NodeId` | `FilesystemNodeId NodeId { get; init; }` | | +| `ParentNodeId` | `FilesystemNodeId ParentNodeId { get; init; }` | | + +#### `FilesystemSnapshotNode` + +Native filesystem object projected into the common driver contract. Name and parent are a convenient primary-link description for simple filesystems; use explicit `FilesystemSnapshotDirectoryEntry` values when one node has multiple directory entries (hard links). + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FilesystemSnapshotNode` | `FilesystemSnapshotNode(FilesystemNodeId NodeId, FilesystemNodeId ParentNodeId, string Name, FilesystemNodeKind Kind, long Size, long AllocatedSize, uint LinkCount = 1, ulong NativeAttributes = 0, DateTimeOffset? Created = null, DateTimeOffset? Modified = null, DateTimeOffset? Accessed = null, DateTimeOffset? Changed = null, string SymbolicLinkTarget = null, Func OpenReadHandle = null)` | Native filesystem object projected into the common driver contract. Name and parent are a convenient primary-link description for simple filesystems; use explicit `FilesystemSnapshotDirectoryEntry` values when one node has multiple directory entries (hard links). | +| `Accessed` | `DateTimeOffset? Accessed { get; init; }` | | +| `AllocatedSize` | `long AllocatedSize { get; init; }` | | +| `Changed` | `DateTimeOffset? Changed { get; init; }` | | +| `Created` | `DateTimeOffset? Created { get; init; }` | | +| `Kind` | `FilesystemNodeKind Kind { get; init; }` | | +| `LinkCount` | `uint LinkCount { get; init; }` | | +| `Modified` | `DateTimeOffset? Modified { get; init; }` | | +| `Name` | `string Name { get; init; }` | | +| `NativeAttributes` | `ulong NativeAttributes { get; init; }` | | +| `NodeId` | `FilesystemNodeId NodeId { get; init; }` | | +| `OpenReadHandle` | `Func OpenReadHandle { get; init; }` | | +| `ParentNodeId` | `FilesystemNodeId ParentNodeId { get; init; }` | | +| `Size` | `long Size { get; init; }` | | +| `SymbolicLinkTarget` | `string SymbolicLinkTarget { get; init; }` | | + #### `FormatCapabilities` -Flags describing what operations a format supports. Write capability is a four-level scale: Unsupported — no descriptor exists.Read-Only — `CanList` and/or `CanExtract` only.WORM (Write-Once-Read-Many) — adds `CanCreate`: a fresh archive can be produced from inputs, but existing archives cannot be modified in place.R/W (Modify) — adds `CanModify`: entries can be added, replaced, or removed in an existing archive without full rewrite. Most archive formats stop at WORM; true in-place modification is rare because compressed archive containers don't generally support entry mutation without a full rebuild. Honesty rule — rebuild-backed modification is WORM, not R/W. A format may implement `IArchiveModifiable` purely to make the add / remove / purge verbs work, backing them with the verified extract → re-create rebuild (the default `IArchiveModifiable` members, or `ModifyRebuilder` / `RebuildVerb`). That is a full rewrite of the container, so such a format advertises `CanCreate` only and must not set `CanModify` — the verb still runs, but no in-place R/W is claimed. `CanModify` is reserved for formats with a genuine in-place writer that edits the existing container (e.g. ZIP/TAR central-directory edits, FAT/NTFS/ext block writes, byte-identity append). `Compression.Tests.Operations.WriteCapabilityHonestyTests` enforces this for every claimant. +Flags describing what operations a format supports. Write capability is a four-level scale: Unsupported — no descriptor exists.Read-Only — `CanList` and/or `CanExtract` only.WORM (Write-Once-Read-Many) — adds `CanCreate`: a fresh archive/image can be produced, but the library has no supported edit of an existing instance.R/W (Modify) — adds `CanModify`: an existing instance supports add/replace/remove and remains valid after the edit.R/W describes the public operation, not the physical write strategy. A format may update allocation metadata in place, append a new index, relayout members, or rebuild the complete image. Those are implementation choices. If callers can open an existing instance, apply add/replace/remove through `IArchiveModifiable`, and obtain a valid instance preserving the semantics the implementation claims to support, the format is R/W at this API surface. Conversely, merely having a writer for fresh images is WORM and must not set `CanModify`. This distinction is especially important for read-only-on-mount filesystem formats such as SquashFS, CramFS and EROFS: the native filesystem driver may intentionally forbid mounted writes while an offline image editor can still support complete, deterministic mutation by relayout/rebuild. `CanModify` reports the latter capability. | Value | Numeric | Summary | | --- | --- | --- | | `None` | `0` | | | `CanList` | `1` | | | `CanExtract` | `2` | | -| `CanCreate` | `4` | WORM: can produce a fresh archive from inputs (no in-place modification). | +| `CanCreate` | `4` | WORM: can produce a fresh archive/image, but has no supported existing-instance edit. | | `CanTest` | `8` | | | `SupportsPassword` | `16` | | | `SupportsMultipleEntries` | `32` | | | `SupportsDirectories` | `64` | | | `SupportsOptimize` | `256` | | | `CanCompoundWithTar` | `512` | | -| `CanModify` | `1024` | R/W: can modify an existing archive (add/replace/remove entries) without full rewrite. Implies `CanCreate`. | +| `CanModify` | `1024` | R/W: can add/replace/remove entries in an existing archive/image. The implementation may edit in place or relayout/rebuild. Implies `CanCreate` for normal writable formats. | #### `FormatCategory` @@ -6461,19 +6757,27 @@ How a `FormatOptionDescriptor` renders + parses. #### `FormatRegistry` -Central registry of all format descriptors. Populated at startup via `Register` calls (typically from source-generated code), then finalized with `Initialize`. +Central registry of all format descriptors and their optional driver sidecars. Populated at startup by generated registration code, then finalized with `Initialize`. | Member | Signature | Summary | | --- | --- | --- | -| `All` | `static IReadOnlyList All { get; }` | All registered descriptors. | -| `GetArchiveOps` | `static IArchiveFormatOperations GetArchiveOps(string id)` | Get archive operations for a format ID, or null if not an archive format. | -| `GetAsyncArchiveOps` | `static IAsyncArchiveOperations GetAsyncArchiveOps(string id)` | Get async archive operations for a format ID, or null if the format doesn't support async listing. | -| `GetByCategory` | `static IEnumerable GetByCategory(FormatCategory category)` | Get all descriptors in a given category. | -| `GetByExtension` | `static IFormatDescriptor GetByExtension(string path)` | Look up a descriptor by file path/extension. Checks compound extensions first (longest match). | -| `GetById` | `static IFormatDescriptor GetById(string id)` | Look up a descriptor by its unique ID. | -| `GetStreamOps` | `static IStreamFormatOperations GetStreamOps(string id)` | Get stream operations for a format ID, or null if not a stream format. | -| `Initialize` | `static void Initialize()` | Finalize the registry by building lookup tables. Safe to call multiple times. Call this after all `Register` calls are complete. | -| `Register` | `static void Register(IFormatDescriptor descriptor)` | Register a format descriptor. Called by generated code and for compound tar auto-generation. Must be called before `Initialize`. | +| `All` | `static IReadOnlyList All { get; }` | | +| `FilesystemFormatIds` | `static IReadOnlyList FilesystemFormatIds { get; }` | All descriptor IDs originating from FileSystem.* projects. | +| `AssessFilesystemDriver` | `static FilesystemDriverReadinessReport AssessFilesystemDriver(string id, Stream image, FilesystemDriverTarget target, string password = null)` | | +| `GetArchiveOps` | `static IArchiveFormatOperations GetArchiveOps(string id)` | | +| `GetAsyncArchiveOps` | `static IAsyncArchiveOperations GetAsyncArchiveOps(string id)` | | +| `GetByCategory` | `static IEnumerable GetByCategory(FormatCategory category)` | | +| `GetByExtension` | `static IFormatDescriptor GetByExtension(string path)` | | +| `GetById` | `static IFormatDescriptor GetById(string id)` | | +| `GetFilesystemDriverCoverage` | `static FilesystemDriverCoverage GetFilesystemDriverCoverage(string id)` | | +| `GetFilesystemDriverCoverage` | `static IReadOnlyList GetFilesystemDriverCoverage()` | Structural driver coverage for all FileSystem.* descriptors. This is safe to inspect without an image and is intended for CI/readiness dashboards. Use `AssessFilesystemDriver` for exact per-image semantics. | +| `GetFilesystemDriver` | `static IFilesystemDriverAdapter GetFilesystemDriver(string id)` | Returns a generated native driver sidecar for the format, when one exists. | +| `GetStreamOps` | `static IStreamFormatOperations GetStreamOps(string id)` | | +| `Initialize` | `static void Initialize()` | | +| `OpenFilesystem` | `static IFilesystemSession OpenFilesystem(string id, Stream image, FilesystemOpenOptions options, string password = null)` | | +| `ProbeFilesystem` | `static FilesystemDriverProfile ProbeFilesystem(string id, Stream image, string password = null)` | | +| `RegisterFilesystemDriver` | `static void RegisterFilesystemDriver(IFilesystemDriverAdapter driver)` | Registers one source-generated native filesystem-driver sidecar. Duplicate adapters for the same format ID are a build/runtime contract error rather than whichever registration happened to win. | +| `Register` | `static void Register(IFormatDescriptor descriptor, bool isFilesystem = false)` | Register a format descriptor. Source-generated calls set `isFilesystem` for descriptors declared under a `FileSystem.*` namespace so filesystem coverage is explicit rather than inferred from extensions or display names. | #### `IArchiveCreatable` @@ -6486,12 +6790,12 @@ Opt-in capability: the descriptor can produce a fresh archive from a list of inp #### `IArchiveDefragmentable` -Opt-in capability: the descriptor can rewrite an archive in place so that every file occupies a contiguous cluster run, optionally with a chosen layout strategy (consolidate at start / end, lazy hole-fill, carve a free region). Complements the allocator's automatic fast-defrag (which fires only when a pending allocation can't find a contiguous hole); this is the user-initiated full pass. +Opt-in capability for physical or logical re-layout. Native mutable filesystems may move extents in place; WORM/archive containers can satisfy the same verb by building a verified staged target and committing it after completion. | Member | Signature | Summary | | --- | --- | --- | -| `Defragment` | `void Defragment(Stream archive)` | Rebuilds the archive content in place so every file is contiguous. Outer byte size is preserved. Free space is consolidated at the end. Default implementation: any descriptor that also implements `IArchiveFormatOperations` + `IArchiveCreatable` gets defragmentation for free — a verified in-place extract → re-create rebuild via `RebuildInPlace` (the rebuild-via-WORM pattern inherently lays every file out contiguously) that refuses to commit a lossy result. Formats with a true in-place block mover override this for efficiency and full mode support. | -| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Rewrites the archive content according to `options`. Default implementation forwards to `Defragment` for `ConsolidateAtStart` and throws for every other mode — implementers should override to support all modes their on-disk format permits. | +| `Defragment` | `void Defragment(Stream archive)` | Defragments using the format's default consolidate-at-start strategy. Generic list/extract/create descriptors use the verified staged rebuild. | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Rewrites according to `options`. Descriptors with their own native parameterless mover retain it. Descriptors relying on the interface default are routed through the progress-reporting, cancellable staged rebuild, so archive repacks and WORM re-layouts drive the same block-map UI as physical filesystem extent moves. | #### `IArchiveFormatOperations` @@ -6499,10 +6803,10 @@ The base capability every archive descriptor implements: list entries and extrac | Member | Signature | Summary | | --- | --- | --- | -| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Extracts a single entry to a byte array without writing to disk. The default implementation now routes through `OpenEntry` so the bounded streaming contract is enforced even when callers ask for a buffered result. Descriptors that have a more efficient native byte-array path (e.g. a reader that already materialises the whole entry) can still override. | +| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Extracts a single entry to a byte array. This is the explicitly buffered convenience API; callers working with large entries should use `OpenEntry` instead. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extract entries from the archive to an output directory. | | `List` | `List List(Stream stream, string password)` | List all entries in the archive. | -| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | Opens a single entry as a read-only `Stream` bounded to that entry's logical bytes — physically incapable of reading slack space, adjacent entries, padding/alignment fillers, or header/metadata regions. This is the canonical per-entry isolation primitive used by streaming conversion pipelines. | +| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | Opens a single entry as a read-only `Stream` bounded to that entry's logical bytes — physically incapable of reading slack space, adjacent entries, padding/alignment fillers, or header/metadata regions. This is the canonical per-entry isolation primitive used by streaming conversion and derived-filesystem pipelines. | #### `IArchiveInMemoryExtract` @@ -6514,20 +6818,32 @@ Opt-in capability: the descriptor can extract a single named entry straight to a #### `IArchiveLayoutMap` -Opt-in capability: the descriptor can enumerate the real byte-level layout of an archive — every entry's header, compressed payload, and inter-entry gaps at their actual offsets. Parallel to `IFilesystemExtentMap` but for archive formats (ZIP, 7z, TAR, LZH, ARJ, etc.). Drives the Defragment/Optimize window block-map preview so the user sees the real archive layout before pressing "Optimize". +Opt-in capability: the descriptor can enumerate the real byte-level layout of an archive — every entry's header, compressed payload, and inter-entry gaps at their actual offsets. Parallel to `IFilesystemExtentMap` but for archive formats (ZIP, 7z, TAR, LZH, ARJ, etc.). Fail-closed contract: omitted bytes are interpreted as unused by maintenance consumers. Any live, structural, ambiguous or undecoded region must therefore be emitted as `MetadataReserved` (or `Used`), never silently omitted. If a layout cannot be proven safe, return no extents and the inherited generic wipe is a no-op.That exact preservation map also makes every implementation an `IWipeEmpty` capability. The generic implementation zeros proven dead gaps while format-specific overrides may additionally scrub tombstones, reserved growth records, stale indexes, or other recoverable metadata.Drives the Defragment/Optimize window block-map preview so the user sees the real archive layout before pressing "Optimize". + +Implements `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | -| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the actual byte layout of `archive`. Coverage may be sparse; callers fill the gaps with `Free`. The stream's position may be modified during enumeration but the caller owns the lifetime — implementations must not dispose `archive`. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the actual byte layout of `archive`. Coverage may be sparse only where omitted bytes are proven unused; callers fill those gaps with `Free`. The stream's position may be modified during enumeration but the caller owns the lifetime — implementations must not dispose `archive`. | #### `IArchiveModifiable` -Opt-in capability: the descriptor exposes add / remove (and thereby the purge verb). Implementing this interface makes the verbs work; it does not by itself entitle the format to advertise `CanModify` (R/W). The default `Add` / `Remove` below — and any override that delegates to `ModifyRebuilder` / `RebuildVerb` — are a verified extract → re-create rebuild, i.e. a full rewrite of the container. A format whose modification is only rebuild-backed is WORM: it advertises `CanCreate` and must NOT advertise `CanModify` (see `FormatCapabilities`). Reserve `CanModify` for a genuine in-place writer that edits the existing bytes (R/W filesystems; central-directory / member edits; byte-identity append). +Opt-in capability for editing an existing archive/image through add/replace/remove. The physical strategy is format-specific: implementations may patch blocks in place, append replacement metadata, relayout members, or perform a verified extract → edit → re-create rebuild. All are valid implementations of the same public mutation contract when the resulting instance preserves the semantics the descriptor claims to support. A descriptor advertising `CanModify` must expose this interface and its supported-profile edit path must actually round-trip. Merely being able to create a fresh instance is not enough. A fully modifiable container is also purgeable: removing all live entries is a required subset of the remove contract. + +Implements `IArchivePurgeable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds files to an existing instance, replacing entries with the same logical path/name. Default implementation: descriptors that also implement `IArchiveFormatOperations` and `IArchiveCreatable` get a verified extract → edit → re-create implementation through `EditViaRebuild`. Formats with a cheaper native editor override it. | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes the named entries from an existing instance. Passing every entry name yields an empty container/image where the format permits one. Default implementation: a verified extract → drop-named-files → re-create edit through `EditViaRebuild`. Native implementations may instead unlink/free in place and optionally wipe released storage. | + +#### `IArchivePurgeable` + +Opt-in capability: all user/live entries can be removed from an existing container while leaving a valid, listable empty instance. This is distinct from `IWipeEmpty`, which preserves live entries and overwrites only unused/dead bytes. | Member | Signature | Summary | | --- | --- | --- | -| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Appends or replaces files inside `archive`. On replacement the previous bytes are wiped the same way `Remove` wipes them. Default implementation: any descriptor that also implements `IArchiveFormatOperations` + `IArchiveCreatable` gets add for free — a verified extract → splat-new-files → re-create rebuild via `EditViaRebuild` (the same WORM rebuild that backs the other verbs). Formats with a true in-place writer override for efficiency. | -| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes the named entries from `archive` and wipes all on-disk traces. Default implementation: a verified extract → drop-named-files → re-create rebuild via `EditViaRebuild`. Passing every entry name (or all files) yields an empty container — i.e. the purge verb. Formats with a true in-place writer override for efficiency and forensic wiping. | +| `Purge` | `void Purge(Stream archive)` | Removes every live non-directory entry from `archive`. Default implementation: descriptors that also implement `IArchiveFormatOperations` and `IArchiveModifiable` get a transactional staged purge through `PurgeViaModifier`. Native implementations may override this when they can empty the container more efficiently. | #### `IArchiveShrinkable` @@ -6558,6 +6874,23 @@ Optional interface for archive formats that support lazy, asynchronous entry enu | --- | --- | --- | | `ListEntriesAsync` | `IAsyncEnumerable ListEntriesAsync(Stream stream, string password, CancellationToken ct = null)` | Lazily enumerates archive entries as an async stream. Each entry is yielded as it is discovered, without requiring the full archive to be scanned first. | +#### `IBlockDeviceFilesystemDriverProvider` + +Optional filesystem-core capability for implementations whose native parser already works directly on a block device. This is the long-term driver core: the same filesystem implementation can mount raw disks, virtual disks, forensic images, or decoded track media without container-specific code. + +| Member | Signature | Summary | +| --- | --- | --- | +| `OpenFilesystem` | `IFilesystemSession OpenFilesystem(IRandomAccessBlockDevice device, FilesystemOpenOptions options)` | | +| `ProbeFilesystem` | `FilesystemDriverProfile ProbeFilesystem(IRandomAccessBlockDevice device)` | | + +#### `IBlockDeviceProvider` + +Compatibility alias for the original block-device provider name. New code uses `IRandomAccessBlockDeviceProvider` so containers, decoded track media and raw images expose exactly one logical-block abstraction. + +Implements `IRandomAccessBlockDeviceProvider`. + +_No public or protected members._ + #### `IBuildingBlock` A raw compression/decompression building block (algorithm primitive) that can be benchmarked. Unlike `IFormatDescriptor`, building blocks have no file format container — they operate directly on raw byte data. @@ -6603,13 +6936,57 @@ Opt-in capability for filesystems that support true in-place defragmentation via | `UpdateAllocationAfterMove` | `void UpdateAllocationAfterMove(Stream image, string fileName, long oldOffset, long newOffset, long length, bool releaseOldSpace)` | Repoints a run the way `UpdateAllocationAfterMove` does, but says whether the space it came from should be released. | | `UpdateAllocationScattered` | `void UpdateAllocationScattered(Stream image, string fileName, IReadOnlyList oldBlockOffsets, IReadOnlyList newBlockOffsets, IReadOnlySet blocksLiveElsewhere)` | Rewrites `fileName`'s allocation so that it occupies `newBlockOffsets` in that order, having previously occupied `oldBlockOffsets`. Both lists are one entry per allocation block, in the file's own order. | +#### `IFilesystemDriverAdapter` + +Sidecar binding from an existing format descriptor ID to a native filesystem driver core. This lets large/legacy descriptors acquire driver semantics without mixing mount state, locking and block-device code into their archive surface. The source generator discovers public parameterless implementations and registers them by `FormatId`. + +Implements `IFilesystemDriverProvider`, `IFilesystemDriverReadinessProvider`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FormatId` | `string FormatId { get; }` | | + +#### `IFilesystemDriverProvider` + +Descriptor-side entry point for a mount-grade filesystem implementation. Probe must be non-destructive and fail closed. Open must reject writable mode unless the returned profile has `CanMountWritable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `OpenFilesystem` | `IFilesystemSession OpenFilesystem(Stream image, FilesystemOpenOptions options)` | | +| `ProbeFilesystem` | `FilesystemDriverProfile ProbeFilesystem(Stream image)` | | + +#### `IFilesystemDriverReadinessProvider` + +Optional filesystem-specific readiness description. The generic derivation layer supplies a conservative report when a descriptor does not implement this interface; native implementations can use it to explain exactly which on-disk semantics still block a complete mounted driver. + +| Member | Signature | Summary | +| --- | --- | --- | +| `DescribeFilesystemDriverReadiness` | `FilesystemDriverReadinessReport DescribeFilesystemDriverReadiness(Stream image, FilesystemDriverTarget target)` | | + #### `IFilesystemExtentMap` -Opt-in capability: the descriptor (or a partner type) can enumerate the actual on-disk byte layout of a filesystem image — every used cluster chain per file (one `DefragBlockInfo` per contiguous run), every metadata-reserved region (boot sector, FAT, bitmap, superblock, MFT, root directory, inode table, BAM, group descriptor table, etc.), and optionally every free region. Coverage may be sparse — gaps in the returned set are interpreted by the caller as `Free`. The yielded extents don't need to be sorted; the caller is responsible for sorting + gap filling. Implementations must not throw for malformed or partially-walked images — they should yield whatever they can identify and return.Drives the Defragment-window block-map preview so the user sees the real fragmented layout before pressing "Defragment" rather than the post-defrag approximation. +Opt-in capability: the descriptor (or a partner type) can enumerate the actual on-disk byte layout of a filesystem image — every used cluster chain per file (one `DefragBlockInfo` per contiguous run), every metadata-reserved region (boot sector, FAT, bitmap, superblock, MFT, root directory, inode table, BAM, group descriptor table, etc.), and optionally every free region. Fail-closed contract: gaps in the returned set are interpreted as free space by maintenance consumers. Therefore an implementation that encounters an allocated-but-undecoded, damaged, ambiguous, or otherwise unproven region MUST emit that region as `MetadataReserved` rather than silently omit it. If the image cannot be walked safely at all, yield no extents; the inherited generic `IWipeEmpty` implementation then wipes nothing.Because this contract identifies all bytes that must be preserved, every extent map is also an `IWipeEmpty` implementation: the default wiper zeros only proven gaps (and cluster tips when a trustworthy logical-size lookup exists). Formats that know about deleted directory records or other hidden remnants may override the wipe for deeper cleaning.Drives the Defragment-window block-map preview so the user sees the real fragmented layout before pressing "Defragment" rather than the post-defrag approximation. + +Implements `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | -| `EnumerateExtents` | `IEnumerable EnumerateExtents(Stream image)` | Enumerates the actual on-disk layout of `image`. Coverage may be sparse; callers fill the gaps with `Free`. The stream's position may be modified during enumeration but the caller owns the lifetime — implementations must not dispose `image`. | +| `EnumerateExtents` | `IEnumerable EnumerateExtents(Stream image)` | Enumerates the actual on-disk layout of `image`. Coverage may be sparse only where the omitted bytes are proven free; callers fill those gaps with `Free`. Unknown allocated bytes must be returned as `MetadataReserved`. The stream's position may be modified during enumeration but the caller owns its lifetime — implementations must not dispose `image`. | + +#### `IFilesystemFileHandle` + +Positional file handle. It deliberately has no shared Stream.Position so two concurrent kernel requests cannot race a mutable cursor. Reads/writes operate at explicit logical offsets and therefore map naturally to filesystem extents. + +Implements `IDisposable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Length` | `long Length { get; }` | | +| `NodeId` | `FilesystemNodeId NodeId { get; }` | | +| `Flush` | `void Flush()` | | +| `Read` | `int Read(long offset, Span destination)` | | +| `SetLength` | `void SetLength(long length)` | | +| `Write` | `void Write(long offset, ReadOnlySpan source)` | | #### `IFilesystemMetadataMover` @@ -6621,6 +6998,42 @@ Opt-in capability for filesystems whose own structures — the MFT, an allocatio | `PrepareMetadataMove` | `void PrepareMetadataMove(Stream image, string metadataName, long oldOffset, long newOffset, long length)` | Gives the filesystem a chance to make the destination safe before the raw bytes are copied there. | | `UpdateMetadataAfterMove` | `void UpdateMetadataAfterMove(Stream image, string metadataName, long oldOffset, long newOffset, long length, IReadOnlyList> liveRanges = null)` | Repoints whatever locates `metadataName` after its bytes have been copied from `oldOffset` to `newOffset`, and moves the allocation with it. | +#### `IFilesystemSession` + +Open filesystem namespace. Operations use stable node ids rather than paths, mirroring the semantics required by FUSE/Dokany/WinFsp-style adapters: a caller may keep a file handle open across rename or unlink. + +Implements `IDisposable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Profile` | `FilesystemDriverProfile Profile { get; }` | | +| `RootNodeId` | `FilesystemNodeId RootNodeId { get; }` | | +| `BeginTransaction` | `IFilesystemTransaction BeginTransaction()` | Begins one durability transaction for this session. Until Commit/Rollback, namespace operations and writes through handles opened by the session belong to that transaction. Providers that do not advertise Transactions throw. | +| `CreateDirectory` | `FilesystemNodeId CreateDirectory(FilesystemNodeId parentDirectory, string name)` | | +| `CreateFile` | `FilesystemNodeId CreateFile(FilesystemNodeId parentDirectory, string name)` | | +| `CreateHardLink` | `void CreateHardLink(FilesystemNodeId existingNode, FilesystemNodeId newParent, string newName)` | | +| `CreateSymbolicLink` | `FilesystemNodeId CreateSymbolicLink(FilesystemNodeId parentDirectory, string name, string target)` | | +| `DeleteFile` | `void DeleteFile(FilesystemNodeId parentDirectory, string name)` | | +| `Enumerate` | `IReadOnlyList Enumerate(FilesystemNodeId directory)` | | +| `Flush` | `void Flush()` | Flushes all dirty data and metadata that are not inside an active transaction. | +| `Lookup` | `FilesystemNodeId? Lookup(FilesystemNodeId parentDirectory, string name)` | | +| `OpenFile` | `IFilesystemFileHandle OpenFile(FilesystemNodeId nodeId, FileAccess access)` | | +| `ReadSymbolicLink` | `string ReadSymbolicLink(FilesystemNodeId nodeId)` | | +| `RemoveDirectory` | `void RemoveDirectory(FilesystemNodeId parentDirectory, string name)` | | +| `Rename` | `void Rename(FilesystemNodeId oldParent, string oldName, FilesystemNodeId newParent, string newName, bool replace)` | | +| `SetMetadata` | `void SetMetadata(FilesystemNodeId nodeId, FilesystemMetadataPatch patch)` | | +| `Stat` | `FilesystemNodeInfo Stat(FilesystemNodeId nodeId)` | | + +#### `IFilesystemTransaction` + +Implements `IDisposable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `IsCompleted` | `bool IsCompleted { get; }` | | +| `Commit` | `void Commit()` | | +| `Rollback` | `void Rollback()` | | + #### `IFormatDescriptor` Self-describing metadata for a file format. Each FileFormat.* project provides one implementation of this interface to register itself with the format registry. @@ -6677,6 +7090,49 @@ Capability marker for archive/disk-container formats whose payload is a raw bloc | --- | --- | --- | | `OpenGuestDiskStream` | `Stream OpenGuestDiskStream(Stream image)` | Opens the inner (guest) disk image as a `Stream` suitable for partition-table editing. The returned stream must support reading, writing, and seeking. The caller owns the returned stream and must dispose it; disposing it must not dispose the outer `image` stream. | +#### `IRandomAccessBlockDevice` + +Implements `IDisposable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `CanWrite` | `bool CanWrite { get; }` | | +| `Geometry` | `BlockDeviceGeometry Geometry { get; }` | | +| `Flush` | `void Flush()` | | +| `ReadBlocks` | `int ReadBlocks(long firstBlock, Span destination)` | | +| `Trim` | `void Trim(long firstBlock, long blockCount)` | | +| `WriteBlocks` | `void WriteBlocks(long firstBlock, ReadOnlySpan source)` | | + +#### `IRandomAccessBlockDeviceProvider` + +Optional descriptor capability for exposing the sector/block device that sits below a filesystem namespace. Container descriptors can implement this without pretending the container itself is a filesystem. + +| Member | Signature | Summary | +| --- | --- | --- | +| `OpenBlockDevice` | `IRandomAccessBlockDevice OpenBlockDevice(Stream image, bool writable, bool leaveOpen = true)` | Opens a random-access block device over `image`. Implementations must fail closed when the exact on-disk profile cannot be projected losslessly/safely at block granularity. | + +#### `IRawTrackDevice` + +Implements `IDisposable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `CanWrite` | `bool CanWrite { get; }` | | +| `TrackCount` | `int TrackCount { get; }` | | +| `ClearTrack` | `void ClearTrack(int index)` | | +| `EnumerateTracks` | `IReadOnlyList EnumerateTracks()` | | +| `Flush` | `void Flush()` | | +| `ReadTrack` | `int ReadTrack(int index, Span destination)` | | +| `WriteTrack` | `void WriteTrack(int index, ReadOnlySpan source, uint? encodingParameter = null)` | | + +#### `IRawTrackDeviceProvider` + +Optional descriptor capability for opening the raw-track layer directly. This keeps track-container mutation separate from filesystem namespace CRUD. + +| Member | Signature | Summary | +| --- | --- | --- | +| `OpenRawTrackDevice` | `IRawTrackDevice OpenRawTrackDevice(Stream image, bool writable, bool leaveOpen = true)` | | + #### `IStreamFormatOperations` Operations for single-stream compression formats. @@ -6692,7 +7148,7 @@ Operations for single-stream compression formats. #### `IWipeEmpty` -Opt-in capability: the descriptor can zero-fill all unused bytes in an image or archive — free clusters/sectors, cluster-tip slack, deleted directory entries, padding regions, and dead archive bytes. This is a forensic-cleanliness tool ensuring no deleted file remnants survive. Implementations that don't need format-specific logic can delegate to `Wipe` which works generically with any `IFilesystemExtentMap` or `IArchiveLayoutMap`. +Opt-in capability: the descriptor can zero-fill all unused bytes in an image or archive — free clusters/sectors, cluster-tip slack, deleted directory entries, padding regions, and dead archive bytes. This is a forensic-cleanliness tool ensuring no deleted file remnants survive. The default implementation is deliberately conservative. It is available only when the same descriptor exposes an exact filesystem extent map or archive layout map; unknown/undecoded regions must therefore be emitted as `MetadataReserved` by those maps rather than omitted. An empty map is treated as "cannot prove anything is free" and wipes nothing. | Member | Signature | Summary | | --- | --- | --- | @@ -6886,15 +7342,92 @@ Which zone a metadata chunk should be placed in relative to the primary data pay | `AfterData` | `1` | Place the chunk after the primary data payload. | | `Remove` | `2` | Remove the chunk entirely during optimization. | +#### `RawTrackInfo` + +Raw variable-length track device for flux/GCR/MFM-style containers that are not yet sector-addressable. G64 belongs here; a decoder can later project it as `IRandomAccessBlockDevice` for a Commodore filesystem driver. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `RawTrackInfo` | `RawTrackInfo(int Index, long Length, uint EncodingParameter = 0, bool IsPresent = true)` | Raw variable-length track device for flux/GCR/MFM-style containers that are not yet sector-addressable. G64 belongs here; a decoder can later project it as `IRandomAccessBlockDevice` for a Commodore filesystem driver. | +| `EncodingParameter` | `uint EncodingParameter { get; init; }` | | +| `Index` | `int Index { get; init; }` | | +| `IsPresent` | `bool IsPresent { get; init; }` | | +| `Length` | `long Length { get; init; }` | | + +#### `ReadOnlyFilesystemSnapshotSession` + +Implements `IDisposable`, `IFilesystemSession`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `ReadOnlyFilesystemSnapshotSession` | `ReadOnlyFilesystemSnapshotSession(FilesystemDriverProfile profile, FilesystemNodeId rootNodeId, IEnumerable nodes)` | | +| `ReadOnlyFilesystemSnapshotSession` | `ReadOnlyFilesystemSnapshotSession(FilesystemDriverProfile profile, FilesystemNodeId rootNodeId, IEnumerable nodes, IEnumerable directoryEntries)` | Full constructor with independent object and directory-entry sets. Multiple entries may target the same node ID; that is how hard links are represented. | +| `Profile` | `FilesystemDriverProfile Profile { get; }` | | +| `RootNodeId` | `FilesystemNodeId RootNodeId { get; }` | | +| `BeginTransaction` | `IFilesystemTransaction BeginTransaction()` | | +| `CreateDirectory` | `FilesystemNodeId CreateDirectory(FilesystemNodeId parentDirectory, string name)` | | +| `CreateFile` | `FilesystemNodeId CreateFile(FilesystemNodeId parentDirectory, string name)` | | +| `CreateHardLink` | `void CreateHardLink(FilesystemNodeId existingNode, FilesystemNodeId newParent, string newName)` | | +| `CreateSymbolicLink` | `FilesystemNodeId CreateSymbolicLink(FilesystemNodeId parentDirectory, string name, string target)` | | +| `DeleteFile` | `void DeleteFile(FilesystemNodeId parentDirectory, string name)` | | +| `Dispose` | `void Dispose()` | | +| `Enumerate` | `IReadOnlyList Enumerate(FilesystemNodeId directory)` | | +| `Flush` | `void Flush()` | | +| `Lookup` | `FilesystemNodeId? Lookup(FilesystemNodeId parentDirectory, string name)` | | +| `OpenFile` | `IFilesystemFileHandle OpenFile(FilesystemNodeId nodeId, FileAccess access)` | | +| `ReadSymbolicLink` | `string ReadSymbolicLink(FilesystemNodeId nodeId)` | | +| `RemoveDirectory` | `void RemoveDirectory(FilesystemNodeId parentDirectory, string name)` | | +| `Rename` | `void Rename(FilesystemNodeId oldParent, string oldName, FilesystemNodeId newParent, string newName, bool replace)` | | +| `SetMetadata` | `void SetMetadata(FilesystemNodeId nodeId, FilesystemMetadataPatch patch)` | | +| `Stat` | `FilesystemNodeInfo Stat(FilesystemNodeId nodeId)` | | + #### `RebuildVerb` -Generic, round-trip-verified "extract → re-create" engine shared by the default implementations of the maintenance verbs (shrink, defragment) for any descriptor that can both enumerate/extract (`IArchiveFormatOperations`) and create (`IArchiveCreatable`) its format. Every rebuild is verified: the freshly created image is listed back and its live-file count compared against the source. If the rebuild would drop files, the operation throws `InvalidOperationException` instead of producing a lossy result — so enabling a verb on a format whose create path doesn't faithfully round-trip fails loudly rather than silently corrupting data. This is what makes broad, default-implementation rollout across filesystems safe. +Generic, round-trip-verified extract → re-create engine shared by maintenance verbs. Rebuilds are staged, verified, progress-reporting, and cancellable; the caller's original stream is not touched until the staged target has been built successfully and cancellation is no longer accepted. | Member | Signature | Summary | | --- | --- | --- | -| `EditViaRebuild` | `static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, Action mutate)` | Rebuild-based in-place edit shared by the default `IArchiveModifiable`: extract the archive, apply `mutate` to the extracted file tree (add/overwrite/delete real files on disk), re-create the image, and overwrite the stream. The original bytes are left untouched on any failure. | -| `RebuildInPlace` | `static void RebuildInPlace(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, IReadOnlyDictionary formatSpecific = null)` | In-place rebuild: re-creates `archive` from its own contents (consolidating live data — the defragmentation side effect of the rebuild-via-WORM pattern) and overwrites the stream only when the rebuild is verified to round-trip. On any failure the original bytes are left untouched. | -| `RebuildToStream` | `static int RebuildToStream(Stream input, Stream output, IArchiveFormatOperations ops, IArchiveCreatable creator, IReadOnlyDictionary formatSpecific = null, IReadOnlySet syntheticNames = null)` | Extracts every entry of `input` and re-creates the image into `output` via `creator`. Returns the source live-file count. Throws if the rebuilt image lists fewer live files than the source (lossy round-trip) — the caller's `output` should be discarded in that case. | +| `EditViaRebuild` | `static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, Action mutate)` | Rebuild-based edit used by the generic modifier. Mutation and validation happen off to the side; the original is overwritten only after a valid staged result exists. | +| `PurgeViaModifier` | `static void PurgeViaModifier(Stream archive, IArchiveFormatOperations ops, IArchiveModifiable modifier)` | Transactional purge for a mutable container. The modifier operates on a staged copy and the caller's stream is replaced only after the result lists successfully with every original live entry gone. | +| `RebuildInPlace` | `static void RebuildInPlace(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, IReadOnlyDictionary formatSpecific = null, Action onProgress = null, CancellationToken cancellationToken = null)` | Rebuilds into a scratch file, verifies it, and only then replaces the caller-supplied stream. Cancellation is honoured until commit starts; once commit begins it runs to completion so a cancellation cannot leave the original half-overwritten. | +| `RebuildToStream` | `static int RebuildToStream(Stream input, Stream output, IArchiveFormatOperations ops, IArchiveCreatable creator, IReadOnlyDictionary formatSpecific = null, IReadOnlySet syntheticNames = null, Action onProgress = null, CancellationToken cancellationToken = null)` | Extracts every live entry, re-creates the container in `output`, verifies the exact live-name multiset, and reports block-map/read/write-head progress suitable for the maintenance UI. | + +#### `SpoolingReadOnlyFileHandle` + +Transitional positional handle for native filesystem readers that can stream a file correctly but do not yet expose a seekable block/extent map. Small files stay in memory; large files are spooled to a delete-on-close temporary file. This preserves driver-style positional reads without imposing a whole- file RAM ceiling while the filesystem's direct block mapping is implemented. + +Implements `IDisposable`, `IFilesystemFileHandle`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `DefaultMemoryThreshold` | `const long DefaultMemoryThreshold` | | +| `Length` | `long Length { get; }` | | +| `NodeId` | `FilesystemNodeId NodeId { get; }` | | +| `Create` | `static SpoolingReadOnlyFileHandle Create(FilesystemNodeId nodeId, long expectedLength, Action writeContent, long memoryThreshold = 8388608)` | | +| `Dispose` | `void Dispose()` | | +| `Flush` | `void Flush()` | | +| `Read` | `int Read(long offset, Span destination)` | | +| `SetLength` | `void SetLength(long length)` | | +| `Write` | `void Write(long offset, ReadOnlySpan source)` | | + +#### `StreamBlockDevice` + +Fixed-size random-access block device over an ordinary seekable stream. This is the bridge for raw filesystem images while parsers migrate away from direct Stream.Position access. + +Implements `IDisposable`, `IRandomAccessBlockDevice`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `StreamBlockDevice` | `StreamBlockDevice(Stream stream, int logicalBlockSize, bool writable, bool leaveOpen = true, int? physicalBlockSize = null)` | | +| `CanWrite` | `bool CanWrite { get; }` | | +| `Geometry` | `BlockDeviceGeometry Geometry { get; }` | | +| `Dispose` | `void Dispose()` | | +| `Flush` | `void Flush()` | | +| `ReadBlocks` | `int ReadBlocks(long firstBlock, Span destination)` | | +| `Trim` | `void Trim(long firstBlock, long blockCount)` | | +| `WriteBlocks` | `void WriteBlocks(long firstBlock, ReadOnlySpan source)` | | #### `SymlinkResolver` diff --git a/Compression.Lib/ArchiveCompressionOptimizer.cs b/Compression.Lib/ArchiveCompressionOptimizer.cs new file mode 100644 index 000000000..0ebbd7a21 --- /dev/null +++ b/Compression.Lib/ArchiveCompressionOptimizer.cs @@ -0,0 +1,154 @@ +using Compression.Registry; + +namespace Compression.Lib; + +/// +/// Searches a creatable archive/container's finite option schema and keeps the +/// smallest verified same-format rebuild. Unlike , +/// which optimizes one compressed stream, this optimizer works on multi-entry +/// containers (compression method/level, dictionary, solid-block size, etc.). +/// +public static class ArchiveCompressionOptimizer { + public sealed record Result(long OriginalSize, long OptimizedSize, int EntriesOptimized, + IReadOnlyDictionary Parameters, int Probes); + + public static Result Optimize( + string inputPath, + string outputPath, + IArchiveFormatOperations ops, + IArchiveCreatable creator, + IFormatOptionsSchema schema, + int maxCombinations = 256) { + ArgumentException.ThrowIfNullOrEmpty(inputPath); + ArgumentException.ThrowIfNullOrEmpty(outputPath); + ArgumentNullException.ThrowIfNull(ops); + ArgumentNullException.ThrowIfNull(creator); + ArgumentNullException.ThrowIfNull(schema); + + var axes = SearchAxes(schema).ToArray(); + var originalSize = new FileInfo(inputPath).Length; + var sourceEntries = CountLiveEntries(inputPath, ops); + var bestSize = originalSize; + var bestParameters = (IReadOnlyDictionary)new Dictionary(); + string? bestPath = null; + var probes = 0; + + try { + if (axes.Length > 0) { + long product = 1; + foreach (var axis in axes) { + product = checked(Math.Min((long)maxCombinations + 1, product * axis.Values.Count)); + if (product > maxCombinations) break; + } + + if (product <= maxCombinations) { + foreach (var combination in EnumerateCombinations(axes)) + Probe(combination); + } else { + var current = axes.ToDictionary(a => a.Key, a => a.Default, StringComparer.Ordinal); + Probe(current); + var improved = true; + while (improved && probes < maxCombinations) { + improved = false; + foreach (var axis in axes) { + foreach (var value in axis.Values) { + if (probes >= maxCombinations) break; + if (current[axis.Key] == value) continue; + var trial = new Dictionary(current, StringComparer.Ordinal) { + [axis.Key] = value, + }; + var before = bestSize; + Probe(trial); + if (bestSize < before) { + current = trial; + improved = true; + } + } + } + } + } + } + + if (bestPath == null) { + AtomicFileWriter.WriteAtomic(outputPath, output => { + using var input = File.OpenRead(inputPath); + input.CopyTo(output); + }); + return new Result(originalSize, originalSize, 0, bestParameters, probes); + } + + AtomicFileWriter.WriteAtomic(outputPath, output => { + using var best = File.OpenRead(bestPath); + best.CopyTo(output); + }); + return new Result(originalSize, bestSize, sourceEntries, bestParameters, probes); + } finally { + if (bestPath != null) TryDelete(bestPath); + } + + void Probe(IReadOnlyDictionary parameters) { + ++probes; + var candidatePath = Path.Combine(Path.GetTempPath(), "cwb_arcopt_" + Guid.NewGuid().ToString("N") + ".tmp"); + try { + using (var input = File.OpenRead(inputPath)) + using (var candidate = new FileStream(candidatePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None)) { + RebuildVerb.RebuildToStream(input, candidate, ops, creator, parameters); + candidate.Flush(flushToDisk: true); + if (candidate.Length >= bestSize) return; + bestSize = candidate.Length; + } + + if (bestPath != null) TryDelete(bestPath); + bestPath = candidatePath; + candidatePath = ""; + bestParameters = new Dictionary(parameters, StringComparer.Ordinal); + } catch (Exception ex) when (ex is InvalidOperationException or NotSupportedException or IOException or ArgumentException) { + // An individual parameter combination may be invalid for the current + // content/profile. It is a rejected probe, not an optimization failure. + } finally { + if (!string.IsNullOrEmpty(candidatePath)) TryDelete(candidatePath); + } + } + } + + private sealed record Axis(string Key, IReadOnlyList Values, string Default); + + private static IEnumerable SearchAxes(IFormatOptionsSchema schema) { + foreach (var option in schema.OptionsSchema) { + IReadOnlyList? values = option.Kind switch { + FormatOptionKind.Enum or FormatOptionKind.Integer when option.AllowedValues is { Count: > 1 } + => option.AllowedValues, + FormatOptionKind.Boolean => ["false", "true"], + _ => null, + }; + if (values is { Count: > 1 }) + yield return new Axis(option.Key, values, option.Default); + } + } + + private static IEnumerable> EnumerateCombinations(IReadOnlyList axes) { + if (axes.Count == 0) yield break; + var indices = new int[axes.Count]; + while (true) { + var result = new Dictionary(axes.Count, StringComparer.Ordinal); + for (var i = 0; i < axes.Count; ++i) result[axes[i].Key] = axes[i].Values[indices[i]]; + yield return result; + + var position = axes.Count - 1; + while (position >= 0 && ++indices[position] == axes[position].Values.Count) { + indices[position] = 0; + --position; + } + if (position < 0) yield break; + } + } + + private static int CountLiveEntries(string path, IArchiveFormatOperations ops) { + using var input = File.OpenRead(path); + return ops.List(input, null).Count(e => !e.IsDirectory); + } + + private static void TryDelete(string path) { + try { File.Delete(path); } catch { /* best effort */ } + } +} diff --git a/Compression.Lib/CompactOperation.cs b/Compression.Lib/CompactOperation.cs index ecce34ce6..4c067b1ad 100644 --- a/Compression.Lib/CompactOperation.cs +++ b/Compression.Lib/CompactOperation.cs @@ -4,59 +4,22 @@ namespace Compression.Lib; /// -/// The composite compact maintenance verb: defrag → optimize → -/// shrink, run as one pass to produce the smallest valid container that -/// still holds the same contents. -/// -/// defrag consolidates live data so it is contiguous; -/// optimize re-encodes the payload with the best methods (where -/// the format is re-encodable); -/// shrink truncates the freed tail and steps the container down -/// to the smallest canonical size that still fits. -/// -/// With the standard trio is replaced -/// by a single minimal-geometry rebuild: the contents are extracted and -/// the container re-created at the smallest geometry the format allows (auto-fit -/// image size, smallest cluster, minimal root-directory entries). For a FAT -/// floppy that turns a fixed 1.44 MB image into a few-KB image whose -/// root-directory and FAT are sized to exactly hold the data — smaller, but no -/// longer a standard mountable floppy. Formats without geometry knobs fall back -/// to the standard compact and say so via . +/// Composite maintenance verb: defragment, optimize and shrink. Every stage is +/// optional and selected from the descriptor's real capabilities. /// public static class CompactOperation { - - /// Outcome of a compact pass. - /// Container size before compacting, in bytes. - /// Container size after compacting, in bytes. - /// Human-readable list of the steps that actually ran. - /// Whether the minimal-geometry rebuild was used. public sealed record CompactResult(long OriginalSize, long NewSize, IReadOnlyList StepsRun, bool Minimal); - /// Tunables for . public sealed class CompactOptions { - /// - /// When true, rebuild the container at the smallest geometry the format - /// allows instead of the conservative defrag+optimize+shrink trio. The - /// result is the smallest possible file but may no longer be a - /// standard/mountable image of that type. - /// public bool Minimal { get; init; } - /// Password for encrypted source containers (read side). public string? Password { get; init; } - /// Optional progress/diagnostic sink — one line per step. public Action? Log { get; init; } } - // Schema keys understood by the minimal-geometry rebuild, grouped by intent. private static readonly string[] SizeKeys = ["ImageSize", "TotalSize", "VolumeSize"]; private static readonly string[] UnitKeys = ["ClusterSize", "BlockSize", "UnitSize", "AllocationUnit", "AllocSize"]; private static readonly string[] CountKeys = ["RootEntries", "InodeCount", "InodeSize"]; - /// - /// Compacts the container at in place. Live contents - /// are preserved byte-for-byte; only layout, encoding and (in - /// mode) geometry change. - /// public static CompactResult Compact(string path, CompactOptions? options = null) { ArgumentException.ThrowIfNullOrEmpty(path); if (!File.Exists(path)) throw new FileNotFoundException("Container not found.", path); @@ -67,10 +30,10 @@ public static CompactResult Compact(string path, CompactOptions? options = null) var originalSize = new FileInfo(path).Length; var format = FormatDetector.Detect(path); var formatId = format.ToString(); + var descriptor = FormatRegistry.GetById(formatId); var ops = FormatRegistry.GetArchiveOps(formatId); var steps = new List(); - // ── Minimal: a single minimal-geometry rebuild replaces the whole trio ── if (options.Minimal) { if (ops is IArchiveCreatable && ops is IFormatOptionsSchema schema && SelectMinimalGeometry(schema) is { Count: > 0 } minimal) { @@ -81,7 +44,6 @@ public static CompactResult Compact(string path, CompactOptions? options = null) log($"compact: '{formatId}' exposes no minimal-geometry knobs — running standard compact instead."); } - // ── 1) Defragment — consolidate live data at the start ────────────────── if (ops is IArchiveDefragmentable defragmentable) { try { using var stream = File.Open(path, FileMode.Open, FileAccess.ReadWrite); @@ -93,9 +55,7 @@ public static CompactResult Compact(string path, CompactOptions? options = null) } } - // ── 2) Optimize — re-encode the payload where the format is re-encodable ─ if (formatId is "DoubleSpace" or "DriveSpace" or "DriveSpace3") { - var descriptor = FormatRegistry.GetById(formatId); if (descriptor != null) { try { var r = CvfOptimizer.Optimize(path, descriptor); @@ -105,6 +65,30 @@ public static CompactResult Compact(string path, CompactOptions? options = null) log($"optimize: skipped ({ex.GetType().Name}: {ex.Message})."); } } + } else if (descriptor?.Capabilities.HasFlag(FormatCapabilities.SupportsOptimize) == true + && ops is IArchiveCreatable creator + && ops is IFormatOptionsSchema archiveSchema + && archiveSchema.OptionsSchema.Count > 0 + && format != F.Zip + && !FormatDetector.IsStreamFormat(format) + && !FormatDetector.GetTarCompression(format).HasValue) { + // Multi-entry containers with their own finite creation schema (EWF, + // SquashFS, etc.) need the archive optimizer, not the stream optimizer. + // It searches the declared axes and accepts only verified same-format + // rebuilds smaller than the source; otherwise it copies through unchanged. + var tempOut = path + ".compact-arcopt.tmp"; + try { + var r = ArchiveCompressionOptimizer.Optimize(path, tempOut, ops, creator, archiveSchema); + File.Move(tempOut, path, overwrite: true); + steps.Add("optimize"); + log(r.OptimizedSize < r.OriginalSize + ? $"optimize: {r.OriginalSize:N0} → {r.OptimizedSize:N0} bytes across {r.Probes} parameter probe(s)." + : $"optimize: no smaller verified representation after {r.Probes} parameter probe(s)."); + } catch (Exception ex) { + log($"optimize: skipped ({ex.GetType().Name}: {ex.Message})."); + } finally { + if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { } + } } else if (format == F.Zip || FormatDetector.IsStreamFormat(format) || FormatDetector.GetTarCompression(format).HasValue) { var tempOut = path + ".compact-opt.tmp"; @@ -116,11 +100,10 @@ public static CompactResult Compact(string path, CompactOptions? options = null) } catch (Exception ex) { log($"optimize: skipped ({ex.GetType().Name}: {ex.Message})."); } finally { - if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { /* best effort */ } + if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { } } } - // ── 3) Shrink — truncate freed tail / step down to the smallest size ──── if (ops is IArchiveShrinkable shrinkable) { var tempOut = path + ".compact-shrink.tmp"; try { @@ -139,41 +122,29 @@ public static CompactResult Compact(string path, CompactOptions? options = null) } catch (Exception ex) { log($"shrink: skipped ({ex.GetType().Name}: {ex.Message})."); } finally { - if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { /* best effort */ } + if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { } } } return new CompactResult(originalSize, new FileInfo(path).Length, steps, Minimal: false); } - /// - /// Extracts every entry, then re-creates the container at minimal geometry - /// using the format's creation path. The swap only happens when the rebuilt - /// image both round-trips (lists at least as many entries as the source) and - /// is no larger than the original — otherwise the source is left untouched. - /// private static void TryMinimalRebuild(string path, F format, IReadOnlyDictionary minimalGeometry, string? password, Action log) { var sourceEntryCount = SafeFileCount(path, password); var tempDir = Path.Combine(Path.GetTempPath(), "cwb_compact_" + Guid.NewGuid().ToString("N")[..8]); - // Keep the original extension so the rebuilt image still content/extension- - // detects as the same format when we re-list it for the safety check - // (weak-magic formats like FAT lean on the extension). var tempOut = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(path))!, Path.GetFileNameWithoutExtension(path) + ".compact-min" + Path.GetExtension(path)); try { Directory.CreateDirectory(tempDir); ArchiveOperations.Extract(path, tempDir, password, files: null); var inputs = ArchiveOperations.EnumerateTempInputs(tempDir); - ArchiveOperations.Create(tempOut, inputs, - new CompressionOptions { Password = password }, - format, minimalGeometry); + new CompressionOptions { Password = password }, format, minimalGeometry); var rebuiltCount = SafeFileCount(tempOut, password); var rebuiltLen = new FileInfo(tempOut).Length; var originalLen = new FileInfo(path).Length; - if (rebuiltCount < sourceEntryCount) { log($"minimal rebuild: aborted — rebuilt image lists {rebuiltCount} file(s) vs {sourceEntryCount}; keeping original."); return; @@ -186,33 +157,22 @@ private static void TryMinimalRebuild(string path, F format, log($"minimal rebuild: re-created at minimal geometry — {originalLen:N0} → {rebuiltLen:N0} bytes " + $"({string.Join(", ", minimalGeometry.Select(kv => $"{kv.Key}={kv.Value}"))})."); } finally { - if (Directory.Exists(tempDir)) try { Directory.Delete(tempDir, recursive: true); } catch { /* best effort */ } - if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { /* best effort */ } + if (Directory.Exists(tempDir)) try { Directory.Delete(tempDir, recursive: true); } catch { } + if (File.Exists(tempOut)) try { File.Delete(tempOut); } catch { } } } private static int SafeFileCount(string path, string? password) { - try { - return ArchiveOperations.List(path, password).Count(e => !e.IsDirectory); - } catch { - return 0; - } + try { return ArchiveOperations.List(path, password).Count(e => !e.IsDirectory); } + catch { return 0; } } - /// - /// Picks the smallest-footprint value for each geometry knob the schema - /// exposes: auto-fit for the image size, the smallest concrete allocation - /// unit, and the smallest root/inode count. Only keys we understand are set; - /// everything else stays at its writer default. - /// private static Dictionary SelectMinimalGeometry(IFormatOptionsSchema schema) { var result = new Dictionary(StringComparer.Ordinal); var hasRealKnob = false; foreach (var opt in schema.OptionsSchema) { if (opt.AllowedValues is not { Count: > 0 } allowed) continue; - if (MatchesAny(opt.Key, SizeKeys)) { - // Auto-fit-to-contents is the minimal image size. var auto = allowed.FirstOrDefault(v => v.Contains("Auto", StringComparison.OrdinalIgnoreCase) || v.Contains("fit", StringComparison.OrdinalIgnoreCase)); @@ -225,8 +185,6 @@ private static Dictionary SelectMinimalGeometry(IFormatOptionsSc if (smallest != null) { result[opt.Key] = smallest; hasRealKnob = true; } } } - // Only a format with a real geometry knob gets the minimal-rebuild path. - // The universal opt-in flag tells the writer to drop its size headroom. if (!hasRealKnob) return []; result["MinimalGeometry"] = "true"; return result; @@ -235,53 +193,52 @@ private static Dictionary SelectMinimalGeometry(IFormatOptionsSc private static bool MatchesAny(string key, string[] candidates) => candidates.Any(c => string.Equals(key, c, StringComparison.OrdinalIgnoreCase)); - /// Returns the allowed value with the smallest parsed byte size (ignoring "Auto"). private static string? SmallestByBytes(IReadOnlyList allowed) { string? best = null; var bestBytes = long.MaxValue; - foreach (var v in allowed) { - var b = ParseByteSize(v); - if (b <= 0) continue; - if (b < bestBytes) { bestBytes = b; best = v; } + foreach (var value in allowed) { + var bytes = ParseByteSize(value); + if (bytes <= 0 || bytes >= bestBytes) continue; + bestBytes = bytes; + best = value; } return best; } - /// Returns the allowed value with the smallest leading integer (ignoring "Auto"). private static string? SmallestByLeadingInt(IReadOnlyList allowed) { string? best = null; - var bestN = long.MaxValue; - foreach (var v in allowed) { - var n = ParseLeadingInt(v); - if (n <= 0) continue; - if (n < bestN) { bestN = n; best = v; } + var bestNumber = long.MaxValue; + foreach (var value in allowed) { + var number = ParseLeadingInt(value); + if (number <= 0 || number >= bestNumber) continue; + bestNumber = number; + best = value; } return best; } - /// Parses "512 B" / "1 KB" / "32 KB" / "1.44 MB" → bytes; 0 if not a size. - private static long ParseByteSize(string s) { - var t = s.Trim(); + private static long ParseByteSize(string text) { + var value = text.Trim(); var i = 0; - while (i < t.Length && (char.IsDigit(t[i]) || t[i] is '.' or ',')) i++; + while (i < value.Length && (char.IsDigit(value[i]) || value[i] is '.' or ',')) ++i; if (i == 0) return 0; - if (!double.TryParse(t[..i].Replace(',', '.'), System.Globalization.NumberStyles.Float, - System.Globalization.CultureInfo.InvariantCulture, out var num)) return 0; - var rest = t[i..].TrimStart().ToUpperInvariant(); - long mult = rest switch { + if (!double.TryParse(value[..i].Replace(',', '.'), System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out var number)) return 0; + var rest = value[i..].TrimStart().ToUpperInvariant(); + long multiplier = rest switch { var r when r.StartsWith("KB") || r.StartsWith("K") => 1024L, var r when r.StartsWith("MB") || r.StartsWith("M") => 1024L * 1024, var r when r.StartsWith("GB") || r.StartsWith("G") => 1024L * 1024 * 1024, var r when r.StartsWith('B') || r.Length == 0 => 1L, _ => 0L, }; - return (long)(num * mult); + return (long)(number * multiplier); } - private static long ParseLeadingInt(string s) { - var t = s.Trim(); + private static long ParseLeadingInt(string text) { + var value = text.Trim(); var i = 0; - while (i < t.Length && char.IsDigit(t[i])) i++; - return i > 0 && long.TryParse(t[..i], out var n) ? n : 0; + while (i < value.Length && char.IsDigit(value[i])) ++i; + return i > 0 && long.TryParse(value[..i], out var number) ? number : 0; } } diff --git a/Compression.Lib/Compression.Lib.csproj b/Compression.Lib/Compression.Lib.csproj index 8e2da84b2..226a5df95 100644 --- a/Compression.Lib/Compression.Lib.csproj +++ b/Compression.Lib/Compression.Lib.csproj @@ -627,4 +627,11 @@ + + + + + + diff --git a/Compression.Lib/FormatRegistration.cs b/Compression.Lib/FormatRegistration.cs index ad1f0fd42..28469e282 100644 --- a/Compression.Lib/FormatRegistration.cs +++ b/Compression.Lib/FormatRegistration.cs @@ -1,3 +1,4 @@ +using System.Runtime.ExceptionServices; using Compression.Core.DiskImage; using Compression.Lib.FsConversion; using Compression.Registry; @@ -6,45 +7,29 @@ namespace Compression.Lib; /// /// Registers all format descriptors with the central registry. -/// The RegisterFormats() partial method is generated at compile time by -/// Compression.Registry.Generator, which discovers all public classes implementing -/// IFormatDescriptor with a parameterless constructor across referenced assemblies. -/// Compound tar descriptors are registered manually since they are composites, not standalone formats. +/// The generated partial methods discover descriptors, building blocks and +/// filesystem-driver sidecars across referenced assemblies without reflection. /// public static partial class FormatRegistration { private static int _initStarted; private static readonly ManualResetEventSlim _initDone = new(initialState: false); + private static ExceptionDispatchInfo? _initFailure; - /// True iff the registry is fully populated and safe to enumerate. - public static bool IsReady => _initDone.IsSet; + /// True iff the registry completed successfully and is safe to enumerate. + public static bool IsReady => _initDone.IsSet && _initFailure == null; - /// - /// Async-friendly variant of . Returns a - /// completed task immediately if the registry is already populated; - /// otherwise dispatches the registration to a worker thread and yields the - /// resulting task so UI callers can await it without blocking the - /// dispatcher. The UI sets a busy cursor + status line while awaiting, - /// giving the user a "loading…" hint instead of an apparent freeze. - /// public static System.Threading.Tasks.Task EnsureInitializedAsync() { - if (_initDone.IsSet) return System.Threading.Tasks.Task.CompletedTask; + if (_initDone.IsSet) { + _initFailure?.Throw(); + return System.Threading.Tasks.Task.CompletedTask; + } return System.Threading.Tasks.Task.Run(EnsureInitialized); } - /// - /// Ensures all format descriptors and building blocks are registered exactly once. - /// Thread-safe: the first caller runs the registration; concurrent callers - /// WAIT for it to finish (rather than seeing a half-populated - /// ). The earlier "flip flag and return" pattern - /// caused a race: a background warm-up Task that had set the flag but not - /// yet finished registering would let a synchronous caller through with an - /// empty registry, producing "No creatable formats are registered." popups. - /// public static void EnsureInitialized() { if (Interlocked.CompareExchange(ref _initStarted, 1, 0) != 0) { - // Another thread is already registering — wait for it to finish so the - // registry is fully populated before we return. _initDone.Wait(); + _initFailure?.Throw(); return; } @@ -52,40 +37,43 @@ public static void EnsureInitialized() { RegisterFormats(); RegisterCompoundTar(); RegisterBuildingBlocks(); + RegisterFilesystemDrivers(); FormatRegistry.Initialize(); - // Wire the in-place FS-variant converter delegate so PartitionEditor. - // ConvertFilesystem can dispatch FAT12↔16↔32 and ext2→3→4 conversions - // without taking a hard dependency on Compression.Lib from Compression.Core. PartitionEditor.InPlaceFilesystemConverter = (stream, srcId, dstId) => InPlaceConverter.TryConvert(stream, srcId, dstId) is InPlaceConversionResult.Succeeded or InPlaceConversionResult.NoOp; - // Wire the in-place FS-resizer delegate so PartitionEditor.ResizePartition - // can dispatch FAT shrink/grow and ext shrink/grow without a Core → - // FileSystem.* dependency. PartitionEditor.InPlaceFilesystemResizer = (stream, fsId, newSize, isShrink) => { if (!FilesystemResizer.IsSupported(fsId)) return false; if (isShrink) FilesystemResizer.Shrink(stream, fsId, newSize); else FilesystemResizer.Grow(stream, fsId, newSize); return true; }; + } catch (Exception e) { + // Registration is deliberately one-shot. Once constructor side effects have + // populated the registries, blindly retrying would create duplicate entries. + // Preserve and rethrow the original failure for every concurrent/later caller + // instead of letting waiters observe a false-success half registry. + _initFailure = ExceptionDispatchInfo.Capture(e); + throw; } finally { _initDone.Set(); } } - /// - /// Generated by Compression.Registry.Generator — calls FormatRegistry.Register() - /// for every discovered format descriptor. Falls back to no-op if generator is not wired up. - /// + /// Generated by Compression.Registry.Generator. static partial void RegisterFormats(); + /// Generated by Compression.Registry.Generator. + static partial void RegisterBuildingBlocks(); + /// - /// Generated by Compression.Registry.Generator — calls BuildingBlockRegistry.Register() - /// for every discovered building block. Falls back to no-op if generator is not wired up. + /// Generated by Compression.Registry.Generator. Registers public parameterless + /// implementations after their format + /// descriptors and before the registry is finalized. /// - static partial void RegisterBuildingBlocks(); + static partial void RegisterFilesystemDrivers(); private static void RegisterCompoundTar() { FormatRegistry.Register(new CompoundTarDescriptor("TarGz", "tar.gz", "Gzip", ".tar.gz", [".tar.gz", ".tgz"])); diff --git a/Compression.Mounting.Dokan/Compression.Mounting.Dokan.csproj b/Compression.Mounting.Dokan/Compression.Mounting.Dokan.csproj new file mode 100644 index 000000000..cf89eb6d5 --- /dev/null +++ b/Compression.Mounting.Dokan/Compression.Mounting.Dokan.csproj @@ -0,0 +1,13 @@ + + + + Compression.Mounting.Dokan + false + CS1591 + + + + + + + diff --git a/Compression.Mounting.Dokan/DokanFilesystemMountBackend.cs b/Compression.Mounting.Dokan/DokanFilesystemMountBackend.cs new file mode 100644 index 000000000..93797a86f --- /dev/null +++ b/Compression.Mounting.Dokan/DokanFilesystemMountBackend.cs @@ -0,0 +1,56 @@ +using Compression.Registry; + +namespace Compression.Mounting.Dokan; + +/// +/// Dokany 2 backend boundary. The runtime probe is real, but mount-mode support +/// remains disabled until the callback bridge has passed backend conformance. +/// +public sealed class DokanFilesystemMountBackend : IFilesystemMountBackend { + private readonly DokanRuntimeStatus _runtime; + + public DokanFilesystemMountBackend() + : this(DokanRuntimeProbe.Probe()) { } + + public DokanFilesystemMountBackend(DokanRuntimeStatus runtime) + => this._runtime = runtime ?? throw new ArgumentNullException(nameof(runtime)); + + public DokanRuntimeStatus RuntimeStatus => this._runtime; + + public MountBackendProfile GetProfile() { + var limitations = new List(); + if (!this._runtime.IsAvailable) + limitations.Add(this._runtime.UnavailableReason ?? "Dokany 2 runtime is unavailable."); + limitations.Add( + "The Dokan callback bridge has not been qualified yet; read-only and read-write mounting are intentionally disabled." + ); + + return new( + Id: "dokan", + DisplayName: "Dokany 2", + IsAvailable: this._runtime.IsAvailable, + SupportsReadOnly: false, + SupportsReadWrite: false, + RequiredReadCapabilities: FilesystemDriverCapabilities.None, + RequiredWriteCapabilities: FilesystemDriverCapabilities.None, + Limitations: limitations + ); + } + + public ValueTask MountAsync( + FilesystemMountRequest request, + CancellationToken cancellationToken = default + ) { + ArgumentNullException.ThrowIfNull(request); + cancellationToken.ThrowIfCancellationRequested(); + + if (!this._runtime.IsAvailable) + throw new PlatformNotSupportedException( + this._runtime.UnavailableReason ?? "Dokany 2 runtime is unavailable." + ); + + throw new NotSupportedException( + "The Dokan callback bridge is not implemented yet; this backend deliberately advertises no supported mount mode." + ); + } +} diff --git a/Compression.Mounting.Dokan/DokanRuntimeProbe.cs b/Compression.Mounting.Dokan/DokanRuntimeProbe.cs new file mode 100644 index 000000000..82930ffaf --- /dev/null +++ b/Compression.Mounting.Dokan/DokanRuntimeProbe.cs @@ -0,0 +1,89 @@ +using System.Runtime.InteropServices; + +namespace Compression.Mounting.Dokan; + +public sealed record DokanRuntimeStatus( + bool IsAvailable, + uint LibraryVersion, + uint DriverVersion, + string? LibraryPath, + string? UnavailableReason +); + +/// +/// Probes the native Dokany 2 user-mode library and its kernel driver without +/// registering a filesystem or requiring administrator privileges. +/// +public static class DokanRuntimeProbe { + private const string LibraryFileName = "dokan2.dll"; + private const string LibraryVersionExport = "DokanVersion"; + private const string DriverVersionExport = "DokanDriverVersion"; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + private delegate uint VersionProcedure(); + + public static DokanRuntimeStatus Probe() { + if (!OperatingSystem.IsWindows()) + return Unavailable("Dokan is a Windows-only mount backend."); + + var candidates = new[] { + Path.Combine(AppContext.BaseDirectory, LibraryFileName), + Path.Combine(Environment.SystemDirectory, LibraryFileName), + }.Distinct(StringComparer.OrdinalIgnoreCase); + + string? lastFailure = null; + foreach (var candidate in candidates) { + if (!File.Exists(candidate)) continue; + + IntPtr library = IntPtr.Zero; + try { + if (!NativeLibrary.TryLoad(candidate, out library)) { + lastFailure = $"Found '{candidate}' but the native loader rejected it."; + continue; + } + + if (!TryGetVersionProcedure(library, LibraryVersionExport, out var libraryVersionProcedure)) + return Unavailable($"'{candidate}' does not export {LibraryVersionExport}().", candidate); + if (!TryGetVersionProcedure(library, DriverVersionExport, out var driverVersionProcedure)) + return Unavailable($"'{candidate}' does not export {DriverVersionExport}().", candidate); + + var libraryVersion = libraryVersionProcedure(); + var driverVersion = driverVersionProcedure(); + if (libraryVersion == 0) + return new(false, 0, driverVersion, candidate, "DokanVersion() returned 0."); + if (driverVersion == 0) + return new( + false, + libraryVersion, + 0, + candidate, + "The Dokan user-mode library is present, but DokanDriverVersion() returned 0; the Dokan 2 driver is unavailable or could not be queried." + ); + + return new(true, libraryVersion, driverVersion, candidate, null); + } catch (BadImageFormatException ex) { + lastFailure = $"'{candidate}' has the wrong architecture or is not a valid native library: {ex.Message}"; + } catch (Exception ex) when (ex is DllNotFoundException or EntryPointNotFoundException) { + lastFailure = $"Unable to use '{candidate}': {ex.Message}"; + } finally { + if (library != IntPtr.Zero) + NativeLibrary.Free(library); + } + } + + return Unavailable(lastFailure ?? $"{LibraryFileName} was not found beside the application or in the Windows system directory."); + } + + private static bool TryGetVersionProcedure(IntPtr library, string export, out VersionProcedure procedure) { + if (NativeLibrary.TryGetExport(library, export, out var address)) { + procedure = Marshal.GetDelegateForFunctionPointer(address); + return true; + } + + procedure = null!; + return false; + } + + private static DokanRuntimeStatus Unavailable(string reason, string? libraryPath = null) + => new(false, 0, 0, libraryPath, reason); +} diff --git a/Compression.Mounting.Dokan/README.md b/Compression.Mounting.Dokan/README.md new file mode 100644 index 000000000..37b4a1e40 --- /dev/null +++ b/Compression.Mounting.Dokan/README.md @@ -0,0 +1,7 @@ +# Compression.Mounting.Dokan + +Windows Dokany 2 adapter for the mount-neutral `Compression.Mounting` contracts. + +The first slice is intentionally limited to a real runtime probe and a truthful backend profile. It checks the application directory and Windows system directory for `dokan2.dll`, resolves `DokanVersion` and `DokanDriverVersion`, and only reports the runtime as available when both the user-mode library and kernel driver answer successfully. + +No filesystem callback is advertised yet. `SupportsReadOnly` and `SupportsReadWrite` remain false until the stable-node-ID callback bridge and its conformance tests exist. A present DLL is not treated as evidence that read or write mounting works. diff --git a/Compression.Mounting/Compression.Mounting.csproj b/Compression.Mounting/Compression.Mounting.csproj new file mode 100644 index 000000000..b227880c9 --- /dev/null +++ b/Compression.Mounting/Compression.Mounting.csproj @@ -0,0 +1,13 @@ + + + + Compression.Mounting + false + CS1591 + + + + + + + diff --git a/Compression.Mounting/FilesystemMountCapabilityResolver.cs b/Compression.Mounting/FilesystemMountCapabilityResolver.cs new file mode 100644 index 000000000..db6f1bbc5 --- /dev/null +++ b/Compression.Mounting/FilesystemMountCapabilityResolver.cs @@ -0,0 +1,93 @@ +using Compression.Registry; + +namespace Compression.Mounting; + +public static class FilesystemMountCapabilityResolver { + public const FilesystemDriverCapabilities CoreReadCapabilities = + FilesystemDriverCapabilities.EnumerateDirectories | + FilesystemDriverCapabilities.ReadData | + FilesystemDriverCapabilities.RandomAccess | + FilesystemDriverCapabilities.StableNodeIds; + + public const FilesystemDriverCapabilities CoreWriteCapabilities = + FilesystemDriverCapabilities.WriteData | + FilesystemDriverCapabilities.Truncate | + FilesystemDriverCapabilities.CreateFile | + FilesystemDriverCapabilities.DeleteFile | + FilesystemDriverCapabilities.CreateDirectory | + FilesystemDriverCapabilities.RemoveDirectory | + FilesystemDriverCapabilities.Rename | + FilesystemDriverCapabilities.Flush; + + public static MountPlan Resolve( + FilesystemDriverProfile driverProfile, + MountBackendProfile backend, + MountAccessMode accessMode, + bool sourceCanWrite + ) { + ArgumentNullException.ThrowIfNull(driverProfile); + ArgumentNullException.ThrowIfNull(backend); + + var reasons = new List(); + var requiredRead = CoreReadCapabilities | backend.RequiredReadCapabilities; + var required = requiredRead; + + if (!backend.IsAvailable) + reasons.Add(new(MountSupportReasonCode.BackendUnavailable, $"Mount backend '{backend.DisplayName}' is not available on this host.")); + + if (!backend.SupportsReadOnly) + reasons.Add(new(MountSupportReasonCode.BackendDoesNotSupportReadOnly, $"Mount backend '{backend.DisplayName}' does not support filesystem reads.")); + + if (!driverProfile.CanMount) + reasons.Add(new(MountSupportReasonCode.FilesystemProfileNotMountable, $"Filesystem profile '{driverProfile.ProfileName}' is not mount-grade.")); + + var missingRead = requiredRead & ~driverProfile.Capabilities; + if (missingRead != FilesystemDriverCapabilities.None) + reasons.Add(MissingCapabilitiesReason(missingRead, "read-only")); + + if (accessMode == MountAccessMode.ReadWrite) { + required |= CoreWriteCapabilities | backend.RequiredWriteCapabilities; + + if (!backend.SupportsReadWrite) + reasons.Add(new(MountSupportReasonCode.BackendDoesNotSupportReadWrite, $"Mount backend '{backend.DisplayName}' does not support writable mounts.")); + + if (!driverProfile.CanMountWritable) + reasons.Add(new(MountSupportReasonCode.FilesystemProfileNotWritable, $"Filesystem profile '{driverProfile.ProfileName}' does not advertise writable mounting.")); + + if (!sourceCanWrite) + reasons.Add(new(MountSupportReasonCode.SourceIsReadOnly, "The backing image or an outer container layer is read-only.")); + + if (driverProfile.MutationModel is FilesystemMutationModel.None or FilesystemMutationModel.WholeImageRebuild) + reasons.Add(new(MountSupportReasonCode.UnsupportedMutationModel, $"Mutation model '{driverProfile.MutationModel}' is not suitable for mounted random writes.")); + + var missingWrite = (CoreWriteCapabilities | backend.RequiredWriteCapabilities) & ~driverProfile.Capabilities; + if (missingWrite != FilesystemDriverCapabilities.None) + reasons.Add(MissingCapabilitiesReason(missingWrite, "read-write")); + } + + var missing = required & ~driverProfile.Capabilities; + var limitations = driverProfile.Limitations.Concat(backend.Limitations).Distinct(StringComparer.Ordinal).ToArray(); + + return new( + accessMode, + reasons.Count == 0, + driverProfile, + backend, + required, + missing, + reasons, + limitations + ); + } + + private static MountSupportReason MissingCapabilitiesReason(FilesystemDriverCapabilities missing, string mode) + => new( + MountSupportReasonCode.MissingDriverCapabilities, + $"Filesystem profile is missing {mode} mount primitives: {FormatCapabilities(missing)}.", + missing + ); + + private static string FormatCapabilities(FilesystemDriverCapabilities capabilities) + => string.Join(", ", Enum.GetValues() + .Where(value => value != FilesystemDriverCapabilities.None && (value & (value - 1)) == 0 && capabilities.HasFlag(value))); +} diff --git a/Compression.Mounting/FilesystemMountLauncher.cs b/Compression.Mounting/FilesystemMountLauncher.cs new file mode 100644 index 000000000..1f80862f8 --- /dev/null +++ b/Compression.Mounting/FilesystemMountLauncher.cs @@ -0,0 +1,119 @@ +using Compression.Registry; + +namespace Compression.Mounting; + +/// +/// Opens a backing image and filesystem session for one already-selected mount +/// request. Capability policy is re-evaluated against the exact stream and the +/// opened session immediately before the backend receives ownership. +/// +public sealed class FilesystemMountLauncher(MountBackendRegistry backends) { + private readonly MountBackendRegistry _backends = backends ?? throw new ArgumentNullException(nameof(backends)); + + public async ValueTask MountAsync( + string imagePath, + string formatId, + MountPlan requestedPlan, + string target, + CancellationToken cancellationToken = default + ) { + ArgumentException.ThrowIfNullOrWhiteSpace(imagePath); + ArgumentException.ThrowIfNullOrWhiteSpace(formatId); + ArgumentNullException.ThrowIfNull(requestedPlan); + ArgumentException.ThrowIfNullOrWhiteSpace(target); + cancellationToken.ThrowIfCancellationRequested(); + + if (!FormatRegistry.FilesystemFormatIds.Contains(formatId, StringComparer.OrdinalIgnoreCase)) + throw new ArgumentException($"Format '{formatId}' is not registered as a filesystem.", nameof(formatId)); + + var backend = this._backends.GetBackend(requestedPlan.BackendId); + FileStream? source = null; + IFilesystemSession? filesystem = null; + var ownershipTransferred = false; + + try { + source = OpenBackingSource(imagePath, requestedPlan.AccessMode); + + var probePosition = source.CanSeek ? source.Position : 0; + FilesystemDriverProfile probedProfile; + try { + probedProfile = FormatRegistry.ProbeFilesystem(formatId, source); + } finally { + if (source.CanSeek) + source.Position = probePosition; + } + + var resolvedPlan = this._backends.ResolveFilesystem( + requestedPlan.BackendId, + probedProfile, + requestedPlan.AccessMode, + source.CanWrite + ); + EnsureSupported(resolvedPlan); + + filesystem = FormatRegistry.OpenFilesystem( + formatId, + source, + new FilesystemOpenOptions( + ReadOnly: requestedPlan.AccessMode == MountAccessMode.ReadOnly, + LeaveOpen: false + ) + ); + + // OpenFilesystem is allowed to specialize the profile further than Probe. + // Re-resolve once more so the backend never receives a session whose exact + // opened profile is weaker than the plan shown by the probe. + resolvedPlan = this._backends.ResolveFilesystem( + requestedPlan.BackendId, + filesystem.Profile, + requestedPlan.AccessMode, + source.CanWrite + ); + EnsureSupported(resolvedPlan); + + cancellationToken.ThrowIfCancellationRequested(); + var mounted = await backend.MountAsync( + new FilesystemMountRequest(filesystem, target, resolvedPlan, OwnsFilesystemSession: true), + cancellationToken + ).ConfigureAwait(false); + + ownershipTransferred = true; + filesystem = null; + source = null; + return mounted; + } finally { + if (!ownershipTransferred) { + filesystem?.Dispose(); + source?.Dispose(); + } + } + } + + private static FileStream OpenBackingSource(string path, MountAccessMode accessMode) + => accessMode switch { + MountAccessMode.ReadOnly => new(path, FileMode.Open, FileAccess.Read, FileShare.Read), + MountAccessMode.ReadWrite => new(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read), + _ => throw new ArgumentOutOfRangeException(nameof(accessMode), accessMode, "Unknown mount access mode."), + }; + + private static void EnsureSupported(MountPlan plan) { + if (plan.IsSupported) return; + throw new FilesystemMountNotSupportedException(plan); + } +} + +public sealed class FilesystemMountNotSupportedException : InvalidOperationException { + public FilesystemMountNotSupportedException(MountPlan plan) + : base(CreateMessage(plan)) + => this.Plan = plan ?? throw new ArgumentNullException(nameof(plan)); + + public MountPlan Plan { get; } + + private static string CreateMessage(MountPlan? plan) { + ArgumentNullException.ThrowIfNull(plan); + var reasons = plan.Reasons.Count == 0 + ? "mount plan is unsupported" + : string.Join("; ", plan.Reasons.Select(static reason => reason.Message)); + return $"{plan.AccessMode} mount through backend '{plan.BackendId}' is not supported: {reasons}."; + } +} diff --git a/Compression.Mounting/IFilesystemMountBackend.cs b/Compression.Mounting/IFilesystemMountBackend.cs new file mode 100644 index 000000000..a8f3ab556 --- /dev/null +++ b/Compression.Mounting/IFilesystemMountBackend.cs @@ -0,0 +1,31 @@ +using Compression.Registry; + +namespace Compression.Mounting; + +/// +/// One mount request over an already-open filesystem session. +/// When is true, ownership transfers to the +/// returned only after +/// completes successfully. If mounting throws or is cancelled, the caller still +/// owns and must dispose the filesystem session. +/// +public sealed record FilesystemMountRequest( + IFilesystemSession Filesystem, + string Target, + MountPlan Plan, + bool OwnsFilesystemSession = false +); + +public interface IFilesystemMountBackend { + MountBackendProfile GetProfile(); + ValueTask MountAsync(FilesystemMountRequest request, CancellationToken cancellationToken = default); +} + +public interface IMountSession : IAsyncDisposable { + string BackendId { get; } + string Target { get; } + MountAccessMode AccessMode { get; } + bool IsMounted { get; } + ValueTask FlushAsync(CancellationToken cancellationToken = default); + ValueTask UnmountAsync(CancellationToken cancellationToken = default); +} diff --git a/Compression.Mounting/MountAccessMode.cs b/Compression.Mounting/MountAccessMode.cs new file mode 100644 index 000000000..18f4e700f --- /dev/null +++ b/Compression.Mounting/MountAccessMode.cs @@ -0,0 +1,6 @@ +namespace Compression.Mounting; + +public enum MountAccessMode { + ReadOnly, + ReadWrite, +} diff --git a/Compression.Mounting/MountBackendRegistry.cs b/Compression.Mounting/MountBackendRegistry.cs new file mode 100644 index 000000000..dd5411f7a --- /dev/null +++ b/Compression.Mounting/MountBackendRegistry.cs @@ -0,0 +1,60 @@ +using Compression.Registry; + +namespace Compression.Mounting; + +public sealed class MountBackendRegistry { + private readonly IReadOnlyList _backends; + private readonly IReadOnlyDictionary _backendsById; + + public MountBackendRegistry(IEnumerable backends) { + ArgumentNullException.ThrowIfNull(backends); + + var materialized = backends.ToArray(); + var byId = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var backend in materialized) { + ArgumentNullException.ThrowIfNull(backend); + var profile = backend.GetProfile(); + if (!byId.TryAdd(profile.Id, backend)) + throw new ArgumentException($"Duplicate mount backend id '{profile.Id}'.", nameof(backends)); + } + + this._backends = materialized; + this._backendsById = byId; + } + + public IReadOnlyList Backends => this._backends; + + public IFilesystemMountBackend GetBackend(string backendId) { + ArgumentException.ThrowIfNullOrWhiteSpace(backendId); + return this._backendsById.TryGetValue(backendId, out var backend) + ? backend + : throw new KeyNotFoundException($"Unknown mount backend '{backendId}'."); + } + + public MountAccessOptions ResolveFilesystem(FilesystemDriverProfile driverProfile, bool sourceCanWrite) { + ArgumentNullException.ThrowIfNull(driverProfile); + + var readOnly = new MountPlan[this._backends.Count]; + var readWrite = new MountPlan[this._backends.Count]; + for (var i = 0; i < this._backends.Count; ++i) { + var profile = this._backends[i].GetProfile(); + readOnly[i] = FilesystemMountCapabilityResolver.Resolve(driverProfile, profile, MountAccessMode.ReadOnly, sourceCanWrite); + readWrite[i] = FilesystemMountCapabilityResolver.Resolve(driverProfile, profile, MountAccessMode.ReadWrite, sourceCanWrite); + } + + return new(readOnly, readWrite); + } + + public MountPlan ResolveFilesystem( + string backendId, + FilesystemDriverProfile driverProfile, + MountAccessMode accessMode, + bool sourceCanWrite + ) { + ArgumentException.ThrowIfNullOrWhiteSpace(backendId); + ArgumentNullException.ThrowIfNull(driverProfile); + + var backend = this.GetBackend(backendId); + return FilesystemMountCapabilityResolver.Resolve(driverProfile, backend.GetProfile(), accessMode, sourceCanWrite); + } +} diff --git a/Compression.Mounting/MountCapabilityModel.cs b/Compression.Mounting/MountCapabilityModel.cs new file mode 100644 index 000000000..ef9b0c16d --- /dev/null +++ b/Compression.Mounting/MountCapabilityModel.cs @@ -0,0 +1,50 @@ +using Compression.Registry; + +namespace Compression.Mounting; + +public enum MountSupportReasonCode { + BackendUnavailable, + BackendDoesNotSupportReadOnly, + BackendDoesNotSupportReadWrite, + FilesystemProfileNotMountable, + FilesystemProfileNotWritable, + SourceIsReadOnly, + UnsupportedMutationModel, + MissingDriverCapabilities, +} + +public sealed record MountSupportReason( + MountSupportReasonCode Code, + string Message, + FilesystemDriverCapabilities MissingCapabilities = FilesystemDriverCapabilities.None +); + +public sealed record MountBackendProfile( + string Id, + string DisplayName, + bool IsAvailable, + bool SupportsReadOnly, + bool SupportsReadWrite, + FilesystemDriverCapabilities RequiredReadCapabilities, + FilesystemDriverCapabilities RequiredWriteCapabilities, + IReadOnlyList Limitations +); + +public sealed record MountPlan( + MountAccessMode AccessMode, + bool IsSupported, + FilesystemDriverProfile DriverProfile, + MountBackendProfile Backend, + FilesystemDriverCapabilities RequiredCapabilities, + FilesystemDriverCapabilities MissingCapabilities, + IReadOnlyList Reasons, + IReadOnlyList Limitations +); + +public sealed record MountAccessOptions( + IReadOnlyList ReadOnly, + IReadOnlyList ReadWrite +) { + public bool CanMountReadOnly => this.ReadOnly.Any(static plan => plan.IsSupported); + public bool CanMountReadWrite => this.ReadWrite.Any(static plan => plan.IsSupported); +} diff --git a/Compression.NativeUI/Compression.NativeUI.csproj b/Compression.NativeUI/Compression.NativeUI.csproj new file mode 100644 index 000000000..dc13bc7a3 --- /dev/null +++ b/Compression.NativeUI/Compression.NativeUI.csproj @@ -0,0 +1,23 @@ + + + + WinExe + Compression.NativeUI + false + false + CS1591 + + + + + + + + + + + + + + + diff --git a/Compression.NativeUI/IMountLauncher.cs b/Compression.NativeUI/IMountLauncher.cs new file mode 100644 index 000000000..390dabede --- /dev/null +++ b/Compression.NativeUI/IMountLauncher.cs @@ -0,0 +1,18 @@ +using Compression.Mounting; + +namespace Compression.NativeUI; + +/// +/// Composition seam between the cross-platform UI and an actual filesystem-session opener. +/// Dokan/FUSE composition owns opening the image and filesystem session; the UI owns the +/// user-selected plan and the returned mount-session lifecycle. +/// +internal interface IMountLauncher { + ValueTask MountAsync( + string imagePath, + string formatId, + MountPlan plan, + string target, + CancellationToken cancellationToken = default + ); +} diff --git a/Compression.NativeUI/MainForm.cs b/Compression.NativeUI/MainForm.cs new file mode 100644 index 000000000..b3d6100ff --- /dev/null +++ b/Compression.NativeUI/MainForm.cs @@ -0,0 +1,303 @@ +using Compression.Lib; +using Compression.Mounting; +using Compression.Registry; +using Hawkynt.NativeForms; + +namespace Compression.NativeUI; + +internal sealed class MainForm : Form { + private readonly IFilesystemMountBackend[] _backends; + private readonly MountBackendRegistry _mountBackends; + private readonly IMountLauncher? _mountLauncher; + + private readonly FilePicker _imagePicker = new() { + Bounds = new(132, 24, 544, 28), + Filter = "All files|*.*", + PlaceholderText = "Filesystem image", + Title = "Select an image to mount", + }; + + private readonly ComboBox _accessPicker = new() { Bounds = new(132, 68, 190, 28) }; + private readonly ComboBox _backendPicker = new() { Bounds = new(410, 68, 266, 28) }; + private readonly TextBox _targetBox = new() { + Bounds = new(132, 112, 544, 28), + PlaceholderText = "Drive letter or mountpoint", + }; + private readonly Button _probeButton = new() { Bounds = new(132, 156, 118, 32), Text = "Probe" }; + private readonly Button _mountButton = new() { Bounds = new(262, 156, 118, 32), Text = "Mount", Enabled = false }; + private readonly Button _unmountButton = new() { Bounds = new(392, 156, 118, 32), Text = "Unmount", Enabled = false }; + private readonly TextBox _detailsBox = new() { Bounds = new(24, 224, 652, 280), Multiline = true, ReadOnly = true }; + private readonly Label _statusLabel = new() { + Bounds = new(24, 520, 652, 28), + Text = "Select an image and probe its mount capabilities.", + }; + + private FilesystemDriverProfile? _driverProfile; + private string? _formatId; + private bool _sourceCanWrite; + private IMountSession? _mountSession; + private bool _busy; + + public MainForm(IEnumerable backends, IMountLauncher? mountLauncher = null) { + ArgumentNullException.ThrowIfNull(backends); + this._backends = backends.ToArray(); + this._mountBackends = new(this._backends); + this._mountLauncher = mountLauncher; + + this.Text = "CompressionWorkbench — Mount"; + this.Bounds = new(0, 0, 720, 600); + this.StartPosition = FormStartPosition.CenterScreen; + this.MinimumSize = new(720, 600); + + this.Controls.AddRange( + new Label { Bounds = new(24, 28, 96, 24), Text = "Image" }, this._imagePicker, + new Label { Bounds = new(24, 72, 96, 24), Text = "Access" }, this._accessPicker, + new Label { Bounds = new(338, 72, 64, 24), Text = "Backend" }, this._backendPicker, + new Label { Bounds = new(24, 116, 96, 24), Text = "Target" }, this._targetBox, + this._probeButton, this._mountButton, this._unmountButton, + new Label { Bounds = new(24, 200, 160, 24), Text = "Resolved capabilities" }, + this._detailsBox, this._statusLabel + ); + + this._accessPicker.DisplaySelector = static item => item switch { + MountAccessMode.ReadOnly => "Read-only", + MountAccessMode.ReadWrite => "Read-write", + _ => string.Empty, + }; + this._accessPicker.Items.Add(MountAccessMode.ReadOnly); + this._accessPicker.Items.Add(MountAccessMode.ReadWrite); + this._accessPicker.SelectedIndex = 0; + + this._backendPicker.DisplaySelector = static item => item is IFilesystemMountBackend backend + ? backend.GetProfile().DisplayName + : string.Empty; + foreach (var backend in this._backends) + this._backendPicker.Items.Add(backend); + + if (this._backends.Length > 0) + this._backendPicker.SelectedIndex = 0; + else { + this._backendPicker.Enabled = false; + this._backendPicker.PlaceholderText = "No mount backend registered"; + } + + this._imagePicker.PathChanged += (_, _) => this.ResetProbe(); + this._accessPicker.SelectedIndexChanged += (_, _) => this.RefreshPlan(); + this._backendPicker.SelectedIndexChanged += (_, _) => this.RefreshPlan(); + this._targetBox.TextChanged += (_, _) => this.RefreshMountButton(); + this._probeButton.Click += (_, _) => this.Probe(); + this._mountButton.Click += async (_, _) => await this.MountAsync(); + this._unmountButton.Click += async (_, _) => await this.UnmountAsync(); + this.FormClosing += (_, _) => this.CleanupActiveMount(); + } + + private string ImagePath => string.IsNullOrWhiteSpace(this._imagePicker.SelectedPath) + ? this._imagePicker.Text.Trim() + : this._imagePicker.SelectedPath; + + private MountAccessMode AccessMode + => this._accessPicker.SelectedItem is MountAccessMode mode ? mode : MountAccessMode.ReadOnly; + + private IFilesystemMountBackend? SelectedBackend + => this._backendPicker.SelectedItem as IFilesystemMountBackend; + + private void ResetProbe() { + if (this._mountSession is not null) return; + this._driverProfile = null; + this._formatId = null; + this._sourceCanWrite = false; + this._detailsBox.Text = string.Empty; + this._statusLabel.Text = "Image changed; probe again."; + this.RefreshMountButton(); + } + + private void Probe() { + if (this._mountSession is not null || this._busy) return; + var path = this.ImagePath; + if (!File.Exists(path)) { + this.ShowProbeFailure("The selected image does not exist."); + return; + } + + try { + FormatRegistration.EnsureInitialized(); + var detected = FormatDetector.DetectByExtension(path); + if (detected == FormatDetector.Format.Unknown) { + this.ShowProbeFailure("No registered format could be detected for this file."); + return; + } + + var formatId = detected.ToString(); + var descriptor = FormatRegistry.GetById(formatId); + if (descriptor is null) { + this.ShowProbeFailure($"Detected format '{formatId}' has no registered descriptor."); + return; + } + + if (!FormatRegistry.FilesystemFormatIds.Contains(formatId, StringComparer.OrdinalIgnoreCase)) { + this.ShowProbeFailure( + $"'{descriptor.DisplayName}' is not registered as a filesystem image. " + + "Archive mounting belongs behind the synthetic archive namespace adapter; it is not filesystem writability." + ); + return; + } + + using var probeSource = OpenProbeSource(path, out var sourceCanWrite); + var profile = FormatRegistry.ProbeFilesystem(formatId, probeSource); + this._formatId = formatId; + this._driverProfile = profile; + this._sourceCanWrite = sourceCanWrite; + this._statusLabel.Text = $"Detected {descriptor.DisplayName}: {profile.ProfileName}."; + this.RefreshPlan(); + } catch (Exception ex) { + this.ShowProbeFailure($"Probe failed: {ex.GetType().Name}: {ex.Message}"); + } + } + + private void RefreshPlan() { + if (this._driverProfile is not { } profile || this._formatId is null) { + this.RefreshMountButton(); + return; + } + + var lines = new List { + $"Format: {profile.FormatId}", + $"Profile: {profile.ProfileName}", + $"Mutation model: {profile.MutationModel}", + $"Backing source writable: {this._sourceCanWrite}", + $"Can mount: {profile.CanMount}", + $"Can mount writable: {profile.CanMountWritable}", + $"Driver capabilities: {profile.Capabilities}", + }; + + if (profile.Limitations.Count > 0) { + lines.Add(string.Empty); + lines.Add("Filesystem limitations:"); + lines.AddRange(profile.Limitations.Select(static limitation => $"- {limitation}")); + } + + if (this.SelectedBackend is { } backend) { + var backendProfile = backend.GetProfile(); + var plan = this._mountBackends.ResolveFilesystem(backendProfile.Id, profile, this.AccessMode, this._sourceCanWrite); + lines.Add(string.Empty); + lines.Add($"Selected backend: {backendProfile.DisplayName}"); + lines.Add($"Selected access: {this.AccessMode}"); + lines.Add($"Supported: {plan.IsSupported}"); + if (plan.Reasons.Count > 0) { + lines.Add("Reasons:"); + lines.AddRange(plan.Reasons.Select(static reason => $"- {reason.Message}")); + } + if (plan.Limitations.Count > 0) { + lines.Add("Backend/profile limitations:"); + lines.AddRange(plan.Limitations.Select(static limitation => $"- {limitation}")); + } + } else { + lines.Add(string.Empty); + lines.Add("No mount backend is registered in this build yet."); + } + + if (this._mountLauncher is null) { + lines.Add(string.Empty); + lines.Add("No mount launcher is composed yet; probing and capability resolution are available, mounting stays disabled."); + } + + this._detailsBox.Text = string.Join(Environment.NewLine, lines); + this.RefreshMountButton(); + } + + private MountPlan? CurrentPlan() { + if (this._driverProfile is not { } profile || this.SelectedBackend is not { } backend) return null; + return this._mountBackends.ResolveFilesystem(backend.GetProfile().Id, profile, this.AccessMode, this._sourceCanWrite); + } + + private void RefreshMountButton() { + var plan = this.CurrentPlan(); + this._mountButton.Enabled = !this._busy + && this._mountSession is null + && this._mountLauncher is not null + && plan?.IsSupported == true + && !string.IsNullOrWhiteSpace(this._targetBox.Text); + this._unmountButton.Enabled = !this._busy && this._mountSession?.IsMounted == true; + } + + private async Task MountAsync() { + if (this._mountSession is not null || this._busy || this._mountLauncher is null) return; + var plan = this.CurrentPlan(); + var formatId = this._formatId; + var target = this._targetBox.Text.Trim(); + if (plan?.IsSupported != true || formatId is null || target.Length == 0) return; + + try { + this.SetBusy(true, $"Mounting at {target}..."); + this._mountSession = await this._mountLauncher.MountAsync(this.ImagePath, formatId, plan, target); + this._statusLabel.Text = $"Mounted at {this._mountSession.Target} via {this._mountSession.BackendId}."; + } catch (Exception ex) { + this._statusLabel.Text = $"Mount failed: {ex.GetType().Name}: {ex.Message}"; + } finally { + this.SetBusy(false, this._statusLabel.Text); + } + } + + private async Task UnmountAsync() { + if (this._mountSession is not { } session || this._busy) return; + try { + this.SetBusy(true, $"Unmounting {session.Target}..."); + if (session.IsMounted) await session.FlushAsync(); + if (session.IsMounted) await session.UnmountAsync(); + await session.DisposeAsync(); + this._mountSession = null; + this._statusLabel.Text = "Unmounted."; + } catch (Exception ex) { + this._statusLabel.Text = $"Unmount failed: {ex.GetType().Name}: {ex.Message}"; + } finally { + this.SetBusy(false, this._statusLabel.Text); + } + } + + private void CleanupActiveMount() { + if (this._mountSession is not { } session) return; + try { + if (session.IsMounted) session.FlushAsync().AsTask().GetAwaiter().GetResult(); + if (session.IsMounted) session.UnmountAsync().AsTask().GetAwaiter().GetResult(); + session.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } catch { + // Best-effort process teardown; forced-unmount policy remains backend-specific. + } finally { + this._mountSession = null; + } + } + + private void SetBusy(bool busy, string status) { + this._busy = busy; + this._probeButton.Enabled = !busy && this._mountSession is null; + this._imagePicker.Enabled = !busy && this._mountSession is null; + this._accessPicker.Enabled = !busy && this._mountSession is null; + this._backendPicker.Enabled = !busy && this._mountSession is null && this._backends.Length > 0; + this._targetBox.Enabled = !busy && this._mountSession is null; + this._statusLabel.Text = status; + this.RefreshMountButton(); + } + + private void ShowProbeFailure(string message) { + this._driverProfile = null; + this._formatId = null; + this._sourceCanWrite = false; + this._detailsBox.Text = message; + this._statusLabel.Text = message; + this.RefreshMountButton(); + } + + private static FileStream OpenProbeSource(string path, out bool canWrite) { + try { + var source = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + canWrite = true; + return source; + } catch (UnauthorizedAccessException) { + canWrite = false; + return new(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + } catch (IOException) { + canWrite = false; + return new(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + } + } +} diff --git a/Compression.NativeUI/Program.cs b/Compression.NativeUI/Program.cs new file mode 100644 index 000000000..52f2c9c22 --- /dev/null +++ b/Compression.NativeUI/Program.cs @@ -0,0 +1,23 @@ +using Compression.Lib; +using Compression.Mounting; +using Compression.Mounting.Dokan; +using Hawkynt.NativeForms; +using Hawkynt.NativeForms.Backends; +using Hawkynt.NativeForms.Backends.Gtk; +using Hawkynt.NativeForms.Backends.Windows; + +BackendRegistry.Register(new Win32Backend()); +BackendRegistry.Register(new GtkBackend()); + +FormatRegistration.EnsureInitialized(); + +var backends = new List(); +if (OperatingSystem.IsWindows()) { + var dokan = new DokanFilesystemMountBackend(); + if (dokan.RuntimeStatus.IsAvailable) + backends.Add(dokan); +} + +var mountBackends = new MountBackendRegistry(backends); +var launcher = new RegistryMountLauncher(new FilesystemMountLauncher(mountBackends)); +Application.Run(new MainForm(backends, launcher)); diff --git a/Compression.NativeUI/README.md b/Compression.NativeUI/README.md new file mode 100644 index 000000000..7ec6d345c --- /dev/null +++ b/Compression.NativeUI/README.md @@ -0,0 +1,26 @@ +# Compression.NativeUI + +Cross-platform NativeForms shell for CompressionWorkbench's mounting workflow. + +The existing WPF `Compression.UI` remains the full Windows workstation while this frontend grows screen by screen. Mounting starts here because it benefits immediately from one UI contract across the Win32 and GTK NativeForms backends instead of duplicating policy in WPF and a future Linux frontend. + +## Current scope + +- select a filesystem image; +- detect and probe the concrete filesystem profile through `FormatRegistry`; +- display per-image mountability, mutation model, driver capabilities, and limitations; +- choose read-only or read-write access; +- choose an injected mount backend and resolve support through `Compression.Mounting`; +- provide a mount target and own mount/unmount lifecycle once an `IMountLauncher` is composed. + +No filesystem mount backend or launcher is registered by this project yet. That is deliberate: the UI does not invent Dokan/FUSE availability. The Dokan/FUSE composition layer will inject real `IFilesystemMountBackend` implementations and an image/session opener after their dependency probes and callback bridges exist. + +Archive files are also rejected for now rather than being mislabeled as filesystem images. They will become mountable through the synthetic archive namespace adapter described in the repository `TODO.md`. + +## Run + +```text +dotnet run --project Compression.NativeUI/Compression.NativeUI.csproj +``` + +NativeForms currently provides working Win32 and GTK backends. Its Cocoa backend remains a placeholder, so this frontend intentionally registers Windows and GTK only. diff --git a/Compression.NativeUI/RegistryMountLauncher.cs b/Compression.NativeUI/RegistryMountLauncher.cs new file mode 100644 index 000000000..44623f87f --- /dev/null +++ b/Compression.NativeUI/RegistryMountLauncher.cs @@ -0,0 +1,15 @@ +using Compression.Mounting; + +namespace Compression.NativeUI; + +internal sealed class RegistryMountLauncher(FilesystemMountLauncher launcher) : IMountLauncher { + private readonly FilesystemMountLauncher _launcher = launcher ?? throw new ArgumentNullException(nameof(launcher)); + + public ValueTask MountAsync( + string imagePath, + string formatId, + MountPlan plan, + string target, + CancellationToken cancellationToken = default + ) => this._launcher.MountAsync(imagePath, formatId, plan, target, cancellationToken); +} diff --git a/Compression.Registry.Generator/FormatDescriptorGenerator.cs b/Compression.Registry.Generator/FormatDescriptorGenerator.cs index 2bf387baa..bf37d849c 100644 --- a/Compression.Registry.Generator/FormatDescriptorGenerator.cs +++ b/Compression.Registry.Generator/FormatDescriptorGenerator.cs @@ -8,23 +8,23 @@ namespace Compression.Registry.Generator; /// /// Roslyn incremental source generator that discovers all public non-abstract types implementing -/// IFormatDescriptor and IBuildingBlock across referenced assemblies and generates: +/// IFormatDescriptor, IBuildingBlock and IFilesystemDriverAdapter across referenced assemblies and generates: /// 1. A partial FormatRegistration class with explicit constructor calls (zero reflection) /// 2. A Format enum inside FormatDetector with all discovered format IDs plus special values +/// +/// Descriptors declared under a FileSystem.* namespace are explicitly marked as +/// filesystems when registered. FormatRegistry can therefore enforce that every +/// filesystem has a common-driver path without guessing from extensions/names. /// [Generator(LanguageNames.CSharp)] public sealed class FormatDescriptorGenerator : IIncrementalGenerator { private const string DescriptorInterfaceFullName = "Compression.Registry.IFormatDescriptor"; private const string BuildingBlockInterfaceFullName = "Compression.Registry.IBuildingBlock"; + private const string FilesystemDriverAdapterInterfaceFullName = "Compression.Registry.IFilesystemDriverAdapter"; - /// - /// Special enum members that are always generated even though they have no descriptor. - /// Unknown = no format detected; Sfx/Iso/Udf = detected by special logic, not a registry descriptor. - /// private static readonly string[] SpecialFormats = [ "Unknown", "Sfx", "Iso", "Udf", - // Compound tar formats — registered manually in FormatRegistration.RegisterCompoundTar() "TarGz", "TarBz2", "TarXz", "TarZst", "TarLz4", "TarLzip", "TarBr", ]; @@ -32,35 +32,55 @@ public void Initialize(IncrementalGeneratorInitializationContext context) { var allTypes = context.CompilationProvider.Select((compilation, ct) => { var descriptorInterface = compilation.GetTypeByMetadataName(DescriptorInterfaceFullName); var buildingBlockInterface = compilation.GetTypeByMetadataName(BuildingBlockInterfaceFullName); + var filesystemDriverAdapterInterface = compilation.GetTypeByMetadataName(FilesystemDriverAdapterInterfaceFullName); - var descriptors = new List<(string FullName, string Id)>(); + var descriptors = new List<(string FullName, string Id, bool IsFilesystem)>(); var buildingBlocks = new List(); + var filesystemDrivers = new List(); - if (descriptorInterface != null || buildingBlockInterface != null) { - // Check all referenced assemblies + if (descriptorInterface != null || buildingBlockInterface != null || filesystemDriverAdapterInterface != null) { foreach (var reference in compilation.References) { if (compilation.GetAssemblyOrModuleSymbol(reference) is not IAssemblySymbol assembly) continue; - FindTypes(assembly.GlobalNamespace, descriptorInterface, buildingBlockInterface, descriptors, buildingBlocks, ct); + FindTypes( + assembly.GlobalNamespace, + descriptorInterface, + buildingBlockInterface, + filesystemDriverAdapterInterface, + descriptors, + buildingBlocks, + filesystemDrivers, + ct); } - // Also check the compilation's own types - FindTypes(compilation.Assembly.GlobalNamespace, descriptorInterface, buildingBlockInterface, descriptors, buildingBlocks, ct); + FindTypes( + compilation.Assembly.GlobalNamespace, + descriptorInterface, + buildingBlockInterface, + filesystemDriverAdapterInterface, + descriptors, + buildingBlocks, + filesystemDrivers, + ct); } descriptors.Sort((a, b) => StringComparer.Ordinal.Compare(a.FullName, b.FullName)); buildingBlocks.Sort(StringComparer.Ordinal); - return (Descriptors: descriptors, BuildingBlocks: buildingBlocks); + filesystemDrivers.Sort(StringComparer.Ordinal); + return (Descriptors: descriptors, BuildingBlocks: buildingBlocks, FilesystemDrivers: filesystemDrivers); }); context.RegisterSourceOutput(allTypes, static (spc, types) => { - EmitFormatRegistration(spc, types.Descriptors, types.BuildingBlocks); + EmitFormatRegistration(spc, types.Descriptors, types.BuildingBlocks, types.FilesystemDrivers); EmitFormatEnum(spc, types.Descriptors); }); } - private static void EmitFormatRegistration(SourceProductionContext spc, - List<(string FullName, string Id)> descriptors, List buildingBlocks) { + private static void EmitFormatRegistration( + SourceProductionContext spc, + List<(string FullName, string Id, bool IsFilesystem)> descriptors, + List buildingBlocks, + List filesystemDrivers) { var sb = new StringBuilder(); sb.AppendLine("// "); sb.AppendLine("using Compression.Registry;"); @@ -69,22 +89,30 @@ private static void EmitFormatRegistration(SourceProductionContext spc, sb.AppendLine(); sb.AppendLine("public static partial class FormatRegistration {"); sb.AppendLine(" static partial void RegisterFormats() {"); - foreach (var (fullName, _) in descriptors) { - sb.AppendLine($" FormatRegistry.Register(new {fullName}());"); + foreach (var (fullName, _, isFilesystem) in descriptors) { + sb.AppendLine(isFilesystem + ? $" FormatRegistry.Register(new {fullName}(), isFilesystem: true);" + : $" FormatRegistry.Register(new {fullName}());"); } sb.AppendLine(" }"); sb.AppendLine(); sb.AppendLine(" static partial void RegisterBuildingBlocks() {"); - foreach (var fullName in buildingBlocks) { + foreach (var fullName in buildingBlocks) sb.AppendLine($" BuildingBlockRegistry.Register(new {fullName}());"); - } + sb.AppendLine(" }"); + sb.AppendLine(); + sb.AppendLine(" static partial void RegisterFilesystemDrivers() {"); + foreach (var fullName in filesystemDrivers) + sb.AppendLine($" FormatRegistry.RegisterFilesystemDriver(new {fullName}());"); sb.AppendLine(" }"); sb.AppendLine("}"); spc.AddSource("FormatRegistration.g.cs", sb.ToString()); } - private static void EmitFormatEnum(SourceProductionContext spc, List<(string FullName, string Id)> types) { + private static void EmitFormatEnum( + SourceProductionContext spc, + List<(string FullName, string Id, bool IsFilesystem)> types) { var sb = new StringBuilder(); sb.AppendLine("// "); sb.AppendLine(); @@ -93,18 +121,13 @@ private static void EmitFormatEnum(SourceProductionContext spc, List<(string Ful sb.AppendLine("public static partial class FormatDetector {"); sb.AppendLine(" public enum Format {"); - // Special values first - foreach (var special in SpecialFormats) { + foreach (var special in SpecialFormats) sb.AppendLine($" {special},"); - } - // All discovered descriptor IDs (excluding any that match special values) var specialSet = new HashSet(SpecialFormats); - foreach (var (_, id) in types) { - if (!specialSet.Contains(id)) { + foreach (var (_, id, _) in types) + if (!specialSet.Contains(id)) sb.AppendLine($" {id},"); - } - } sb.AppendLine(" }"); sb.AppendLine("}"); @@ -116,13 +139,23 @@ private static void FindTypes( INamespaceSymbol ns, INamedTypeSymbol? descriptorInterface, INamedTypeSymbol? buildingBlockInterface, - List<(string FullName, string Id)> descriptors, + INamedTypeSymbol? filesystemDriverAdapterInterface, + List<(string FullName, string Id, bool IsFilesystem)> descriptors, List buildingBlocks, + List filesystemDrivers, System.Threading.CancellationToken ct) { foreach (var member in ns.GetMembers()) { ct.ThrowIfCancellationRequested(); if (member is INamespaceSymbol childNs) { - FindTypes(childNs, descriptorInterface, buildingBlockInterface, descriptors, buildingBlocks, ct); + FindTypes( + childNs, + descriptorInterface, + buildingBlockInterface, + filesystemDriverAdapterInterface, + descriptors, + buildingBlocks, + filesystemDrivers, + ct); } else if (member is INamedTypeSymbol type && type.TypeKind == TypeKind.Class && !type.IsAbstract && @@ -132,22 +165,23 @@ private static void FindTypes( if (descriptorInterface != null && Implements(type, descriptorInterface)) { var id = DeriveId(type.Name, "FormatDescriptor"); - descriptors.Add((fullName, id)); + var nsName = type.ContainingNamespace?.ToDisplayString() ?? string.Empty; + var isFilesystem = nsName.Equals("FileSystem", StringComparison.Ordinal) || + nsName.StartsWith("FileSystem.", StringComparison.Ordinal); + descriptors.Add((fullName, id, isFilesystem)); } - if (buildingBlockInterface != null && Implements(type, buildingBlockInterface)) { + if (buildingBlockInterface != null && Implements(type, buildingBlockInterface)) buildingBlocks.Add(fullName); - } + + if (filesystemDriverAdapterInterface != null && Implements(type, filesystemDriverAdapterInterface)) + filesystemDrivers.Add(fullName); } } } - /// - /// Derives the format ID from the class name by stripping a suffix. - /// E.g., "GzipFormatDescriptor" → "Gzip", "SevenZipFormatDescriptor" → "SevenZip". - /// private static string DeriveId(string className, string suffix) { - if (className.EndsWith(suffix)) + if (className.EndsWith(suffix, StringComparison.Ordinal)) return className.Substring(0, className.Length - suffix.Length); return className; } @@ -163,10 +197,9 @@ private static bool HasParameterlessConstructor(INamedTypeSymbol type) { } private static bool Implements(INamedTypeSymbol type, INamedTypeSymbol iface) { - foreach (var i in type.AllInterfaces) { + foreach (var i in type.AllInterfaces) if (SymbolEqualityComparer.Default.Equals(i, iface)) return true; - } return false; } } diff --git a/Compression.Registry/BlockDeviceAdapters.cs b/Compression.Registry/BlockDeviceAdapters.cs new file mode 100644 index 000000000..bb1ca94c9 --- /dev/null +++ b/Compression.Registry/BlockDeviceAdapters.cs @@ -0,0 +1,282 @@ +#pragma warning disable CS1591 + +namespace Compression.Registry; + +/// +/// Compatibility alias for the original block-device provider name. New code +/// uses so containers, decoded +/// track media and raw images expose exactly one logical-block abstraction. +/// +[Obsolete("Use IRandomAccessBlockDeviceProvider; both names represent the same logical-block boundary.")] +public interface IBlockDeviceProvider : IRandomAccessBlockDeviceProvider { } + +/// +/// Optional filesystem-core capability for implementations whose native parser +/// already works directly on a block device. This is the long-term driver core: +/// the same filesystem implementation can mount raw disks, virtual disks, +/// forensic images, or decoded track media without container-specific code. +/// +public interface IBlockDeviceFilesystemDriverProvider { + FilesystemDriverProfile ProbeFilesystem(IRandomAccessBlockDevice device); + IFilesystemSession OpenFilesystem(IRandomAccessBlockDevice device, FilesystemOpenOptions options); +} + +/// +/// Fixed-size random-access block device over an ordinary seekable stream. This +/// is the bridge for raw filesystem images while parsers migrate away from +/// direct Stream.Position access. +/// +public sealed class StreamBlockDevice : IRandomAccessBlockDevice { + private readonly Stream _stream; + private readonly bool _leaveOpen; + private readonly object _gate = new(); + private bool _disposed; + + public StreamBlockDevice( + Stream stream, + int logicalBlockSize, + bool writable, + bool leaveOpen = true, + int? physicalBlockSize = null) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("A stream block device requires a readable, seekable stream.", nameof(stream)); + if (logicalBlockSize <= 0 || (logicalBlockSize & (logicalBlockSize - 1)) != 0) + throw new ArgumentOutOfRangeException(nameof(logicalBlockSize), "Logical block size must be a positive power of two."); + if (stream.Length % logicalBlockSize != 0) + throw new InvalidDataException( + $"Stream length {stream.Length:N0} is not an exact multiple of logical block size {logicalBlockSize:N0}."); + if (writable && !stream.CanWrite) + throw new ArgumentException("Writable block-device access requires a writable stream.", nameof(stream)); + + _stream = stream; + _leaveOpen = leaveOpen; + CanWrite = writable; + Geometry = new BlockDeviceGeometry( + logicalBlockSize, + stream.Length / logicalBlockSize, + physicalBlockSize.GetValueOrDefault(logicalBlockSize), + SupportsTrim: false); + } + + public BlockDeviceGeometry Geometry { get; } + public bool CanWrite { get; } + + public int ReadBlocks(long firstBlock, Span destination) { + ThrowIfDisposed(); + ValidateBuffer(firstBlock, destination.Length, nameof(destination)); + if (destination.Length == 0) return 0; + lock (_gate) { + _stream.Position = checked(firstBlock * Geometry.LogicalBlockSize); + _stream.ReadExactly(destination); + } + return destination.Length / Geometry.LogicalBlockSize; + } + + public void WriteBlocks(long firstBlock, ReadOnlySpan source) { + ThrowIfDisposed(); + if (!CanWrite) throw new NotSupportedException("The stream block device was opened read-only."); + ValidateBuffer(firstBlock, source.Length, nameof(source)); + if (source.Length == 0) return; + lock (_gate) { + _stream.Position = checked(firstBlock * Geometry.LogicalBlockSize); + _stream.Write(source); + } + } + + public void Trim(long firstBlock, long blockCount) { + ThrowIfDisposed(); + if (!CanWrite) throw new NotSupportedException("The stream block device was opened read-only."); + ValidateRange(firstBlock, blockCount); + throw new NotSupportedException("An ordinary stream has no portable deallocate/TRIM primitive."); + } + + public void Flush() { + ThrowIfDisposed(); + _stream.Flush(); + } + + public void Dispose() { + if (_disposed) return; + if (CanWrite) _stream.Flush(); + _disposed = true; + if (!_leaveOpen) _stream.Dispose(); + } + + private void ValidateBuffer(long firstBlock, int byteCount, string parameterName) { + if (byteCount < 0 || byteCount % Geometry.LogicalBlockSize != 0) + throw new ArgumentException("Block I/O buffers must contain a whole number of logical blocks.", parameterName); + ValidateRange(firstBlock, byteCount / Geometry.LogicalBlockSize); + } + + private void ValidateRange(long firstBlock, long blockCount) { + if (firstBlock < 0) throw new ArgumentOutOfRangeException(nameof(firstBlock)); + if (blockCount < 0) throw new ArgumentOutOfRangeException(nameof(blockCount)); + if (firstBlock > Geometry.BlockCount || blockCount > Geometry.BlockCount - firstBlock) + throw new ArgumentOutOfRangeException(nameof(firstBlock), "Block range extends beyond the device."); + } + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} + +/// +/// Seekable byte-stream view over a block device. Legacy filesystem parsers can +/// therefore run on VHD/QCOW2/GCR-backed logical disks before they are rewritten +/// to issue block requests directly. Unaligned writes use read-modify-write of +/// only the touched edge blocks; unrelated blocks are never rewritten. +/// +public sealed class BlockDeviceStream : Stream { + private readonly IRandomAccessBlockDevice _device; + private readonly bool _leaveOpen; + private readonly object _gate = new(); + private long _position; + private bool _disposed; + + public BlockDeviceStream(IRandomAccessBlockDevice device, bool leaveOpen = true) { + ArgumentNullException.ThrowIfNull(device); + _device = device; + _leaveOpen = leaveOpen; + } + + public override bool CanRead => !_disposed; + public override bool CanSeek => !_disposed; + public override bool CanWrite => !_disposed && _device.CanWrite; + public override long Length { + get { + ThrowIfDisposed(); + return _device.Geometry.Length; + } + } + + public override long Position { + get { + ThrowIfDisposed(); + return _position; + } + set { + ThrowIfDisposed(); + if (value < 0 || value > Length) throw new ArgumentOutOfRangeException(nameof(value)); + _position = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) { + ArgumentNullException.ThrowIfNull(buffer); + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) { + ThrowIfDisposed(); + if (buffer.Length == 0 || _position >= Length) return 0; + var count = checked((int)Math.Min(buffer.Length, Length - _position)); + ReadAt(_position, buffer[..count]); + _position += count; + return count; + } + + public override int ReadByte() { + Span one = stackalloc byte[1]; + return Read(one) == 0 ? -1 : one[0]; + } + + public override void Write(byte[] buffer, int offset, int count) { + ArgumentNullException.ThrowIfNull(buffer); + Write(buffer.AsSpan(offset, count)); + } + + public override void Write(ReadOnlySpan buffer) { + ThrowIfDisposed(); + if (!CanWrite) throw new NotSupportedException("The block-device stream is read-only."); + if (buffer.Length == 0) return; + if ((long)buffer.Length > Length - _position) + throw new IOException("Block-device streams have fixed length and cannot be extended."); + WriteAt(_position, buffer); + _position += buffer.Length; + } + + public override void WriteByte(byte value) { + Span one = stackalloc byte[1] { value }; + Write(one); + } + + public override long Seek(long offset, SeekOrigin origin) { + ThrowIfDisposed(); + long target; + try { + target = origin switch { + SeekOrigin.Begin => offset, + SeekOrigin.Current => checked(_position + offset), + SeekOrigin.End => checked(Length + offset), + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + } catch (OverflowException) { + throw new IOException("Seek target overflows the block-device address space."); + } + if (target < 0 || target > Length) throw new IOException("Seek target lies outside the fixed block device."); + return _position = target; + } + + public override void SetLength(long value) + => throw new NotSupportedException("Block-device streams have fixed geometry."); + + public override void Flush() { + ThrowIfDisposed(); + _device.Flush(); + } + + protected override void Dispose(bool disposing) { + if (!_disposed && disposing) { + if (_device.CanWrite) _device.Flush(); + if (!_leaveOpen) _device.Dispose(); + } + _disposed = true; + base.Dispose(disposing); + } + + private void ReadAt(long offset, Span destination) { + var blockSize = _device.Geometry.LogicalBlockSize; + lock (_gate) { + var cursor = 0; + var logicalOffset = offset; + while (cursor < destination.Length) { + var block = logicalOffset / blockSize; + var within = checked((int)(logicalOffset % blockSize)); + var take = Math.Min(destination.Length - cursor, blockSize - within); + if (within == 0 && take == blockSize) { + _device.ReadBlocks(block, destination.Slice(cursor, blockSize)); + } else { + var scratch = new byte[blockSize]; + _device.ReadBlocks(block, scratch); + scratch.AsSpan(within, take).CopyTo(destination.Slice(cursor, take)); + } + cursor += take; + logicalOffset += take; + } + } + } + + private void WriteAt(long offset, ReadOnlySpan source) { + var blockSize = _device.Geometry.LogicalBlockSize; + lock (_gate) { + var cursor = 0; + var logicalOffset = offset; + while (cursor < source.Length) { + var block = logicalOffset / blockSize; + var within = checked((int)(logicalOffset % blockSize)); + var take = Math.Min(source.Length - cursor, blockSize - within); + if (within == 0 && take == blockSize) { + _device.WriteBlocks(block, source.Slice(cursor, blockSize)); + } else { + var scratch = new byte[blockSize]; + _device.ReadBlocks(block, scratch); + source.Slice(cursor, take).CopyTo(scratch.AsSpan(within, take)); + _device.WriteBlocks(block, scratch); + } + cursor += take; + logicalOffset += take; + } + } + } + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} diff --git a/Compression.Registry/DefragOptions.cs b/Compression.Registry/DefragOptions.cs index 865c7ffc7..383001e52 100644 --- a/Compression.Registry/DefragOptions.cs +++ b/Compression.Registry/DefragOptions.cs @@ -100,14 +100,19 @@ public sealed record class DefragOptions { public long HoleAt { get; init; } = -1; /// - /// Optional progress callback. When non-null, the defragmenter emits at - /// least three events: a "scanning" event with the pre-defrag block map, - /// periodic "writing" events with read/write offsets during the rebuild, - /// and a "complete" event with the post-defrag block map. UI consumers - /// can render a live tile chart from these events. + /// Optional progress callback. When non-null, the defragmenter emits snapshots + /// and incremental read/write-head updates that can drive the maintenance block + /// map. Staged archive rebuilds use the same contract as native block movers. /// public Action? OnProgress { get; init; } + /// + /// Cooperative cancellation for long maintenance operations. Generic staged + /// rebuilds honour it while reading and writing and never commit a cancelled + /// target. Native in-place movers may honour it at their next safe move boundary. + /// + public CancellationToken CancellationToken { get; init; } + /// /// Layout profile for planner-driven defragmentation. Controls whether /// the defragmenter performs full zone-based rearrangement diff --git a/Compression.Registry/DefragProgress.cs b/Compression.Registry/DefragProgress.cs index 3b0ff79a1..3725f250c 100644 --- a/Compression.Registry/DefragProgress.cs +++ b/Compression.Registry/DefragProgress.cs @@ -2,7 +2,7 @@ namespace Compression.Registry; /// /// What kind of bytes a contiguous region holds. Used by the live-progress -/// block map to color-code regions as defrag proceeds. +/// block map to color-code regions as maintenance proceeds. /// public enum DefragBlockKind { /// Free space — not allocated to any file. @@ -11,34 +11,33 @@ public enum DefragBlockKind { Used, /// Marked bad / quarantined (FAT-style "BAD" cluster, or post-fsck flag). Bad, - /// Reserved for filesystem metadata (boot sectors, superblock, MFT, FAT, bitmap, root directory). + /// Reserved for filesystem/container metadata. MetadataReserved, - /// Currently being read or written by the in-progress defrag operation. + /// Currently being read, moved, compressed, grouped, or written. InProgress, } /// -/// Heuristic classification of a file's "thermal" zone based on its -/// modification time. Drives layout placement: hot at start, normal in -/// the middle, frozen near the end. Used by the live-progress block map -/// for tile coloring. +/// Heuristic classification used for the maintenance block-map colors. Filesystem +/// defraggers commonly map this to hot/cold placement; archive rebuilds may map it +/// to storage/compression classes so a staged target remains visually informative. /// public enum DefragBlockClass { - /// File modified recently (top quartile) — placed near start. + /// Hot / heavy-processing class. Hot, - /// File modified normally — placed in the middle. + /// Normal class. Normal, - /// File modified a while ago — placed near end. + /// Cold / alternative-processing class. Cold, - /// File hasn't been touched in a long time (bottom quartile) — placed at end. + /// Frozen / stored-verbatim class. Frozen, - /// Directory metadata (folder contents, B-tree dir node, etc.) — rendered gold to make placement visible. + /// Directory or structural metadata class. Directory, } /// -/// One contiguous region of an image's address space, as seen by the -/// live-progress block map. +/// One contiguous region in the address-space currently visualized by the +/// maintenance block map. /// public sealed record DefragBlockInfo( long Offset, @@ -48,18 +47,30 @@ public sealed record DefragBlockInfo( DefragBlockClass? Classification = null); /// -/// Snapshot of an image's block layout at a moment in time. Emitted by -/// -/// implementations through DefragOptions.OnProgress at scan -/// start, periodically during writes, and at completion. +/// Snapshot emitted by defrag/re-layout/rebuild maintenance operations. +/// Native in-place movers normally report one physical image address-space. +/// Transactional WORM/archive rebuilds may report the source read head and staged +/// target write head in their respective byte-spaces, projected onto the same +/// chart. In that mode the two head offsets are progress visualization and do not +/// assert that identical numerical offsets refer to the same physical bytes. /// -/// Progress phase identifier ("scanning" / "writing" / "complete" / "error"). +/// +/// Progress phase identifier. Common values are scanning, reading, +/// writing, verifying, staged, committing, +/// complete, and error. +/// /// 0..1 fraction of work done. -1 = indeterminate. -/// Byte offset currently being read; -1 if not reading. -/// Byte offset currently being written; -1 if not writing. -/// Total image size in bytes (helpful for tile binning). -/// Block-map snapshot, present at scan start + completion. Null during incremental updates. -/// Optional human-readable status (e.g. "moving extent 23 of 87"). +/// Current source/read offset; -1 when not reading. +/// Current destination/write offset; -1 when not writing. +/// +/// Address-space size used for visualization/binning. For staged rebuilds this is +/// a display scale large enough to project the source and target progress. +/// +/// +/// Optional block-map snapshot. Null incremental events retain the previous map +/// and only move heads/progress, keeping redraw cost low on large archives. +/// +/// Optional human-readable phase/status text. public sealed record DefragProgressEvent( string Phase, double Fraction, diff --git a/Compression.Registry/FilesystemDriverContracts.cs b/Compression.Registry/FilesystemDriverContracts.cs new file mode 100644 index 000000000..77dea6de0 --- /dev/null +++ b/Compression.Registry/FilesystemDriverContracts.cs @@ -0,0 +1,228 @@ +#pragma warning disable CS1591 +namespace Compression.Registry; + +/// +/// Stable, path-independent identity of a filesystem object. A real driver must +/// not use a pathname as identity: rename/unlink can change names while open +/// handles keep referring to the same inode/object. Providers map their native +/// inode, file-reference, object-id or directory-slot identity into these two +/// opaque 64-bit words. +/// +public readonly record struct FilesystemNodeId(ulong Value, ulong Generation = 0); + +public enum FilesystemNodeKind { + Unknown, + RegularFile, + Directory, + SymbolicLink, + BlockDevice, + CharacterDevice, + Fifo, + Socket, +} + +[Flags] +public enum FilesystemDriverCapabilities : ulong { + None = 0, + EnumerateDirectories = 1UL << 0, + ReadData = 1UL << 1, + RandomAccess = 1UL << 2, + StableNodeIds = 1UL << 3, + WriteData = 1UL << 4, + Truncate = 1UL << 5, + CreateFile = 1UL << 6, + DeleteFile = 1UL << 7, + CreateDirectory = 1UL << 8, + RemoveDirectory = 1UL << 9, + Rename = 1UL << 10, + HardLinks = 1UL << 11, + SymbolicLinks = 1UL << 12, + SetMetadata = 1UL << 13, + SparseFiles = 1UL << 14, + Flush = 1UL << 15, + Transactions = 1UL << 16, + CaseSensitiveNames = 1UL << 17, + CasePreservingNames = 1UL << 18, +} + +/// +/// Describes how namespace/data writes become durable on this exact on-disk +/// profile. This is intentionally separate from : +/// archive-level Add/Remove may legitimately rebuild a whole image, while a +/// writable mounted filesystem driver needs bounded, handle-safe mutations. +/// +public enum FilesystemMutationModel { + None, + Direct, + Journaled, + CopyOnWrite, + LogStructured, + WholeImageRebuild, +} + +/// +/// Per-image probe result. Capabilities are not assumed from the format name: +/// an EROFS flat profile, a compressed EROFS profile, a damaged FAT image, or a +/// ReFS version with an unsupported metadata feature can have different safe +/// operations even though they share one descriptor. +/// +public sealed record FilesystemDriverProfile( + string FormatId, + string ProfileName, + FilesystemDriverCapabilities Capabilities, + FilesystemMutationModel MutationModel, + bool CanMount, + bool CanMountWritable, + IReadOnlyList Limitations +); + +public sealed record FilesystemOpenOptions( + bool ReadOnly = true, + bool LeaveOpen = true +); + +public sealed record FilesystemDirectoryEntry( + string Name, + FilesystemNodeId NodeId, + FilesystemNodeKind Kind +); + +public sealed record FilesystemNodeInfo( + FilesystemNodeId NodeId, + FilesystemNodeKind Kind, + long Size, + long AllocatedSize, + uint LinkCount = 1, + ulong NativeAttributes = 0, + DateTimeOffset? Created = null, + DateTimeOffset? Modified = null, + DateTimeOffset? Accessed = null, + DateTimeOffset? Changed = null +); + +/// Optional metadata changes; null means leave the field unchanged. +public sealed record FilesystemMetadataPatch( + DateTimeOffset? Created = null, + DateTimeOffset? Modified = null, + DateTimeOffset? Accessed = null, + ulong? NativeAttributes = null +); + +/// +/// Descriptor-side entry point for a mount-grade filesystem implementation. +/// Probe must be non-destructive and fail closed. Open must reject writable mode +/// unless the returned profile has . +/// +public interface IFilesystemDriverProvider { + FilesystemDriverProfile ProbeFilesystem(Stream image); + IFilesystemSession OpenFilesystem(Stream image, FilesystemOpenOptions options); +} + +/// +/// Open filesystem namespace. Operations use stable node ids rather than paths, +/// mirroring the semantics required by FUSE/Dokany/WinFsp-style adapters: a +/// caller may keep a file handle open across rename or unlink. +/// +public interface IFilesystemSession : IDisposable { + FilesystemDriverProfile Profile { get; } + FilesystemNodeId RootNodeId { get; } + + FilesystemNodeInfo Stat(FilesystemNodeId nodeId); + FilesystemNodeId? Lookup(FilesystemNodeId parentDirectory, string name); + IReadOnlyList Enumerate(FilesystemNodeId directory); + IFilesystemFileHandle OpenFile(FilesystemNodeId nodeId, FileAccess access); + + FilesystemNodeId CreateFile(FilesystemNodeId parentDirectory, string name); + FilesystemNodeId CreateDirectory(FilesystemNodeId parentDirectory, string name); + void DeleteFile(FilesystemNodeId parentDirectory, string name); + void RemoveDirectory(FilesystemNodeId parentDirectory, string name); + void Rename(FilesystemNodeId oldParent, string oldName, FilesystemNodeId newParent, string newName, bool replace); + void CreateHardLink(FilesystemNodeId existingNode, FilesystemNodeId newParent, string newName); + FilesystemNodeId CreateSymbolicLink(FilesystemNodeId parentDirectory, string name, string target); + string ReadSymbolicLink(FilesystemNodeId nodeId); + void SetMetadata(FilesystemNodeId nodeId, FilesystemMetadataPatch patch); + + /// Flushes all dirty data and metadata that are not inside an active transaction. + void Flush(); + + /// + /// Begins one durability transaction for this session. Until Commit/Rollback, + /// namespace operations and writes through handles opened by the session belong + /// to that transaction. Providers that do not advertise Transactions throw. + /// + IFilesystemTransaction BeginTransaction(); +} + +/// +/// Positional file handle. It deliberately has no shared Stream.Position so two +/// concurrent kernel requests cannot race a mutable cursor. Reads/writes operate +/// at explicit logical offsets and therefore map naturally to filesystem extents. +/// +public interface IFilesystemFileHandle : IDisposable { + FilesystemNodeId NodeId { get; } + long Length { get; } + int Read(long offset, Span destination); + void Write(long offset, ReadOnlySpan source); + void SetLength(long length); + void Flush(); +} + +public interface IFilesystemTransaction : IDisposable { + bool IsCompleted { get; } + void Commit(); + void Rollback(); +} + +/// +/// Geometry of a sector/block-addressable device exposed beneath a filesystem. +/// Container formats such as VHD/QCOW2/EWF should eventually implement this +/// layer; FAT/ext/ReFS drivers then consume block devices rather than knowing +/// how their outer container stores bytes. +/// +public sealed record BlockDeviceGeometry( + int LogicalBlockSize, + long BlockCount, + int PhysicalBlockSize = 0, + bool SupportsTrim = false +) { + public long Length => checked(BlockCount * LogicalBlockSize); +} + +public interface IRandomAccessBlockDevice : IDisposable { + BlockDeviceGeometry Geometry { get; } + bool CanWrite { get; } + int ReadBlocks(long firstBlock, Span destination); + void WriteBlocks(long firstBlock, ReadOnlySpan source); + void Trim(long firstBlock, long blockCount); + void Flush(); +} + +/// +/// Raw variable-length track device for flux/GCR/MFM-style containers that are +/// not yet sector-addressable. G64 belongs here; a decoder can later project it +/// as for a Commodore filesystem driver. +/// +public sealed record RawTrackInfo( + int Index, + long Length, + uint EncodingParameter = 0, + bool IsPresent = true +); + +public interface IRawTrackDevice : IDisposable { + int TrackCount { get; } + bool CanWrite { get; } + IReadOnlyList EnumerateTracks(); + int ReadTrack(int index, Span destination); + void WriteTrack(int index, ReadOnlySpan source, uint? encodingParameter = null); + void ClearTrack(int index); + void Flush(); +} + +/// +/// Optional descriptor capability for opening the raw-track layer directly. +/// This keeps track-container mutation separate from filesystem namespace CRUD. +/// +public interface IRawTrackDeviceProvider { + IRawTrackDevice OpenRawTrackDevice(Stream image, bool writable, bool leaveOpen = true); +} diff --git a/Compression.Registry/FilesystemDriverCoverage.cs b/Compression.Registry/FilesystemDriverCoverage.cs new file mode 100644 index 000000000..7b0bc26e1 --- /dev/null +++ b/Compression.Registry/FilesystemDriverCoverage.cs @@ -0,0 +1,35 @@ +#pragma warning disable CS1591 +namespace Compression.Registry; + +/// +/// Structural binding used to reach the common filesystem-driver contract. +/// This deliberately says nothing about the exact image profile: probing an +/// image can still refuse a damaged/unsupported feature set. +/// +public enum FilesystemDriverBindingKind { + None, + ArchiveProjection, + SidecarNative, + DescriptorNative, +} + +/// +/// Machine-readable repository coverage for one FileSystem.* descriptor. +/// It answers whether the implementation has a path to an IFilesystemSession +/// and which lower-level primitives are already available for finishing a +/// native read/write driver. +/// +public sealed record FilesystemDriverCoverage( + string FormatId, + string DisplayName, + FilesystemDriverBindingKind Binding, + bool HasArchiveProjection, + bool HasArchiveMutation, + bool HasExtentMap, + bool HasBlockMover, + bool HasBlockDeviceProvider, + bool HasNativeReadinessProvider +) { + public bool HasDriverPath => Binding != FilesystemDriverBindingKind.None; + public bool IsNative => Binding is FilesystemDriverBindingKind.DescriptorNative or FilesystemDriverBindingKind.SidecarNative; +} diff --git a/Compression.Registry/FilesystemDriverDerivation.cs b/Compression.Registry/FilesystemDriverDerivation.cs new file mode 100644 index 000000000..1b483a824 --- /dev/null +++ b/Compression.Registry/FilesystemDriverDerivation.cs @@ -0,0 +1,597 @@ +#pragma warning disable CS1591 + +namespace Compression.Registry; + +[Flags] +public enum FilesystemDriverReadinessLayer : ulong { + None = 0, + ImageValidation = 1UL << 0, + Namespace = 1UL << 1, + SessionStableNodeIds = 1UL << 2, + NativeStableNodeIds = 1UL << 3, + ReadData = 1UL << 4, + RandomAccessRead = 1UL << 5, + AllocationMap = 1UL << 6, + WriteData = 1UL << 7, + Truncate = 1UL << 8, + NamespaceMutation = 1UL << 9, + MetadataMutation = 1UL << 10, + Links = 1UL << 11, + Flush = 1UL << 12, + DurabilityModel = 1UL << 13, + Recovery = 1UL << 14, + Concurrency = 1UL << 15, + ValidationCorpus = 1UL << 16, +} + +public enum FilesystemDriverTarget { + ReadOnly, + ReadWrite, +} + +public sealed record FilesystemDriverReadinessReport( + string FormatId, + FilesystemDriverTarget Target, + FilesystemDriverReadinessLayer AvailableLayers, + FilesystemDriverReadinessLayer RequiredLayers, + bool Derivable, + bool UsesNativeProvider, + IReadOnlyList Blockers +); + +/// +/// Optional filesystem-specific readiness description. The generic derivation +/// layer supplies a conservative report when a descriptor does not implement +/// this interface; native implementations can use it to explain exactly which +/// on-disk semantics still block a complete mounted driver. +/// +public interface IFilesystemDriverReadinessProvider { + FilesystemDriverReadinessReport DescribeFilesystemDriverReadiness( + Stream image, + FilesystemDriverTarget target); +} + +/// +/// Common entry point for filesystem frontends. Native filesystem providers are +/// always preferred. A descriptor that only exposes the normalized archive +/// listing/open-entry surface still gets a real read-only filesystem session: +/// hierarchy is reconstructed, node ids remain stable for the lifetime of the +/// mount, symlinks are represented, and file handles use positional reads. +/// +/// The fallback is deliberately read-only. It never turns archive-level +/// rebuild/Add/Remove support into mounted write support. This makes every +/// filesystem parser usable by FUSE/Dokany/WinFsp-style frontends immediately, +/// while leaving a precise upgrade path to native allocation and mutation code. +/// +public static class FilesystemDriverDerivation { + private const FilesystemDriverReadinessLayer ReadOnlyRequired = + FilesystemDriverReadinessLayer.ImageValidation | + FilesystemDriverReadinessLayer.Namespace | + FilesystemDriverReadinessLayer.SessionStableNodeIds | + FilesystemDriverReadinessLayer.ReadData | + FilesystemDriverReadinessLayer.RandomAccessRead; + + private const FilesystemDriverReadinessLayer ReadWriteRequired = + ReadOnlyRequired | + FilesystemDriverReadinessLayer.AllocationMap | + FilesystemDriverReadinessLayer.WriteData | + FilesystemDriverReadinessLayer.Truncate | + FilesystemDriverReadinessLayer.NamespaceMutation | + FilesystemDriverReadinessLayer.Flush | + FilesystemDriverReadinessLayer.DurabilityModel | + FilesystemDriverReadinessLayer.Concurrency; + + public static FilesystemDriverProfile Probe( + IFormatDescriptor descriptor, + Stream image, + string? password = null) { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(image); + + if (descriptor is IFilesystemDriverProvider native) + return native.ProbeFilesystem(image); + + if (descriptor is not IArchiveFormatOperations archive) + return new FilesystemDriverProfile( + descriptor.Id, + "no filesystem projection", + FilesystemDriverCapabilities.None, + FilesystemMutationModel.None, + CanMount: false, + CanMountWritable: false, + ["Descriptor exposes neither IFilesystemDriverProvider nor IArchiveFormatOperations."]); + + try { + using var snapshot = DerivedFilesystemSnapshot.Capture(image); + using var probe = snapshot.OpenRead(); + _ = archive.List(probe, password); + return DerivedReadOnlyProfile(descriptor.Id); + } catch (Exception e) when (e is InvalidDataException or NotSupportedException or IOException or ArgumentException) { + return new FilesystemDriverProfile( + descriptor.Id, + "archive-view probe failed", + FilesystemDriverCapabilities.None, + FilesystemMutationModel.None, + CanMount: false, + CanMountWritable: false, + [$"Filesystem projection could not enumerate this image: {FirstLine(e.Message)}"]); + } + } + + public static IFilesystemSession Open( + IFormatDescriptor descriptor, + Stream image, + FilesystemOpenOptions options, + string? password = null) { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(options); + + if (descriptor is IFilesystemDriverProvider native) + return native.OpenFilesystem(image, options); + + if (!options.ReadOnly) + throw new NotSupportedException( + $"{descriptor.Id} has no native writable filesystem provider. " + + "Archive-level rebuild/modify support is not a mount-grade write path."); + + if (descriptor is not IArchiveFormatOperations archive) + throw new NotSupportedException( + $"{descriptor.Id} exposes no filesystem-driver provider and no list/open-entry projection."); + + return new DerivedReadOnlyFilesystemSession(descriptor, archive, image, password); + } + + public static FilesystemDriverReadinessReport Assess( + IFormatDescriptor descriptor, + Stream image, + FilesystemDriverTarget target, + string? password = null) { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(image); + + if (descriptor is IFilesystemDriverReadinessProvider specific) + return specific.DescribeFilesystemDriverReadiness(image, target); + + var required = target == FilesystemDriverTarget.ReadOnly ? ReadOnlyRequired : ReadWriteRequired; + var blockers = new List(); + FilesystemDriverReadinessLayer available = FilesystemDriverReadinessLayer.None; + var usesNative = descriptor is IFilesystemDriverProvider; + + if (usesNative) { + var profile = ((IFilesystemDriverProvider)descriptor).ProbeFilesystem(image); + if (profile.CanMount) { + available |= FilesystemDriverReadinessLayer.ImageValidation | + FilesystemDriverReadinessLayer.Namespace | + FilesystemDriverReadinessLayer.SessionStableNodeIds; + available |= LayersFor(profile.Capabilities); + } + if (profile.MutationModel != FilesystemMutationModel.None && + profile.MutationModel != FilesystemMutationModel.WholeImageRebuild) + available |= FilesystemDriverReadinessLayer.DurabilityModel; + blockers.AddRange(profile.Limitations); + } else if (descriptor is IArchiveFormatOperations archive) { + try { + using var snapshot = DerivedFilesystemSnapshot.Capture(image); + using var source = snapshot.OpenRead(); + _ = archive.List(source, password); + available |= ReadOnlyRequired; + blockers.Add("Node ids are stable only within the derived session; native on-disk object identity is not exposed yet."); + blockers.Add("Allocation/extents are not exposed through the generic archive projection."); + } catch (Exception e) when (e is InvalidDataException or NotSupportedException or IOException or ArgumentException) { + blockers.Add($"Current parser cannot enumerate this image: {FirstLine(e.Message)}"); + } + } else { + blockers.Add("Descriptor does not expose IArchiveFormatOperations or a native filesystem provider."); + } + + if (target == FilesystemDriverTarget.ReadWrite && (available & ReadWriteRequired) != ReadWriteRequired) { + if ((available & FilesystemDriverReadinessLayer.AllocationMap) == 0) + blockers.Add("Expose native allocation/free-space ownership instead of inferring it from extracted files."); + if ((available & FilesystemDriverReadinessLayer.WriteData) == 0) + blockers.Add("Implement bounded native file-data writes through the filesystem allocator."); + if ((available & FilesystemDriverReadinessLayer.NamespaceMutation) == 0) + blockers.Add("Implement native create/unlink/rename namespace mutation."); + if ((available & FilesystemDriverReadinessLayer.DurabilityModel) == 0) + blockers.Add("Model the filesystem's real write-ordering/journal/CoW durability boundary."); + if ((available & FilesystemDriverReadinessLayer.Concurrency) == 0) + blockers.Add("Define handle/cache/locking behavior for concurrent frontend requests."); + } + + var derivable = (available & required) == required; + return new FilesystemDriverReadinessReport( + descriptor.Id, + target, + available, + required, + derivable, + usesNative, + blockers.Distinct(StringComparer.Ordinal).ToArray()); + } + + private static FilesystemDriverProfile DerivedReadOnlyProfile(string formatId) + => new( + formatId, + "derived read-only archive view", + FilesystemDriverCapabilities.EnumerateDirectories | + FilesystemDriverCapabilities.ReadData | + FilesystemDriverCapabilities.RandomAccess | + FilesystemDriverCapabilities.StableNodeIds | + FilesystemDriverCapabilities.CasePreservingNames, + FilesystemMutationModel.None, + CanMount: true, + CanMountWritable: false, + [ + "Read-only compatibility projection over List/OpenEntry; allocation metadata is not exposed.", + "Node ids are deterministic and stable for this session, not claimed to be native inode/object identifiers.", + ]); + + private static FilesystemDriverReadinessLayer LayersFor(FilesystemDriverCapabilities capabilities) { + var result = FilesystemDriverReadinessLayer.None; + if ((capabilities & FilesystemDriverCapabilities.EnumerateDirectories) != 0) + result |= FilesystemDriverReadinessLayer.Namespace; + if ((capabilities & FilesystemDriverCapabilities.StableNodeIds) != 0) + result |= FilesystemDriverReadinessLayer.SessionStableNodeIds | + FilesystemDriverReadinessLayer.NativeStableNodeIds; + if ((capabilities & FilesystemDriverCapabilities.ReadData) != 0) + result |= FilesystemDriverReadinessLayer.ReadData; + if ((capabilities & FilesystemDriverCapabilities.RandomAccess) != 0) + result |= FilesystemDriverReadinessLayer.RandomAccessRead; + if ((capabilities & FilesystemDriverCapabilities.WriteData) != 0) + result |= FilesystemDriverReadinessLayer.WriteData; + if ((capabilities & FilesystemDriverCapabilities.Truncate) != 0) + result |= FilesystemDriverReadinessLayer.Truncate; + if ((capabilities & (FilesystemDriverCapabilities.CreateFile | + FilesystemDriverCapabilities.DeleteFile | + FilesystemDriverCapabilities.CreateDirectory | + FilesystemDriverCapabilities.RemoveDirectory | + FilesystemDriverCapabilities.Rename)) != 0) + result |= FilesystemDriverReadinessLayer.NamespaceMutation; + if ((capabilities & FilesystemDriverCapabilities.SetMetadata) != 0) + result |= FilesystemDriverReadinessLayer.MetadataMutation; + if ((capabilities & (FilesystemDriverCapabilities.HardLinks | + FilesystemDriverCapabilities.SymbolicLinks)) != 0) + result |= FilesystemDriverReadinessLayer.Links; + if ((capabilities & FilesystemDriverCapabilities.Flush) != 0) + result |= FilesystemDriverReadinessLayer.Flush; + return result; + } + + private static string FirstLine(string message) { + var index = message.IndexOfAny(['\r', '\n']); + return index < 0 ? message : message[..index]; + } +} + +internal sealed class DerivedReadOnlyFilesystemSession : IFilesystemSession { + private readonly IArchiveFormatOperations _operations; + private readonly DerivedFilesystemSnapshot _snapshot; + private readonly string? _password; + private readonly Dictionary _nodes = []; + private readonly Dictionary> _children = []; + private bool _disposed; + + private sealed class Node { + public required FilesystemNodeId Id { get; init; } + public required string Name { get; init; } + public required string Path { get; init; } + public required FilesystemNodeId Parent { get; init; } + public required FilesystemNodeKind Kind { get; set; } + public ArchiveEntryInfo? Entry { get; set; } + } + + public DerivedReadOnlyFilesystemSession( + IFormatDescriptor descriptor, + IArchiveFormatOperations operations, + Stream image, + string? password) { + _operations = operations; + _password = password; + _snapshot = DerivedFilesystemSnapshot.Capture(image); + Profile = new FilesystemDriverProfile( + descriptor.Id, + "derived read-only archive view", + FilesystemDriverCapabilities.EnumerateDirectories | + FilesystemDriverCapabilities.ReadData | + FilesystemDriverCapabilities.RandomAccess | + FilesystemDriverCapabilities.StableNodeIds | + FilesystemDriverCapabilities.SymbolicLinks | + FilesystemDriverCapabilities.CasePreservingNames, + FilesystemMutationModel.None, + CanMount: true, + CanMountWritable: false, + [ + "Compatibility filesystem projection; native allocation/extents are not exposed.", + "Writes require a native IFilesystemDriverProvider and are never emulated by rebuilding the image.", + ]); + + BuildNamespace(); + } + + public FilesystemDriverProfile Profile { get; } + public FilesystemNodeId RootNodeId { get; } = new(1, 1); + + public FilesystemNodeInfo Stat(FilesystemNodeId nodeId) { + ThrowIfDisposed(); + var node = RequireNode(nodeId); + var entry = node.Entry; + var logical = node.Kind == FilesystemNodeKind.Directory ? 0 : Math.Max(0, entry?.OriginalSize ?? 0); + var allocated = node.Kind == FilesystemNodeKind.Directory + ? 0 + : entry?.CompressedSize is >= 0 ? entry.CompressedSize : logical; + return new FilesystemNodeInfo( + node.Id, + node.Kind, + logical, + Math.Max(0, allocated), + LinkCount: 1, + Modified: ToOffset(entry?.LastModified)); + } + + public FilesystemNodeId? Lookup(FilesystemNodeId parentDirectory, string name) { + ArgumentNullException.ThrowIfNull(name); + ThrowIfDisposed(); + var parent = RequireNode(parentDirectory); + if (parent.Kind != FilesystemNodeKind.Directory) + throw new DirectoryNotFoundException(parent.Path); + if (!_children.TryGetValue(parentDirectory, out var children)) return null; + + var exact = children.FirstOrDefault(child => string.Equals(child.Name, name, StringComparison.Ordinal)); + if (exact != null) return exact.Id; + var folded = children.Where(child => string.Equals(child.Name, name, StringComparison.OrdinalIgnoreCase)).ToArray(); + return folded.Length == 1 ? folded[0].Id : null; + } + + public IReadOnlyList Enumerate(FilesystemNodeId directory) { + ThrowIfDisposed(); + var parent = RequireNode(directory); + if (parent.Kind != FilesystemNodeKind.Directory) + throw new DirectoryNotFoundException(parent.Path); + return (_children.TryGetValue(directory, out var children) ? children : []) + .OrderBy(child => child.Name, StringComparer.Ordinal) + .Select(child => new FilesystemDirectoryEntry(child.Name, child.Id, child.Kind)) + .ToArray(); + } + + public IFilesystemFileHandle OpenFile(FilesystemNodeId nodeId, FileAccess access) { + ThrowIfDisposed(); + if (access != FileAccess.Read) + throw new NotSupportedException("The derived filesystem projection is read-only."); + var node = RequireNode(nodeId); + if (node.Kind != FilesystemNodeKind.RegularFile) + throw new UnauthorizedAccessException($"'{node.Path}' is not a regular file."); + if (node.Entry == null) + throw new InvalidDataException($"Derived node '{node.Path}' has no backing archive entry."); + + var backing = node.Entry; + return SpoolingReadOnlyFileHandle.Create( + node.Id, + Math.Max(0, backing.OriginalSize), + output => { + using var archive = _snapshot.OpenRead(); + using var entry = _operations.OpenEntry(archive, backing.Name, _password); + entry.CopyTo(output); + }); + } + + public FilesystemNodeId CreateFile(FilesystemNodeId parentDirectory, string name) + => throw ReadOnly(); + public FilesystemNodeId CreateDirectory(FilesystemNodeId parentDirectory, string name) + => throw ReadOnly(); + public void DeleteFile(FilesystemNodeId parentDirectory, string name) + => throw ReadOnly(); + public void RemoveDirectory(FilesystemNodeId parentDirectory, string name) + => throw ReadOnly(); + public void Rename(FilesystemNodeId oldParent, string oldName, FilesystemNodeId newParent, string newName, bool replace) + => throw ReadOnly(); + public void CreateHardLink(FilesystemNodeId existingNode, FilesystemNodeId newParent, string newName) + => throw ReadOnly(); + public FilesystemNodeId CreateSymbolicLink(FilesystemNodeId parentDirectory, string name, string target) + => throw ReadOnly(); + + public string ReadSymbolicLink(FilesystemNodeId nodeId) { + ThrowIfDisposed(); + var node = RequireNode(nodeId); + if (node.Kind != FilesystemNodeKind.SymbolicLink || node.Entry?.LinkTarget == null) + throw new InvalidOperationException($"'{node.Path}' is not a symbolic link with a decoded target."); + return node.Entry.LinkTarget; + } + + public void SetMetadata(FilesystemNodeId nodeId, FilesystemMetadataPatch patch) + => throw ReadOnly(); + + public void Flush() { + ThrowIfDisposed(); + } + + public IFilesystemTransaction BeginTransaction() + => throw new NotSupportedException("The derived read-only filesystem projection has no transactions."); + + public void Dispose() { + if (_disposed) return; + _disposed = true; + _snapshot.Dispose(); + } + + private void BuildNamespace() { + var root = new Node { + Id = RootNodeId, + Name = string.Empty, + Path = string.Empty, + Parent = default, + Kind = FilesystemNodeKind.Directory, + }; + _nodes[root.Id] = root; + _children[root.Id] = []; + + List entries; + using (var source = _snapshot.OpenRead()) + entries = _operations.List(source, _password); + + var normalized = entries + .Select(entry => (Entry: entry, Path: NormalizePath(entry.Name))) + .Where(item => item.Path.Length > 0) + .OrderBy(item => item.Path, StringComparer.Ordinal) + .ToArray(); + + var byPath = new Dictionary(StringComparer.Ordinal) { [string.Empty] = root }; + ulong nextId = 2; + + foreach (var item in normalized) { + var segments = item.Path.Split('/'); + var currentPath = string.Empty; + var parent = root; + for (var i = 0; i < segments.Length; ++i) { + var segment = segments[i]; + currentPath = currentPath.Length == 0 ? segment : currentPath + "/" + segment; + var isLeaf = i == segments.Length - 1; + if (!byPath.TryGetValue(currentPath, out var node)) { + node = new Node { + Id = new FilesystemNodeId(nextId++, 1), + Name = segment, + Path = currentPath, + Parent = parent.Id, + Kind = isLeaf ? KindFor(item.Entry) : FilesystemNodeKind.Directory, + Entry = isLeaf ? item.Entry : null, + }; + byPath[currentPath] = node; + _nodes[node.Id] = node; + _children.TryAdd(node.Id, []); + _children[parent.Id].Add(node); + } else if (isLeaf) { + if (node.Entry != null) + throw new InvalidDataException($"Filesystem projection contains duplicate path '{currentPath}'."); + if (node.Kind != FilesystemNodeKind.Directory && item.Entry.IsDirectory) + throw new InvalidDataException($"Filesystem projection path '{currentPath}' changes node kind."); + node.Entry = item.Entry; + node.Kind = KindFor(item.Entry); + } + parent = node; + } + } + } + + private Node RequireNode(FilesystemNodeId nodeId) + => _nodes.TryGetValue(nodeId, out var node) + ? node + : throw new FileNotFoundException($"Filesystem node {nodeId.Value}:{nodeId.Generation} is not present in this session."); + + private static FilesystemNodeKind KindFor(ArchiveEntryInfo entry) + => entry.IsDirectory ? FilesystemNodeKind.Directory + : entry.IsSymlink ? FilesystemNodeKind.SymbolicLink + : FilesystemNodeKind.RegularFile; + + private static string NormalizePath(string path) { + ArgumentNullException.ThrowIfNull(path); + var result = new List(); + foreach (var raw in path.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries)) { + if (raw == ".") continue; + if (raw == "..") + throw new InvalidDataException($"Filesystem entry '{path}' escapes its root with '..'."); + result.Add(raw); + } + return string.Join('/', result); + } + + private static DateTimeOffset? ToOffset(DateTime? value) { + if (value == null) return null; + return value.Value.Kind switch { + DateTimeKind.Utc => new DateTimeOffset(value.Value, TimeSpan.Zero), + DateTimeKind.Local => new DateTimeOffset(value.Value), + _ => new DateTimeOffset(DateTime.SpecifyKind(value.Value, DateTimeKind.Utc), TimeSpan.Zero), + }; + } + + private static NotSupportedException ReadOnly() + => new("The derived filesystem projection is read-only; use a native filesystem provider for mounted writes."); + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} + +internal sealed class DerivedReadOnlyFileHandle : IFilesystemFileHandle { + private readonly byte[] _data; + private bool _disposed; + + public DerivedReadOnlyFileHandle(FilesystemNodeId nodeId, byte[] data) { + NodeId = nodeId; + _data = data; + } + + public FilesystemNodeId NodeId { get; } + public long Length { + get { + ThrowIfDisposed(); + return _data.LongLength; + } + } + + public int Read(long offset, Span destination) { + ThrowIfDisposed(); + if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset)); + if (offset >= _data.LongLength || destination.Length == 0) return 0; + var count = checked((int)Math.Min(destination.Length, _data.LongLength - offset)); + _data.AsSpan(checked((int)offset), count).CopyTo(destination); + return count; + } + + public void Write(long offset, ReadOnlySpan source) + => throw new NotSupportedException("The derived filesystem projection is read-only."); + public void SetLength(long length) + => throw new NotSupportedException("The derived filesystem projection is read-only."); + public void Flush() => ThrowIfDisposed(); + public void Dispose() => _disposed = true; + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} + +internal sealed class DerivedFilesystemSnapshot : IDisposable { + private const long MemoryThreshold = 16L * 1024 * 1024; + private readonly byte[]? _memory; + private readonly string? _temporaryPath; + private bool _disposed; + + private DerivedFilesystemSnapshot(byte[] memory) => _memory = memory; + private DerivedFilesystemSnapshot(string temporaryPath) => _temporaryPath = temporaryPath; + + public static DerivedFilesystemSnapshot Capture(Stream source) { + ArgumentNullException.ThrowIfNull(source); + if (!source.CanRead) throw new ArgumentException("Filesystem derivation requires a readable image.", nameof(source)); + + var originalPosition = source.CanSeek ? source.Position : 0; + try { + if (source.CanSeek) source.Position = 0; + if (source.CanSeek && source.Length <= MemoryThreshold) { + using var memory = new MemoryStream(checked((int)source.Length)); + source.CopyTo(memory); + return new DerivedFilesystemSnapshot(memory.ToArray()); + } + + var path = Path.Combine(Path.GetTempPath(), "cwb_fs_" + Guid.NewGuid().ToString("N") + ".img"); + try { + using (var target = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read, + 64 * 1024, FileOptions.SequentialScan)) + source.CopyTo(target); + return new DerivedFilesystemSnapshot(path); + } catch { + try { File.Delete(path); } catch { } + throw; + } + } finally { + if (source.CanSeek) source.Position = originalPosition; + } + } + + public Stream OpenRead() { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_memory != null) return new MemoryStream(_memory, writable: false); + return new FileStream(_temporaryPath!, FileMode.Open, FileAccess.Read, + FileShare.Read | FileShare.Delete, 64 * 1024, FileOptions.RandomAccess); + } + + public void Dispose() { + if (_disposed) return; + _disposed = true; + if (_temporaryPath != null) { + try { File.Delete(_temporaryPath); } catch { } + } + } +} diff --git a/Compression.Registry/FormatCapabilities.cs b/Compression.Registry/FormatCapabilities.cs index b5a77191b..cff72da2a 100644 --- a/Compression.Registry/FormatCapabilities.cs +++ b/Compression.Registry/FormatCapabilities.cs @@ -8,23 +8,23 @@ namespace Compression.Registry; /// /// Unsupported — no descriptor exists. /// Read-Only and/or only. -/// WORM (Write-Once-Read-Many) — adds : a fresh archive can be produced from inputs, but existing archives cannot be modified in place. -/// R/W (Modify) — adds : entries can be added, replaced, or removed in an existing archive without full rewrite. +/// WORM (Write-Once-Read-Many) — adds : a fresh archive/image can be produced, but the library has no supported edit of an existing instance. +/// R/W (Modify) — adds : an existing instance supports add/replace/remove and remains valid after the edit. /// /// -/// Most archive formats stop at WORM; true in-place modification is rare because compressed -/// archive containers don't generally support entry mutation without a full rebuild. +/// R/W describes the public operation, not the physical write strategy. +/// A format may update allocation metadata in place, append a new index, relayout members, +/// or rebuild the complete image. Those are implementation choices. If callers can open an +/// existing instance, apply add/replace/remove through , and +/// obtain a valid instance preserving the semantics the implementation claims to support, +/// the format is R/W at this API surface. Conversely, merely having a writer for fresh images +/// is WORM and must not set . /// /// -/// Honesty rule — rebuild-backed modification is WORM, not R/W. A format may implement -/// purely to make the add / remove / purge verbs work, -/// backing them with the verified extract → re-create rebuild (the default -/// members, or ModifyRebuilder / ). That is a full rewrite of the -/// container, so such a format advertises only and must not set -/// — the verb still runs, but no in-place R/W is claimed. -/// is reserved for formats with a genuine in-place writer that edits the existing container -/// (e.g. ZIP/TAR central-directory edits, FAT/NTFS/ext block writes, byte-identity append). -/// Compression.Tests.Operations.WriteCapabilityHonestyTests enforces this for every claimant. +/// This distinction is especially important for read-only-on-mount filesystem formats such as +/// SquashFS, CramFS and EROFS: the native filesystem driver may intentionally forbid mounted +/// writes while an offline image editor can still support complete, deterministic mutation by +/// relayout/rebuild. reports the latter capability. /// /// [Flags] @@ -32,7 +32,7 @@ public enum FormatCapabilities { None = 0, CanList = 1 << 0, CanExtract = 1 << 1, - /// WORM: can produce a fresh archive from inputs (no in-place modification). + /// WORM: can produce a fresh archive/image, but has no supported existing-instance edit. CanCreate = 1 << 2, CanTest = 1 << 3, SupportsPassword = 1 << 4, @@ -40,6 +40,6 @@ public enum FormatCapabilities { SupportsDirectories = 1 << 6, SupportsOptimize = 1 << 8, CanCompoundWithTar = 1 << 9, - /// R/W: can modify an existing archive (add/replace/remove entries) without full rewrite. Implies . + /// R/W: can add/replace/remove entries in an existing archive/image. The implementation may edit in place or relayout/rebuild. Implies for normal writable formats. CanModify = 1 << 10, -} +} \ No newline at end of file diff --git a/Compression.Registry/FormatRegistry.cs b/Compression.Registry/FormatRegistry.cs index 3fba96f8e..a7d93702f 100644 --- a/Compression.Registry/FormatRegistry.cs +++ b/Compression.Registry/FormatRegistry.cs @@ -1,8 +1,9 @@ namespace Compression.Registry; /// -/// Central registry of all format descriptors. Populated at startup via -/// calls (typically from source-generated code), then finalized with . +/// Central registry of all format descriptors and their optional driver sidecars. +/// Populated at startup by generated registration code, then finalized with +/// . /// public static class FormatRegistry { @@ -13,25 +14,53 @@ public static class FormatRegistry { private static readonly Dictionary _streamOps = new(StringComparer.OrdinalIgnoreCase); private static readonly Dictionary _archiveOps = new(StringComparer.OrdinalIgnoreCase); private static readonly Dictionary _asyncArchiveOps = new(StringComparer.OrdinalIgnoreCase); + private static readonly Dictionary _filesystemDrivers = new(StringComparer.OrdinalIgnoreCase); + private static readonly HashSet _filesystemFormatIds = new(StringComparer.OrdinalIgnoreCase); private static bool _initialized; - /// - /// Finalize the registry by building lookup tables. Safe to call multiple times. - /// Call this after all calls are complete. - /// public static void Initialize() { if (_initialized) return; - _initialized = true; BuildLookups(); + + foreach (var (id, _) in _filesystemDrivers) + if (!_byId.ContainsKey(id)) + throw new InvalidOperationException($"Filesystem driver adapter '{id}' has no registered format descriptor."); + + // A FileSystem.* project is not allowed to stop at an isolated parser API. + // Every registered filesystem must be reachable through the common driver + // contract either natively/through a sidecar or through the conservative + // read-only List/OpenEntry projection. This is structural; per-image Probe + // can still reject damaged or unsupported feature profiles. + foreach (var id in _filesystemFormatIds) { + if (!_byId.TryGetValue(id, out var descriptor)) + throw new InvalidOperationException($"Generated filesystem id '{id}' has no registered descriptor."); + if (descriptor is IFilesystemDriverProvider) continue; + if (_filesystemDrivers.ContainsKey(id)) continue; + if (descriptor is IArchiveFormatOperations) continue; + throw new InvalidOperationException( + $"Filesystem '{id}' exposes no IFilesystemDriverProvider, generated sidecar, or IArchiveFormatOperations projection."); + } + + // Publish initialized only after every cross-registry invariant succeeds. + // A failed coverage check must not poison the registry into an apparently + // initialized state that subsequent callers can no longer repair/reset in tests. + _initialized = true; } /// - /// Register a format descriptor. Called by generated code and for compound tar auto-generation. - /// Must be called before . + /// Register a format descriptor. Source-generated calls set + /// for descriptors declared under a + /// FileSystem.* namespace so filesystem coverage is explicit rather + /// than inferred from extensions or display names. /// - public static void Register(IFormatDescriptor descriptor) { + public static void Register(IFormatDescriptor descriptor, bool isFilesystem = false) { + ArgumentNullException.ThrowIfNull(descriptor); + if (_initialized) throw new InvalidOperationException("FormatRegistry is already initialized."); + if (_byId.ContainsKey(descriptor.Id)) + throw new InvalidOperationException($"A format descriptor with id '{descriptor.Id}' is already registered."); _all.Add(descriptor); - _byId[descriptor.Id] = descriptor; + _byId.Add(descriptor.Id, descriptor); + if (isFilesystem) _filesystemFormatIds.Add(descriptor.Id); if (descriptor is IStreamFormatOperations streamOps) _streamOps[descriptor.Id] = streamOps; if (descriptor is IArchiveFormatOperations archiveOps) @@ -40,45 +69,131 @@ public static void Register(IFormatDescriptor descriptor) { _asyncArchiveOps[descriptor.Id] = asyncArchiveOps; } - /// All registered descriptors. + /// + /// Registers one source-generated native filesystem-driver sidecar. Duplicate + /// adapters for the same format ID are a build/runtime contract error rather + /// than whichever registration happened to win. + /// + public static void RegisterFilesystemDriver(IFilesystemDriverAdapter driver) { + ArgumentNullException.ThrowIfNull(driver); + if (_initialized) throw new InvalidOperationException("FormatRegistry is already initialized."); + if (string.IsNullOrWhiteSpace(driver.FormatId)) + throw new ArgumentException("Filesystem driver adapter must provide a format ID.", nameof(driver)); + if (!_filesystemDrivers.TryAdd(driver.FormatId, driver)) + throw new InvalidOperationException($"A filesystem driver adapter for '{driver.FormatId}' is already registered."); + } + public static IReadOnlyList All => _all; - /// Look up a descriptor by its unique ID. + /// All descriptor IDs originating from FileSystem.* projects. + public static IReadOnlyList FilesystemFormatIds + => _filesystemFormatIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase).ToArray(); + public static IFormatDescriptor? GetById(string id) => _byId.GetValueOrDefault(id); - /// Look up a descriptor by file path/extension. Checks compound extensions first (longest match). public static IFormatDescriptor? GetByExtension(string path) { var lower = path.ToLowerInvariant(); - - // Check compound extensions first (e.g. .tar.gz, .tar.bz2) - foreach (var (ext, desc) in _byCompoundExtension) { - if (lower.EndsWith(ext)) - return desc; - } - - // Fall back to single extension + foreach (var (ext, desc) in _byCompoundExtension) + if (lower.EndsWith(ext)) return desc; var singleExt = Path.GetExtension(lower); return string.IsNullOrEmpty(singleExt) ? null : _byExtension.GetValueOrDefault(singleExt); } - /// Get all descriptors in a given category. public static IEnumerable GetByCategory(FormatCategory category) => _all.Where(d => d.Category == category); - /// Get stream operations for a format ID, or null if not a stream format. public static IStreamFormatOperations? GetStreamOps(string id) => _streamOps.GetValueOrDefault(id); - /// Get archive operations for a format ID, or null if not an archive format. public static IArchiveFormatOperations? GetArchiveOps(string id) => _archiveOps.GetValueOrDefault(id); - /// Get async archive operations for a format ID, or null if the format doesn't support async listing. + /// Returns a generated native driver sidecar for the format, when one exists. + public static IFilesystemDriverAdapter? GetFilesystemDriver(string id) + => _filesystemDrivers.GetValueOrDefault(id); + + /// + /// Structural driver coverage for all FileSystem.* descriptors. This is safe + /// to inspect without an image and is intended for CI/readiness dashboards. + /// Use for exact per-image semantics. + /// + public static IReadOnlyList GetFilesystemDriverCoverage() + => _filesystemFormatIds + .OrderBy(id => id, StringComparer.OrdinalIgnoreCase) + .Select(GetFilesystemDriverCoverage) + .ToArray(); + + public static FilesystemDriverCoverage GetFilesystemDriverCoverage(string id) { + if (!_filesystemFormatIds.Contains(id)) + throw new KeyNotFoundException($"Format '{id}' is not registered as a FileSystem.* descriptor."); + var descriptor = GetById(id) + ?? throw new KeyNotFoundException($"Unknown format id '{id}'."); + var sidecar = _filesystemDrivers.GetValueOrDefault(id); + var binding = descriptor is IFilesystemDriverProvider + ? FilesystemDriverBindingKind.DescriptorNative + : sidecar != null + ? FilesystemDriverBindingKind.SidecarNative + : descriptor is IArchiveFormatOperations + ? FilesystemDriverBindingKind.ArchiveProjection + : FilesystemDriverBindingKind.None; + return new FilesystemDriverCoverage( + descriptor.Id, + descriptor.DisplayName, + binding, + HasArchiveProjection: descriptor is IArchiveFormatOperations, + HasArchiveMutation: descriptor is IArchiveModifiable, + HasExtentMap: descriptor is IFilesystemExtentMap, + HasBlockMover: descriptor is IFilesystemBlockMover, + HasBlockDeviceProvider: descriptor is IRandomAccessBlockDeviceProvider, + HasNativeReadinessProvider: + descriptor is IFilesystemDriverReadinessProvider || sidecar is IFilesystemDriverReadinessProvider); + } + + public static FilesystemDriverProfile ProbeFilesystem( + string id, + Stream image, + string? password = null) { + var descriptor = GetById(id) + ?? throw new KeyNotFoundException($"Unknown format id '{id}'."); + if (descriptor is IFilesystemDriverProvider native) + return native.ProbeFilesystem(image); + if (_filesystemDrivers.TryGetValue(id, out var adapter)) + return adapter.ProbeFilesystem(image); + return FilesystemDriverDerivation.Probe(descriptor, image, password); + } + + public static IFilesystemSession OpenFilesystem( + string id, + Stream image, + FilesystemOpenOptions options, + string? password = null) { + var descriptor = GetById(id) + ?? throw new KeyNotFoundException($"Unknown format id '{id}'."); + if (descriptor is IFilesystemDriverProvider native) + return native.OpenFilesystem(image, options); + if (_filesystemDrivers.TryGetValue(id, out var adapter)) + return adapter.OpenFilesystem(image, options); + return FilesystemDriverDerivation.Open(descriptor, image, options, password); + } + + public static FilesystemDriverReadinessReport AssessFilesystemDriver( + string id, + Stream image, + FilesystemDriverTarget target, + string? password = null) { + var descriptor = GetById(id) + ?? throw new KeyNotFoundException($"Unknown format id '{id}'."); + if (descriptor is IFilesystemDriverReadinessProvider native) + return native.DescribeFilesystemDriverReadiness(image, target); + if (_filesystemDrivers.TryGetValue(id, out var adapter)) + return adapter.DescribeFilesystemDriverReadiness(image, target); + return FilesystemDriverDerivation.Assess(descriptor, image, target, password); + } + public static IAsyncArchiveOperations? GetAsyncArchiveOps(string id) => _asyncArchiveOps.GetValueOrDefault(id); - /// Reset the registry (for testing only). internal static void Reset() { _all.Clear(); _byId.Clear(); @@ -87,6 +202,8 @@ internal static void Reset() { _streamOps.Clear(); _archiveOps.Clear(); _asyncArchiveOps.Clear(); + _filesystemDrivers.Clear(); + _filesystemFormatIds.Clear(); _initialized = false; } @@ -94,7 +211,6 @@ private static void BuildLookups() { foreach (var desc in _all) { foreach (var ext in desc.CompoundExtensions) _byCompoundExtension.TryAdd(ext.ToLowerInvariant(), desc); - foreach (var ext in desc.Extensions) _byExtension.TryAdd(ext.ToLowerInvariant(), desc); } diff --git a/Compression.Registry/IArchiveDefragmentable.cs b/Compression.Registry/IArchiveDefragmentable.cs index d62d7ca49..53a59841d 100644 --- a/Compression.Registry/IArchiveDefragmentable.cs +++ b/Compression.Registry/IArchiveDefragmentable.cs @@ -1,47 +1,52 @@ namespace Compression.Registry; /// -/// Opt-in capability: the descriptor can rewrite an archive in place so that every file -/// occupies a contiguous cluster run, optionally with a chosen layout strategy -/// (consolidate at start / end, lazy hole-fill, carve a free region). Complements the -/// allocator's automatic fast-defrag (which fires only when a pending allocation can't -/// find a contiguous hole); this is the user-initiated full pass. +/// Opt-in capability for physical or logical re-layout. Native mutable filesystems +/// may move extents in place; WORM/archive containers can satisfy the same verb by +/// building a verified staged target and committing it after completion. /// public interface IArchiveDefragmentable { /// - /// Rebuilds the archive content in place so every file is contiguous. Outer byte size - /// is preserved. Free space is consolidated at the end. - /// - /// Default implementation: any descriptor that also implements - /// + gets - /// defragmentation for free — a verified in-place extract → re-create rebuild via - /// (the rebuild-via-WORM pattern inherently - /// lays every file out contiguously) that refuses to commit a lossy result. Formats - /// with a true in-place block mover override this for efficiency and full mode support. + /// Defragments using the format's default consolidate-at-start strategy. + /// Generic list/extract/create descriptors use the verified staged rebuild. /// void Defragment(Stream archive) { if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) - throw new System.NotSupportedException( - "The default Defragment requires the descriptor to also implement IArchiveFormatOperations + IArchiveCreatable."); + throw new NotSupportedException( + "The default Defragment requires IArchiveFormatOperations + IArchiveCreatable."); RebuildVerb.RebuildInPlace(archive, ops, creator); } /// - /// Rewrites the archive content according to . Default - /// implementation forwards to for - /// and throws for every other mode — - /// implementers should override to support all modes their on-disk format permits. + /// Rewrites according to . Descriptors with their own + /// native parameterless mover retain it. Descriptors relying on the interface + /// default are routed through the progress-reporting, cancellable staged rebuild, + /// so archive repacks and WORM re-layouts drive the same block-map UI as physical + /// filesystem extent moves. /// - /// If the implementer doesn't support - /// the requested mode (default for any mode other than - /// ). void Defragment(Stream archive, DefragOptions options) { - System.ArgumentNullException.ThrowIfNull(options); - if (options.Mode == DefragMode.ConsolidateAtStart) { + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException( + $"This descriptor only supports DefragMode.ConsolidateAtStart; got {options.Mode}."); + + // Preserve a concrete native parameterless implementation when one exists. + // Generic promoted descriptors have no such method and therefore get the + // staged rebuild below, including block-map progress and safe cancellation. + var native = this.GetType().GetMethod(nameof(Defragment), [typeof(Stream)]); + if (native != null && native.DeclaringType != typeof(IArchiveDefragmentable)) { + options.CancellationToken.ThrowIfCancellationRequested(); this.Defragment(archive); return; } - throw new System.NotSupportedException( - $"This descriptor only supports DefragMode.ConsolidateAtStart; got {options.Mode}."); + + if (this is IArchiveFormatOperations ops && this is IArchiveCreatable creator) { + RebuildVerb.RebuildInPlace(archive, ops, creator, + onProgress: options.OnProgress, + cancellationToken: options.CancellationToken); + return; + } + + this.Defragment(archive); } } diff --git a/Compression.Registry/IArchiveFormatOperations.cs b/Compression.Registry/IArchiveFormatOperations.cs index ce8dd587e..120498064 100644 --- a/Compression.Registry/IArchiveFormatOperations.cs +++ b/Compression.Registry/IArchiveFormatOperations.cs @@ -20,59 +20,48 @@ public interface IArchiveFormatOperations { /// entry's logical bytes — physically incapable of reading slack space, /// adjacent entries, padding/alignment fillers, or header/metadata regions. /// This is the canonical per-entry isolation primitive used by streaming - /// conversion pipelines. + /// conversion and derived-filesystem pipelines. /// /// /// - /// The returned stream is always a (or a - /// wrapper that satisfies the same contract). Reads past the entry's - /// logical size return 0 (EOF); seek targets are clamped to the bound. - /// The caller owns disposal. + /// Reads past the logical size return 0 (EOF); seek targets cannot escape the + /// entry. The caller owns disposal. /// /// - /// Default implementation buffers the entry's bytes via - /// and wraps a - /// over the result. Descriptors with native per-entry readers (FAT cluster - /// chains, ZIP DEFLATE wrapper, TAR positional slice, 7z folder slot) - /// should override to return a properly bounded streaming view of their - /// decoder output. + /// The default implementation intentionally does not materialize a + /// byte[]. It asks for the selected entry in an + /// isolated temporary directory, opens the resulting file as a seekable + /// stream, and deletes that tree on dispose. This gives every descriptor a + /// large-file-safe streaming fallback even before it grows a native per-entry + /// reader. Native readers (FAT chains, ZIP decoder streams, TAR slices, etc.) + /// should still override this to avoid the temporary extraction pass. /// /// public virtual Stream OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryName); - var bytes = this.ExtractEntryToMemory(archive, entryName, password); - return new BoundedEntryStream(new MemoryStream(bytes, writable: false), bytes.Length, leaveOpen: false); + ArgumentException.ThrowIfNullOrWhiteSpace(entryName); + var extracted = TemporaryExtractedEntryStream.Open(this, archive, entryName, password); + return new BoundedEntryStream(extracted, extracted.Length, leaveOpen: false); } /// - /// Extracts a single entry to a byte array without writing to disk. The default - /// implementation now routes through so the bounded - /// streaming contract is enforced even when callers ask for a buffered result. - /// Descriptors that have a more efficient native byte-array path (e.g. a - /// reader that already materialises the whole entry) can still override. + /// Extracts a single entry to a byte array. This is the explicitly buffered + /// convenience API; callers working with large entries should use + /// instead. /// /// - /// The wrapper rewinds to position 0 when - /// possible, opens the entry as a bounded stream, and copies it into a - /// fresh byte array. The bound on guarantees the - /// result contains only the entry's logical bytes. + /// The default routes through , so descriptor-specific + /// isolation/decoding semantics are preserved. A result past the runtime array + /// limit naturally fails here rather than imposing that limit on the streaming + /// API or filesystem-driver layer. /// public virtual byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryName); + ArgumentException.ThrowIfNullOrWhiteSpace(entryName); if (archive.CanSeek) archive.Position = 0; - // Fall back to the tempdir path when the descriptor has not overridden - // either method — that's the only way to break the recursion between the - // two virtual defaults. - var tempDir = Path.Combine(Path.GetTempPath(), "cwb_x2m_" + Guid.NewGuid().ToString("N")[..8]); - try { - Directory.CreateDirectory(tempDir); - this.Extract(archive, tempDir, password, [entryName]); - var file = Path.Combine(tempDir, entryName.Replace('/', Path.DirectorySeparatorChar)); - return File.Exists(file) ? File.ReadAllBytes(file) : []; - } finally { - try { if (Directory.Exists(tempDir)) Directory.Delete(tempDir, true); } catch { /* best-effort */ } - } + using var entry = this.OpenEntry(archive, entryName, password); + using var memory = new MemoryStream(); + entry.CopyTo(memory); + return memory.ToArray(); } } diff --git a/Compression.Registry/IArchiveLayoutMap.cs b/Compression.Registry/IArchiveLayoutMap.cs index 606200a12..a4742b24b 100644 --- a/Compression.Registry/IArchiveLayoutMap.cs +++ b/Compression.Registry/IArchiveLayoutMap.cs @@ -7,16 +7,28 @@ namespace Compression.Registry; /// gaps at their actual offsets. Parallel to /// but for archive formats (ZIP, 7z, TAR, LZH, ARJ, etc.). /// +/// Fail-closed contract: omitted bytes are interpreted as unused by +/// maintenance consumers. Any live, structural, ambiguous or undecoded region +/// must therefore be emitted as +/// (or ), never silently omitted. If a layout +/// cannot be proven safe, return no extents and the inherited generic wipe is a +/// no-op. +/// +/// That exact preservation map also makes every implementation an +/// capability. The generic implementation zeros proven +/// dead gaps while format-specific overrides may additionally scrub tombstones, +/// reserved growth records, stale indexes, or other recoverable metadata. +/// /// Drives the Defragment/Optimize window block-map preview so the user /// sees the real archive layout before pressing "Optimize". /// -public interface IArchiveLayoutMap { +public interface IArchiveLayoutMap : IWipeEmpty { /// /// Enumerates the actual byte layout of . - /// Coverage may be sparse; callers fill the gaps with - /// . The stream's position may be - /// modified during enumeration but the caller owns the lifetime — - /// implementations must not dispose . + /// Coverage may be sparse only where omitted bytes are proven unused; callers + /// fill those gaps with . The stream's + /// position may be modified during enumeration but the caller owns the + /// lifetime — implementations must not dispose . /// IEnumerable EnumerateLayout(Stream archive); } diff --git a/Compression.Registry/IArchiveModifiable.cs b/Compression.Registry/IArchiveModifiable.cs index bcaa47969..323ec77ea 100644 --- a/Compression.Registry/IArchiveModifiable.cs +++ b/Compression.Registry/IArchiveModifiable.cs @@ -1,29 +1,28 @@ namespace Compression.Registry; /// -/// Opt-in capability: the descriptor exposes add / remove (and thereby the purge verb). +/// Opt-in capability for editing an existing archive/image through add/replace/remove. /// -/// Implementing this interface makes the verbs work; it does not by itself -/// entitle the format to advertise (R/W). The -/// default / below — and any override that delegates -/// to ModifyRebuilder / — are a verified extract → re-create -/// rebuild, i.e. a full rewrite of the container. A format whose modification is only -/// rebuild-backed is WORM: it advertises and must -/// NOT advertise (see ). -/// Reserve for a genuine in-place writer that edits -/// the existing bytes (R/W filesystems; central-directory / member edits; byte-identity append). +/// The physical strategy is format-specific: implementations may patch blocks in place, +/// append replacement metadata, relayout members, or perform a verified extract → edit → +/// re-create rebuild. All are valid implementations of the same public mutation contract +/// when the resulting instance preserves the semantics the descriptor claims to support. +/// +/// +/// A descriptor advertising must expose this +/// interface and its supported-profile edit path must actually round-trip. Merely being able +/// to create a fresh instance is not enough. A fully modifiable container is also purgeable: +/// removing all live entries is a required subset of the remove contract. /// /// -public interface IArchiveModifiable { +public interface IArchiveModifiable : IArchivePurgeable { /// - /// Appends or replaces files inside . On replacement the - /// previous bytes are wiped the same way wipes them. + /// Adds files to an existing instance, replacing entries with the same logical path/name. /// - /// Default implementation: any descriptor that also implements - /// + gets - /// add for free — a verified extract → splat-new-files → re-create rebuild via - /// (the same WORM rebuild that backs the - /// other verbs). Formats with a true in-place writer override for efficiency. + /// Default implementation: descriptors that also implement + /// and get a verified + /// extract → edit → re-create implementation through . + /// Formats with a cheaper native editor override it. /// void Add(Stream archive, IReadOnlyList inputs) { if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) @@ -41,11 +40,12 @@ void Add(Stream archive, IReadOnlyList inputs) { } /// - /// Removes the named entries from and wipes all on-disk - /// traces. Default implementation: a verified extract → drop-named-files → - /// re-create rebuild via . Passing every - /// entry name (or all files) yields an empty container — i.e. the purge verb. - /// Formats with a true in-place writer override for efficiency and forensic wiping. + /// Removes the named entries from an existing instance. Passing every entry name yields an + /// empty container/image where the format permits one. + /// + /// Default implementation: a verified extract → drop-named-files → re-create + /// edit through . Native implementations may instead + /// unlink/free in place and optionally wipe released storage. /// void Remove(Stream archive, string[] entryNames) { if (this is not IArchiveFormatOperations ops || this is not IArchiveCreatable creator) diff --git a/Compression.Registry/IArchivePurgeable.cs b/Compression.Registry/IArchivePurgeable.cs new file mode 100644 index 000000000..ea1b0128c --- /dev/null +++ b/Compression.Registry/IArchivePurgeable.cs @@ -0,0 +1,25 @@ +namespace Compression.Registry; + +/// +/// Opt-in capability: all user/live entries can be removed from an existing +/// container while leaving a valid, listable empty instance. This is distinct +/// from , which preserves live entries and overwrites only +/// unused/dead bytes. +/// +public interface IArchivePurgeable { + /// + /// Removes every live non-directory entry from . + /// + /// Default implementation: descriptors that also implement + /// and + /// get a transactional staged purge through . + /// Native implementations may override this when they can empty the container + /// more efficiently. + /// + void Purge(Stream archive) { + if (this is not IArchiveFormatOperations ops || this is not IArchiveModifiable modifier) + throw new NotSupportedException( + "The default Purge requires IArchiveFormatOperations + IArchiveModifiable."); + RebuildVerb.PurgeViaModifier(archive, ops, modifier); + } +} diff --git a/Compression.Registry/IFilesystemDriverAdapter.cs b/Compression.Registry/IFilesystemDriverAdapter.cs new file mode 100644 index 000000000..78b0a1748 --- /dev/null +++ b/Compression.Registry/IFilesystemDriverAdapter.cs @@ -0,0 +1,15 @@ +#pragma warning disable CS1591 +namespace Compression.Registry; + +/// +/// Sidecar binding from an existing format descriptor ID to a native filesystem +/// driver core. This lets large/legacy descriptors acquire driver semantics +/// without mixing mount state, locking and block-device code into their archive +/// surface. The source generator discovers public parameterless implementations +/// and registers them by . +/// +public interface IFilesystemDriverAdapter : + IFilesystemDriverProvider, + IFilesystemDriverReadinessProvider { + string FormatId { get; } +} diff --git a/Compression.Registry/IFilesystemExtentMap.cs b/Compression.Registry/IFilesystemExtentMap.cs index 13d394aef..0960b35c7 100644 --- a/Compression.Registry/IFilesystemExtentMap.cs +++ b/Compression.Registry/IFilesystemExtentMap.cs @@ -9,23 +9,32 @@ namespace Compression.Registry; /// superblock, MFT, root directory, inode table, BAM, group descriptor table, /// etc.), and optionally every free region. /// -/// Coverage may be sparse — gaps in the returned set are interpreted by -/// the caller as . The yielded extents -/// don't need to be sorted; the caller is responsible for sorting + gap -/// filling. Implementations must not throw for malformed or partially-walked -/// images — they should yield whatever they can identify and return. +/// Fail-closed contract: gaps in the returned set are interpreted +/// as free space by maintenance consumers. Therefore an implementation that +/// encounters an allocated-but-undecoded, damaged, ambiguous, or otherwise +/// unproven region MUST emit that region as +/// rather than silently omit it. +/// If the image cannot be walked safely at all, yield no extents; the inherited +/// generic implementation then wipes nothing. +/// +/// Because this contract identifies all bytes that must be preserved, +/// every extent map is also an implementation: the +/// default wiper zeros only proven gaps (and cluster tips when a trustworthy +/// logical-size lookup exists). Formats that know about deleted directory +/// records or other hidden remnants may override the wipe for deeper cleaning. /// /// Drives the Defragment-window block-map preview so the user sees the /// real fragmented layout before pressing "Defragment" rather than the /// post-defrag approximation. /// -public interface IFilesystemExtentMap { +public interface IFilesystemExtentMap : IWipeEmpty { /// /// Enumerates the actual on-disk layout of . - /// Coverage may be sparse; callers fill the gaps with - /// . The stream's position may be - /// modified during enumeration but the caller owns the lifetime — - /// implementations must not dispose . + /// Coverage may be sparse only where the omitted bytes are proven free; + /// callers fill those gaps with . Unknown + /// allocated bytes must be returned as . + /// The stream's position may be modified during enumeration but the caller + /// owns its lifetime — implementations must not dispose . /// /// The filesystem image to walk. Must be readable and /// seekable. diff --git a/Compression.Registry/IRandomAccessBlockDeviceProvider.cs b/Compression.Registry/IRandomAccessBlockDeviceProvider.cs new file mode 100644 index 000000000..c70483b26 --- /dev/null +++ b/Compression.Registry/IRandomAccessBlockDeviceProvider.cs @@ -0,0 +1,16 @@ +#pragma warning disable CS1591 +namespace Compression.Registry; + +/// +/// Optional descriptor capability for exposing the sector/block device that +/// sits below a filesystem namespace. Container descriptors can implement this +/// without pretending the container itself is a filesystem. +/// +public interface IRandomAccessBlockDeviceProvider { + /// + /// Opens a random-access block device over . + /// Implementations must fail closed when the exact on-disk profile cannot be + /// projected losslessly/safely at block granularity. + /// + IRandomAccessBlockDevice OpenBlockDevice(Stream image, bool writable, bool leaveOpen = true); +} diff --git a/Compression.Registry/IWipeEmpty.cs b/Compression.Registry/IWipeEmpty.cs index 41e535937..a465eb24d 100644 --- a/Compression.Registry/IWipeEmpty.cs +++ b/Compression.Registry/IWipeEmpty.cs @@ -7,9 +7,11 @@ namespace Compression.Registry; /// entries, padding regions, and dead archive bytes. This is a forensic-cleanliness /// tool ensuring no deleted file remnants survive. /// -/// Implementations that don't need format-specific logic can delegate to -/// which works generically with any -/// or . +/// The default implementation is deliberately conservative. It is available +/// only when the same descriptor exposes an exact filesystem extent map or archive +/// layout map; unknown/undecoded regions must therefore be emitted as +/// by those maps rather than omitted. +/// An empty map is treated as "cannot prove anything is free" and wipes nothing. /// public interface IWipeEmpty { /// @@ -21,7 +23,45 @@ public interface IWipeEmpty { /// When true, also zero the tail of cluster-aligned /// extents where the actual file size is smaller than the allocated extent. /// When true, zero deleted directory entries - /// and any other recoverable remnants beyond simple free-space gaps. + /// and any other recoverable remnants beyond simple free-space gaps. The generic + /// implementation can only wipe gaps/tips; format-specific implementations may + /// additionally scrub deleted metadata records. /// The number of bytes that were overwritten with zeros. - long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true); + long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("Wipe requires a readable, writable, seekable stream.", nameof(image)); + + List extents = this switch { + IFilesystemExtentMap fs => fs.EnumerateExtents(image).ToList(), + IArchiveLayoutMap archive => archive.EnumerateLayout(image).ToList(), + _ => throw new NotSupportedException( + "The generic wipe requires IFilesystemExtentMap or IArchiveLayoutMap."), + }; + + // Fail closed. A parser that cannot prove even one region exists must never + // turn "unknown image" into "everything is free". + if (extents.Count == 0) + return 0; + + Func? sizeLookup = null; + if (wipeClusterTips && this is IArchiveFormatOperations ops) { + try { + image.Position = 0; + var sizes = ops.List(image, null) + .Where(e => !e.IsDirectory) + .GroupBy(e => e.Name, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => Math.Max(0L, g.First().OriginalSize), StringComparer.Ordinal); + sizeLookup = name => sizes.TryGetValue(name, out var size) ? size : -1; + } catch { + // Lack of a trustworthy logical-size table merely disables tip wiping; + // it must not disable proven whole free gaps. + sizeLookup = null; + wipeClusterTips = false; + } + } + + image.Position = 0; + return UnusedSpaceWiper.Wipe(image, extents, image.Length, wipeClusterTips, sizeLookup); + } } diff --git a/Compression.Registry/ReadOnlyFilesystemSnapshotSession.cs b/Compression.Registry/ReadOnlyFilesystemSnapshotSession.cs new file mode 100644 index 000000000..508091bdf --- /dev/null +++ b/Compression.Registry/ReadOnlyFilesystemSnapshotSession.cs @@ -0,0 +1,201 @@ +#pragma warning disable CS1591 + +namespace Compression.Registry; + +/// +/// Native filesystem object projected into the common driver contract. Name and +/// parent are a convenient primary-link description for simple filesystems; use +/// explicit values when one node +/// has multiple directory entries (hard links). +/// +public sealed record FilesystemSnapshotNode( + FilesystemNodeId NodeId, + FilesystemNodeId ParentNodeId, + string Name, + FilesystemNodeKind Kind, + long Size, + long AllocatedSize, + uint LinkCount = 1, + ulong NativeAttributes = 0, + DateTimeOffset? Created = null, + DateTimeOffset? Modified = null, + DateTimeOffset? Accessed = null, + DateTimeOffset? Changed = null, + string? SymbolicLinkTarget = null, + Func? OpenReadHandle = null +); + +public sealed record FilesystemSnapshotDirectoryEntry( + FilesystemNodeId ParentNodeId, + string Name, + FilesystemNodeId NodeId +); + +public sealed class ReadOnlyFilesystemSnapshotSession : IFilesystemSession { + private sealed record Child(string Name, FilesystemSnapshotNode Node); + private sealed record SnapshotInput( + FilesystemSnapshotNode[] Nodes, + FilesystemSnapshotDirectoryEntry[] Entries); + + private readonly Dictionary _nodes; + private readonly Dictionary _children; + private bool _disposed; + + public ReadOnlyFilesystemSnapshotSession( + FilesystemDriverProfile profile, + FilesystemNodeId rootNodeId, + IEnumerable nodes) + : this(profile, rootNodeId, Prepare(nodes, rootNodeId)) { } + + private ReadOnlyFilesystemSnapshotSession( + FilesystemDriverProfile profile, + FilesystemNodeId rootNodeId, + SnapshotInput input) + : this(profile, rootNodeId, input.Nodes, input.Entries) { } + + /// + /// Full constructor with independent object and directory-entry sets. Multiple + /// entries may target the same node ID; that is how hard links are represented. + /// + public ReadOnlyFilesystemSnapshotSession( + FilesystemDriverProfile profile, + FilesystemNodeId rootNodeId, + IEnumerable nodes, + IEnumerable directoryEntries) { + ArgumentNullException.ThrowIfNull(profile); + ArgumentNullException.ThrowIfNull(nodes); + ArgumentNullException.ThrowIfNull(directoryEntries); + if (!profile.CanMount) + throw new ArgumentException("A snapshot session requires a mountable filesystem profile.", nameof(profile)); + if (profile.CanMountWritable) + throw new ArgumentException("ReadOnlyFilesystemSnapshotSession cannot represent a writable profile.", nameof(profile)); + + Profile = profile; + RootNodeId = rootNodeId; + _nodes = new Dictionary(); + foreach (var node in nodes) { + if (!_nodes.TryAdd(node.NodeId, node)) + throw new InvalidDataException( + $"Filesystem snapshot defines node {node.NodeId.Value}:{node.NodeId.Generation} more than once."); + } + if (!_nodes.TryGetValue(rootNodeId, out var root) || root.Kind != FilesystemNodeKind.Directory) + throw new InvalidDataException("Filesystem snapshot must contain its root directory node."); + + var children = _nodes.Keys.ToDictionary(id => id, _ => new List()); + foreach (var entry in directoryEntries) { + ArgumentNullException.ThrowIfNull(entry.Name); + if (entry.Name.Length == 0 || entry.Name is "." or ".." || entry.Name.Contains('/') || entry.Name.Contains('\\')) + throw new InvalidDataException($"Filesystem directory entry name '{entry.Name}' is not a single valid path component."); + if (!_nodes.TryGetValue(entry.ParentNodeId, out var parent) || parent.Kind != FilesystemNodeKind.Directory) + throw new InvalidDataException( + $"Filesystem directory entry '{entry.Name}' has no directory parent {entry.ParentNodeId.Value}:{entry.ParentNodeId.Generation}."); + if (!_nodes.TryGetValue(entry.NodeId, out var target)) + throw new InvalidDataException( + $"Filesystem directory entry '{entry.Name}' targets missing node {entry.NodeId.Value}:{entry.NodeId.Generation}."); + var list = children[entry.ParentNodeId]; + if (list.Any(existing => string.Equals(existing.Name, entry.Name, StringComparison.Ordinal))) + throw new InvalidDataException($"Filesystem directory contains duplicate name '{entry.Name}'."); + list.Add(new Child(entry.Name, target)); + } + + _children = children.ToDictionary( + item => item.Key, + item => item.Value.OrderBy(child => child.Name, StringComparer.Ordinal).ToArray()); + } + + public FilesystemDriverProfile Profile { get; } + public FilesystemNodeId RootNodeId { get; } + + public FilesystemNodeInfo Stat(FilesystemNodeId nodeId) { + ThrowIfDisposed(); + var node = RequireNode(nodeId); + return new FilesystemNodeInfo( + node.NodeId, + node.Kind, + Math.Max(0, node.Size), + Math.Max(0, node.AllocatedSize), + node.LinkCount, + node.NativeAttributes, + node.Created, + node.Modified, + node.Accessed, + node.Changed); + } + + public FilesystemNodeId? Lookup(FilesystemNodeId parentDirectory, string name) { + ArgumentNullException.ThrowIfNull(name); + ThrowIfDisposed(); + var parent = RequireNode(parentDirectory); + if (parent.Kind != FilesystemNodeKind.Directory) throw new DirectoryNotFoundException(parent.Name); + if (!_children.TryGetValue(parentDirectory, out var children)) return null; + var exact = children.FirstOrDefault(child => string.Equals(child.Name, name, StringComparison.Ordinal)); + if (exact != null) return exact.Node.NodeId; + if ((Profile.Capabilities & FilesystemDriverCapabilities.CaseSensitiveNames) != 0) + return null; + var folded = children.Where(child => string.Equals(child.Name, name, StringComparison.OrdinalIgnoreCase)).ToArray(); + return folded.Length == 1 ? folded[0].Node.NodeId : null; + } + + public IReadOnlyList Enumerate(FilesystemNodeId directory) { + ThrowIfDisposed(); + var node = RequireNode(directory); + if (node.Kind != FilesystemNodeKind.Directory) throw new DirectoryNotFoundException(node.Name); + return (_children.TryGetValue(directory, out var children) ? children : []) + .Select(child => new FilesystemDirectoryEntry(child.Name, child.Node.NodeId, child.Node.Kind)) + .ToArray(); + } + + public IFilesystemFileHandle OpenFile(FilesystemNodeId nodeId, FileAccess access) { + ThrowIfDisposed(); + if (access != FileAccess.Read) throw ReadOnly(); + var node = RequireNode(nodeId); + if (node.Kind != FilesystemNodeKind.RegularFile) + throw new UnauthorizedAccessException($"'{node.Name}' is not a regular file."); + return node.OpenReadHandle?.Invoke() + ?? throw new NotSupportedException($"Filesystem node '{node.Name}' has no native data handle."); + } + + public FilesystemNodeId CreateFile(FilesystemNodeId parentDirectory, string name) => throw ReadOnly(); + public FilesystemNodeId CreateDirectory(FilesystemNodeId parentDirectory, string name) => throw ReadOnly(); + public void DeleteFile(FilesystemNodeId parentDirectory, string name) => throw ReadOnly(); + public void RemoveDirectory(FilesystemNodeId parentDirectory, string name) => throw ReadOnly(); + public void Rename(FilesystemNodeId oldParent, string oldName, FilesystemNodeId newParent, string newName, bool replace) => throw ReadOnly(); + public void CreateHardLink(FilesystemNodeId existingNode, FilesystemNodeId newParent, string newName) => throw ReadOnly(); + public FilesystemNodeId CreateSymbolicLink(FilesystemNodeId parentDirectory, string name, string target) => throw ReadOnly(); + + public string ReadSymbolicLink(FilesystemNodeId nodeId) { + ThrowIfDisposed(); + var node = RequireNode(nodeId); + if (node.Kind != FilesystemNodeKind.SymbolicLink || node.SymbolicLinkTarget == null) + throw new InvalidOperationException($"'{node.Name}' is not a decoded symbolic link."); + return node.SymbolicLinkTarget; + } + + public void SetMetadata(FilesystemNodeId nodeId, FilesystemMetadataPatch patch) => throw ReadOnly(); + public void Flush() => ThrowIfDisposed(); + public IFilesystemTransaction BeginTransaction() + => throw new NotSupportedException("The native snapshot session is read-only and has no write transaction."); + public void Dispose() => _disposed = true; + + private FilesystemSnapshotNode RequireNode(FilesystemNodeId nodeId) + => _nodes.TryGetValue(nodeId, out var node) + ? node + : throw new FileNotFoundException($"Filesystem node {nodeId.Value}:{nodeId.Generation} does not exist in this session."); + + private static SnapshotInput Prepare( + IEnumerable nodes, + FilesystemNodeId rootNodeId) { + ArgumentNullException.ThrowIfNull(nodes); + var materialized = nodes.ToArray(); + var entries = materialized + .Where(node => node.NodeId != rootNodeId) + .Select(node => new FilesystemSnapshotDirectoryEntry(node.ParentNodeId, node.Name, node.NodeId)) + .ToArray(); + return new SnapshotInput(materialized, entries); + } + + private static NotSupportedException ReadOnly() + => new("This native filesystem snapshot session is read-only."); + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} diff --git a/Compression.Registry/RebuildVerb.cs b/Compression.Registry/RebuildVerb.cs index c1cb3f17a..9c6e3d297 100644 --- a/Compression.Registry/RebuildVerb.cs +++ b/Compression.Registry/RebuildVerb.cs @@ -1,49 +1,111 @@ namespace Compression.Registry; /// -/// Generic, round-trip-verified "extract → re-create" engine shared by the -/// default implementations of the maintenance verbs (shrink, defragment) for -/// any descriptor that can both enumerate/extract () -/// and create () its format. -/// -/// Every rebuild is verified: the freshly created image is listed -/// back and its live-file count compared against the source. If the rebuild -/// would drop files, the operation throws -/// instead of producing a lossy result — so enabling a verb on a format whose -/// create path doesn't faithfully round-trip fails loudly rather than silently -/// corrupting data. This is what makes broad, default-implementation rollout -/// across filesystems safe. +/// Generic, round-trip-verified extract → re-create engine shared by maintenance +/// verbs. Rebuilds are staged, verified, progress-reporting, and cancellable; +/// the caller's original stream is not touched until the staged target has been +/// built successfully and cancellation is no longer accepted. /// public static class RebuildVerb { /// - /// Extracts every entry of and re-creates the image - /// into via . Returns the - /// source live-file count. Throws if the rebuilt image lists fewer live files - /// than the source (lossy round-trip) — the caller's - /// should be discarded in that case. + /// Extracts every live entry, re-creates the container in , + /// verifies the exact live-name multiset, and reports block-map/read/write-head + /// progress suitable for the maintenance UI. /// - public static int RebuildToStream(Stream input, Stream output, - IArchiveFormatOperations ops, IArchiveCreatable creator, + public static int RebuildToStream( + Stream input, + Stream output, + IArchiveFormatOperations ops, + IArchiveCreatable creator, IReadOnlyDictionary? formatSpecific = null, - IReadOnlySet? syntheticNames = null) { + IReadOnlySet? syntheticNames = null, + Action? onProgress = null, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(output); ArgumentNullException.ThrowIfNull(ops); ArgumentNullException.ThrowIfNull(creator); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("Rebuild input must be readable and seekable.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("Rebuild output must be writable and seekable.", nameof(output)); + + cancellationToken.ThrowIfCancellationRequested(); + input.Position = 0; + var sourceEntries = ops.List(input, null); + var sourceNames = LiveNameList(sourceEntries); + var sourceFileCount = sourceNames.Count; + var sourceLength = Math.Max(1L, input.Length); + var liveEntries = sourceEntries + .Where(e => !e.IsDirectory && (syntheticNames == null || !syntheticNames.Contains(e.Name))) + .ToArray(); + var totalLogical = Math.Max(1L, liveEntries.Sum(e => Math.Max(0L, e.OriginalSize))); + var sourceLayout = BuildSourceLayout(input, ops, sourceEntries); + + onProgress?.Invoke(new DefragProgressEvent( + "scanning", 0, 0, -1, sourceLength, sourceLayout, + $"Scanning {sourceFileCount:N0} live entr{(sourceFileCount == 1 ? "y" : "ies")} before staged rebuild")); var tmpDir = Path.Combine(Path.GetTempPath(), "cwb_rebuild_" + Guid.NewGuid().ToString("N")[..8]); Directory.CreateDirectory(tmpDir); try { - // Capture the source's live entry names (as a sorted MULTISET — duplicates - // matter) as the descriptor itself reports them. This is the identity a - // faithful rebuild must reproduce exactly. - input.Position = 0; - var sourceNames = LiveNameList(ops, input); - var sourceFileCount = sourceNames.Count; + // Extract entry-by-entry rather than through the opaque bulk Extract call. + // This makes large WORM/re-layout passes cancellable and gives the UI an + // honest moving read head while bytes are consumed from the source. + long logicalDone = 0; + var liveIndex = 0; + foreach (var entry in sourceEntries) { + cancellationToken.ThrowIfCancellationRequested(); + var target = SafeExtractPath(tmpDir, entry.Name); + if (entry.IsDirectory) { + Directory.CreateDirectory(target); + continue; + } + if (syntheticNames != null && syntheticNames.Contains(entry.Name)) + continue; - input.Position = 0; - ops.Extract(input, tmpDir, null, null); + ++liveIndex; + var parent = Path.GetDirectoryName(target); + if (!string.IsNullOrEmpty(parent)) Directory.CreateDirectory(parent); + + var entrySize = Math.Max(0L, entry.OriginalSize); + onProgress?.Invoke(new DefragProgressEvent( + "reading", 0.45 * logicalDone / totalLogical, + ScaleOffset(logicalDone, totalLogical, sourceLength), -1, + sourceLength, HighlightEntry(sourceLayout, entry.Name), + $"Reading {liveIndex:N0}/{liveEntries.Length:N0}: {entry.Name}")); + + input.Position = 0; + using var src = ops.OpenEntry(input, entry.Name, null); + using var dst = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 64 * 1024, + FileOptions.SequentialScan); + var buffer = new byte[64 * 1024]; + long entryDone = 0; + var lastReport = Environment.TickCount64; + while (true) { + cancellationToken.ThrowIfCancellationRequested(); + var read = src.Read(buffer, 0, buffer.Length); + if (read <= 0) break; + dst.Write(buffer, 0, read); + entryDone += read; + var now = Environment.TickCount64; + if (now - lastReport >= 50) { + lastReport = now; + var effective = logicalDone + (entrySize > 0 ? Math.Min(entrySize, entryDone) : entryDone); + var fraction = 0.45 * Math.Clamp((double)effective / totalLogical, 0, 1); + onProgress?.Invoke(new DefragProgressEvent( + "reading", fraction, + ScaleOffset(effective, totalLogical, sourceLength), -1, + sourceLength, null, + $"Reading {liveIndex:N0}/{liveEntries.Length:N0}: {entry.Name} ({entryDone:N0} bytes)")); + } + } + dst.Flush(flushToDisk: true); + logicalDone += entrySize > 0 ? entrySize : entryDone; + } + + cancellationToken.ThrowIfCancellationRequested(); var inputs = new List(); foreach (var dir in Directory.GetDirectories(tmpDir, "*", SearchOption.AllDirectories)) { @@ -52,28 +114,37 @@ public static int RebuildToStream(Stream input, Stream output, } foreach (var file in Directory.GetFiles(tmpDir, "*", SearchOption.AllDirectories)) { var rel = Path.GetRelativePath(tmpDir, file).Replace('\\', '/'); - // A reader that also surfaces the raw image and a metadata sheet must - // not have them written back as files: the rebuilt volume would carry - // a copy of its own previous self, and its reader would surface fresh - // synthetic entries on top of them. - if (syntheticNames != null && syntheticNames.Contains(rel)) continue; inputs.Add(new ArchiveInputInfo(file, rel, false)); } + var visualSize = Math.Max(sourceLength, totalLogical); + var targetLayout = BuildWeightedLayout(liveEntries, visualSize); + onProgress?.Invoke(new DefragProgressEvent( + "writing", 0.45, -1, 0, visualSize, targetLayout, + "Building staged target — original container is still unchanged")); + var options = new FormatCreateOptions { FormatSpecific = formatSpecific }; output.Position = 0; output.SetLength(0); - creator.Create(output, inputs, options); + using (var progressOutput = new ProgressWriteStream(output, cancellationToken, maxPosition => { + var fraction = 0.45 + 0.45 * Math.Clamp((double)maxPosition / sourceLength, 0, 1); + onProgress?.Invoke(new DefragProgressEvent( + "writing", fraction, -1, maxPosition, visualSize, null, + $"Writing staged target: {maxPosition:N0} bytes")); + })) { + creator.Create(progressOutput, inputs, options); + progressOutput.Flush(); + } + + cancellationToken.ThrowIfCancellationRequested(); + onProgress?.Invoke(new DefragProgressEvent( + "verifying", 0.92, -1, Math.Max(0, output.Position), Math.Max(1, output.Length), null, + "Verifying rebuilt container before commit")); - // Identity guard: a faithful rebuild must list back the EXACT same set of - // live entry names. Anything else — dropped, duplicated, renamed (e.g. - // content-addressed/hashed names, synthetic metadata entries the writer - // re-derives) — means the round-trip isn't identity-preserving, so refuse - // to commit and leave the caller to fall back / keep the original. output.Position = 0; List rebuiltNames; try { - rebuiltNames = LiveNameList(ops, output); + rebuiltNames = LiveNameList(ops.List(output, null)); } catch (Exception ex) { throw new InvalidOperationException( $"Rebuilt image could not be listed back ({ex.GetType().Name}: {ex.Message}); refusing a lossy rebuild.", ex); @@ -82,6 +153,12 @@ public static int RebuildToStream(Stream input, Stream output, throw new InvalidOperationException( $"Rebuild changed the entry set ({sourceFileCount} → {rebuiltNames.Count}); refusing a non-identity-preserving rebuild."); + cancellationToken.ThrowIfCancellationRequested(); + var finalLength = Math.Max(1L, output.Length); + onProgress?.Invoke(new DefragProgressEvent( + "staged", 0.98, -1, Math.Max(0, output.Length - 1), finalLength, + BuildWeightedLayout(liveEntries, finalLength), + "Staged rebuild verified; ready to commit")); output.Position = 0; return sourceFileCount; } finally { @@ -89,41 +166,49 @@ public static int RebuildToStream(Stream input, Stream output, } } - /// The sorted multiset of live (non-directory) entry names the descriptor lists. - private static List LiveNameList(IArchiveFormatOperations ops, Stream stream) { - stream.Position = 0; - return ops.List(stream, null).Where(e => !e.IsDirectory).Select(e => e.Name) - .OrderBy(n => n, StringComparer.Ordinal).ToList(); - } - /// - /// In-place rebuild: re-creates from its own - /// contents (consolidating live data — the defragmentation side effect of the - /// rebuild-via-WORM pattern) and overwrites the stream only when the rebuild - /// is verified to round-trip. On any failure the original bytes are left - /// untouched. + /// Rebuilds into a scratch file, verifies it, and only then replaces the + /// caller-supplied stream. Cancellation is honoured until commit starts; + /// once commit begins it runs to completion so a cancellation cannot leave + /// the original half-overwritten. /// - public static void RebuildInPlace(Stream archive, - IArchiveFormatOperations ops, IArchiveCreatable creator, - IReadOnlyDictionary? formatSpecific = null) { + public static void RebuildInPlace( + Stream archive, + IArchiveFormatOperations ops, + IArchiveCreatable creator, + IReadOnlyDictionary? formatSpecific = null, + Action? onProgress = null, + CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(archive); - // A scratch file, not a MemoryStream: the rebuilt image is a whole volume, - // and a MemoryStream cannot hold one past 2 GB ("Stream was too long"). using var rebuilt = CreateScratchStream(); - // Throws (leaving `archive` untouched) if the rebuild would lose data. - RebuildToStream(archive, rebuilt, ops, creator, formatSpecific); + RebuildToStream(archive, rebuilt, ops, creator, formatSpecific, + onProgress: onProgress, cancellationToken: cancellationToken); + + // Point of no return. Do not inspect cancellation again after announcing + // commit; callers disable Cancel for this phase. + onProgress?.Invoke(new DefragProgressEvent( + "committing", 0.99, -1, 0, Math.Max(1, rebuilt.Length), null, + "Committing verified staged target — cancellation is no longer safe")); + archive.Position = 0; archive.SetLength(0); rebuilt.Position = 0; rebuilt.CopyTo(archive); archive.Flush(); + + var finalLength = Math.Max(1L, archive.Length); + archive.Position = 0; + var entries = ops.List(archive, null); + onProgress?.Invoke(new DefragProgressEvent( + "complete", 1, -1, -1, finalLength, + BuildWeightedLayout(entries.Where(e => !e.IsDirectory).ToArray(), finalLength), + "Rebuild committed successfully")); } /// - /// Rebuild-based in-place edit shared by the default : - /// extract the archive, apply to the extracted file - /// tree (add/overwrite/delete real files on disk), re-create the image, and - /// overwrite the stream. The original bytes are left untouched on any failure. + /// Rebuild-based edit used by the generic modifier. Mutation and validation + /// happen off to the side; the original is overwritten only after a valid + /// staged result exists. /// public static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, Action mutate) { @@ -134,7 +219,6 @@ public static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, try { archive.Position = 0; ops.Extract(archive, tmpDir, null, null); - mutate(tmpDir); var inputs = new List(); @@ -149,8 +233,6 @@ public static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, using var rebuilt = CreateScratchStream(); creator.Create(rebuilt, inputs, new FormatCreateOptions()); - - // Verify the result lists back (a valid image) before committing. rebuilt.Position = 0; _ = ops.List(rebuilt, null); @@ -163,13 +245,204 @@ public static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, try { Directory.Delete(tmpDir, true); } catch { /* best effort */ } } } + /// - /// A writable scratch stream that is not bounded by what a byte[] can hold. - /// Deleted on close. + /// Transactional purge for a mutable container. The modifier operates on a + /// staged copy and the caller's stream is replaced only after the result lists + /// successfully with every original live entry gone. /// + public static void PurgeViaModifier(Stream archive, IArchiveFormatOperations ops, IArchiveModifiable modifier) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(ops); + ArgumentNullException.ThrowIfNull(modifier); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Purge requires a readable, writable, seekable stream.", nameof(archive)); + + using var staged = CreateScratchStream(); + archive.Position = 0; + archive.CopyTo(staged); + staged.Flush(); + + staged.Position = 0; + var sourceNames = ops.List(staged, null) + .Where(e => !e.IsDirectory) + .Select(e => e.Name) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (sourceNames.Length == 0) return; + + staged.Position = 0; + modifier.Remove(staged, sourceNames); + staged.Position = 0; + var remaining = ops.List(staged, null) + .Where(e => !e.IsDirectory) + .Select(e => e.Name) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var survivors = sourceNames.Where(remaining.Contains).ToArray(); + if (survivors.Length != 0) + throw new InvalidOperationException( + $"Purge left {survivors.Length} original live entr{(survivors.Length == 1 ? "y" : "ies")} behind; original container retained."); + + archive.Position = 0; + archive.SetLength(0); + staged.Position = 0; + staged.CopyTo(archive); + archive.Flush(); + } + + private static List LiveNameList(IEnumerable entries) + => entries.Where(e => !e.IsDirectory).Select(e => e.Name) + .OrderBy(n => n, StringComparer.Ordinal).ToList(); + + private static IReadOnlyList BuildSourceLayout( + Stream input, IArchiveFormatOperations ops, IReadOnlyList entries) { + try { + input.Position = 0; + var extents = ops switch { + IFilesystemExtentMap fs => fs.EnumerateExtents(input).ToArray(), + IArchiveLayoutMap archive => archive.EnumerateLayout(input).ToArray(), + _ => [], + }; + if (extents.Length > 0) return extents; + } catch { + // Visualization is best-effort; the actual rebuild remains fully verified. + } + return BuildWeightedLayout(entries.Where(e => !e.IsDirectory).ToArray(), Math.Max(1, input.Length)); + } + + private static IReadOnlyList BuildWeightedLayout( + IReadOnlyList entries, long totalSize) { + totalSize = Math.Max(1, totalSize); + if (entries.Count == 0) + return [new DefragBlockInfo(0, totalSize, DefragBlockKind.Free)]; + + var weights = entries.Select(e => Math.Max(1L, e.OriginalSize)).ToArray(); + var totalWeight = Math.Max(1L, weights.Sum()); + var result = new List(entries.Count); + long cumulative = 0; + long cursor = 0; + for (var i = 0; i < entries.Count; i++) { + cumulative += weights[i]; + var end = i == entries.Count - 1 + ? totalSize + : (long)((double)cumulative / totalWeight * totalSize); + end = Math.Clamp(end, cursor, totalSize); + var length = end - cursor; + if (length > 0) + result.Add(new DefragBlockInfo(cursor, length, DefragBlockKind.Used, + entries[i].Name, Classify(entries[i].Method))); + cursor = end; + } + if (cursor < totalSize) + result.Add(new DefragBlockInfo(cursor, totalSize - cursor, DefragBlockKind.Free)); + return result; + } + + private static IReadOnlyList HighlightEntry( + IReadOnlyList source, string entryName) { + var changed = false; + var result = new DefragBlockInfo[source.Count]; + for (var i = 0; i < source.Count; i++) { + var block = source[i]; + if (block.FileName != null && string.Equals(block.FileName, entryName, StringComparison.Ordinal)) { + result[i] = block with { Kind = DefragBlockKind.InProgress }; + changed = true; + } else { + result[i] = block; + } + } + return changed ? result : source; + } + + private static DefragBlockClass Classify(string? method) { + var value = (method ?? "").ToUpperInvariant(); + if (value.Contains("STORE") || value.Contains("COPY") || value == "NONE" || value.Length == 0) + return DefragBlockClass.Frozen; + if (value.Contains("LZMA") || value.Contains("PPMD") || value.Contains("BZIP")) + return DefragBlockClass.Hot; + if (value.Contains("ZSTD") || value.Contains("LZ4")) + return DefragBlockClass.Cold; + return DefragBlockClass.Normal; + } + + private static long ScaleOffset(long done, long total, long imageSize) + => total <= 0 ? 0 : Math.Clamp((long)((double)done / total * imageSize), 0, Math.Max(0, imageSize - 1)); + + private static string SafeExtractPath(string root, string archiveName) { + var normalized = archiveName.Replace('/', Path.DirectorySeparatorChar).TrimStart(Path.DirectorySeparatorChar); + var rootFull = Path.GetFullPath(root) + Path.DirectorySeparatorChar; + var candidate = Path.GetFullPath(Path.Combine(root, normalized)); + if (!candidate.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase) && + !string.Equals(candidate, rootFull.TrimEnd(Path.DirectorySeparatorChar), StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Entry path escapes rebuild staging directory: {archiveName}"); + return candidate; + } + + /// A writable scratch stream not bounded by byte[] / MemoryStream size. internal static FileStream CreateScratchStream() => new(Path.Combine(Path.GetTempPath(), "cwb_rebuild_" + Guid.NewGuid().ToString("N") + ".tmp"), - FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, - FileOptions.DeleteOnClose); + FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.DeleteOnClose); + /// + /// Write-through wrapper used solely to expose actual target-byte progress and + /// make long encoder writes cancellable without giving ownership of the target + /// stream to the wrapper. + /// + private sealed class ProgressWriteStream( + Stream inner, CancellationToken cancellationToken, Action report) : Stream { + private long _maxPosition; + private long _lastReportTick; + + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => inner.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) { + cancellationToken.ThrowIfCancellationRequested(); + inner.Write(buffer, offset, count); + Report(); + } + + public override void Write(ReadOnlySpan buffer) { + cancellationToken.ThrowIfCancellationRequested(); + inner.Write(buffer); + Report(); + } + + public override void WriteByte(byte value) { + cancellationToken.ThrowIfCancellationRequested(); + inner.WriteByte(value); + Report(); + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, + CancellationToken cancellationTokenFromCaller = default) { + cancellationToken.ThrowIfCancellationRequested(); + cancellationTokenFromCaller.ThrowIfCancellationRequested(); + await inner.WriteAsync(buffer, cancellationTokenFromCaller).ConfigureAwait(false); + Report(); + } + + private void Report() { + _maxPosition = Math.Max(_maxPosition, inner.Position); + var now = Environment.TickCount64; + if (now - _lastReportTick < 50) return; + _lastReportTick = now; + report(_maxPosition); + } + + protected override void Dispose(bool disposing) { + if (disposing) { + try { inner.Flush(); } catch { } + report(Math.Max(_maxPosition, inner.CanSeek ? inner.Position : _maxPosition)); + } + base.Dispose(disposing); + } + } } diff --git a/Compression.Registry/SpoolingReadOnlyFileHandle.cs b/Compression.Registry/SpoolingReadOnlyFileHandle.cs new file mode 100644 index 000000000..068ceb785 --- /dev/null +++ b/Compression.Registry/SpoolingReadOnlyFileHandle.cs @@ -0,0 +1,105 @@ +#pragma warning disable CS1591 + +namespace Compression.Registry; + +/// +/// Transitional positional handle for native filesystem readers that can stream +/// a file correctly but do not yet expose a seekable block/extent map. Small +/// files stay in memory; large files are spooled to a delete-on-close temporary +/// file. This preserves driver-style positional reads without imposing a whole- +/// file RAM ceiling while the filesystem's direct block mapping is implemented. +/// +public sealed class SpoolingReadOnlyFileHandle : IFilesystemFileHandle { + public const long DefaultMemoryThreshold = 8L * 1024 * 1024; + + private readonly Stream _spool; + private readonly object _gate = new(); + private readonly long _length; + private bool _disposed; + + private SpoolingReadOnlyFileHandle(FilesystemNodeId nodeId, Stream spool, long length) { + NodeId = nodeId; + _spool = spool; + _length = length; + } + + public static SpoolingReadOnlyFileHandle Create( + FilesystemNodeId nodeId, + long expectedLength, + Action writeContent, + long memoryThreshold = DefaultMemoryThreshold) { + ArgumentNullException.ThrowIfNull(writeContent); + if (expectedLength < 0) throw new ArgumentOutOfRangeException(nameof(expectedLength)); + if (memoryThreshold < 0) throw new ArgumentOutOfRangeException(nameof(memoryThreshold)); + + Stream spool; + if (expectedLength <= memoryThreshold && expectedLength <= int.MaxValue) { + spool = new MemoryStream(checked((int)expectedLength)); + } else { + var path = Path.Combine(Path.GetTempPath(), "cwb_handle_" + Guid.NewGuid().ToString("N") + ".tmp"); + spool = new FileStream( + path, + FileMode.CreateNew, + FileAccess.ReadWrite, + FileShare.Read | FileShare.Delete, + 64 * 1024, + FileOptions.RandomAccess | FileOptions.DeleteOnClose); + } + + try { + writeContent(spool); + if (spool.Length < expectedLength) + throw new InvalidDataException( + $"Filesystem reader produced {spool.Length:N0} bytes for a {expectedLength:N0}-byte file."); + if (spool.Length > expectedLength) spool.SetLength(expectedLength); + spool.Position = 0; + return new SpoolingReadOnlyFileHandle(nodeId, spool, expectedLength); + } catch { + spool.Dispose(); + throw; + } + } + + public FilesystemNodeId NodeId { get; } + public long Length { + get { + ThrowIfDisposed(); + return _length; + } + } + + public int Read(long offset, Span destination) { + ThrowIfDisposed(); + if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset)); + if (destination.Length == 0 || offset >= _length) return 0; + var count = checked((int)Math.Min(destination.Length, _length - offset)); + lock (_gate) { + _spool.Position = offset; + var total = 0; + while (total < count) { + var read = _spool.Read(destination.Slice(total, count - total)); + if (read == 0) break; + total += read; + } + if (total != count) + throw new EndOfStreamException("The filesystem spool ended before the advertised logical file length."); + return total; + } + } + + public void Write(long offset, ReadOnlySpan source) + => throw new NotSupportedException("The spooled filesystem handle is read-only."); + + public void SetLength(long length) + => throw new NotSupportedException("The spooled filesystem handle is read-only."); + + public void Flush() => ThrowIfDisposed(); + + public void Dispose() { + if (_disposed) return; + _disposed = true; + _spool.Dispose(); + } + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} diff --git a/Compression.Registry/Streaming/TemporaryExtractedEntryStream.cs b/Compression.Registry/Streaming/TemporaryExtractedEntryStream.cs new file mode 100644 index 000000000..3694209e3 --- /dev/null +++ b/Compression.Registry/Streaming/TemporaryExtractedEntryStream.cs @@ -0,0 +1,128 @@ +namespace Compression.Registry.Streaming; + +/// +/// Seekable read-only stream over an entry extracted into an isolated temporary +/// directory. The directory tree is removed when the stream is disposed. +/// +/// This is the default bridge for archive/filesystem descriptors that have not +/// yet implemented a native OpenEntry. Unlike the historical byte-array +/// fallback it has no Array.MaxLength / whole-file-RAM ceiling and therefore is +/// safe to use beneath filesystem-driver positional spooling. +/// +internal sealed class TemporaryExtractedEntryStream : Stream { + private readonly FileStream _inner; + private readonly string _temporaryDirectory; + private bool _disposed; + + private TemporaryExtractedEntryStream(FileStream inner, string temporaryDirectory) { + _inner = inner; + _temporaryDirectory = temporaryDirectory; + } + + public static TemporaryExtractedEntryStream Open( + IArchiveFormatOperations operations, + Stream archive, + string entryName, + string? password) { + ArgumentNullException.ThrowIfNull(operations); + ArgumentNullException.ThrowIfNull(archive); + ArgumentException.ThrowIfNullOrWhiteSpace(entryName); + + var tempDir = Path.Combine(Path.GetTempPath(), "cwb_entry_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try { + if (archive.CanSeek) archive.Position = 0; + operations.Extract(archive, tempDir, password, [entryName]); + + var wanted = Normalize(entryName); + string? path = null; + foreach (var candidate in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) { + var relative = Normalize(Path.GetRelativePath(tempDir, candidate)); + if (!relative.Equals(wanted, StringComparison.OrdinalIgnoreCase) && + !Path.GetFileName(relative).Equals(Path.GetFileName(wanted), StringComparison.OrdinalIgnoreCase)) + continue; + if (path != null && !relative.Equals(wanted, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Extraction of '{entryName}' produced multiple ambiguous files."); + path = candidate; + if (relative.Equals(wanted, StringComparison.OrdinalIgnoreCase)) break; + } + + if (path == null) + throw new FileNotFoundException( + $"Archive extraction did not materialize requested entry '{entryName}'.", entryName); + + var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read | FileShare.Delete, + 64 * 1024, + FileOptions.RandomAccess); + return new TemporaryExtractedEntryStream(stream, tempDir); + } catch { + TryDelete(tempDir); + throw; + } + } + + public override bool CanRead => !_disposed && _inner.CanRead; + public override bool CanSeek => !_disposed && _inner.CanSeek; + public override bool CanWrite => false; + public override long Length { get { ThrowIfDisposed(); return _inner.Length; } } + public override long Position { + get { ThrowIfDisposed(); return _inner.Position; } + set { ThrowIfDisposed(); _inner.Position = value; } + } + + public override int Read(byte[] buffer, int offset, int count) { + ThrowIfDisposed(); + return _inner.Read(buffer, offset, count); + } + + public override int Read(Span buffer) { + ThrowIfDisposed(); + return _inner.Read(buffer); + } + + public override int ReadByte() { + ThrowIfDisposed(); + return _inner.ReadByte(); + } + + public override long Seek(long offset, SeekOrigin origin) { + ThrowIfDisposed(); + return _inner.Seek(offset, origin); + } + + public override void Flush() => ThrowIfDisposed(); + public override void SetLength(long value) => throw new NotSupportedException("Temporary extracted entry streams are read-only."); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException("Temporary extracted entry streams are read-only."); + public override void Write(ReadOnlySpan buffer) => throw new NotSupportedException("Temporary extracted entry streams are read-only."); + + protected override void Dispose(bool disposing) { + if (_disposed) { + base.Dispose(disposing); + return; + } + _disposed = true; + if (disposing) { + _inner.Dispose(); + TryDelete(_temporaryDirectory); + } + base.Dispose(disposing); + } + + private static string Normalize(string value) + => value.Replace('\\', '/').TrimStart('/'); + + private static void TryDelete(string directory) { + try { + if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true); + } catch { + // Best-effort temporary cleanup. The OS/temp cleaner may remove an entry + // held by an external scanner after our file handle has already closed. + } + } + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); +} diff --git a/Compression.Tests/Btrfs/BtrfsFilesystemDriverTests.cs b/Compression.Tests/Btrfs/BtrfsFilesystemDriverTests.cs new file mode 100644 index 000000000..62dad99ed --- /dev/null +++ b/Compression.Tests/Btrfs/BtrfsFilesystemDriverTests.cs @@ -0,0 +1,84 @@ +using Compression.Registry; +using FileSystem.Btrfs; + +namespace Compression.Tests.Btrfs; + +[TestFixture] +public sealed class BtrfsFilesystemDriverTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void NativeSession_UsesInodeIdentityAndDirectPositionalReads() { + var payload = new byte[32 * 1024 + 503]; + for (var i = 0; i < payload.Length; i++) payload[i] = (byte)((i * 41 + 13) & 0xFF); + + var writer = new BtrfsWriter(); + writer.AddFile("dir/data.bin", payload); + writer.AddFile("dir/tiny.txt", "tiny"u8.ToArray()); // exercises an inline extent too + using var image = new MemoryStream(); + writer.WriteTo(image); + + var adapter = new BtrfsFilesystemDriverAdapter(); + image.Position = 0; + var profile = adapter.ProbeFilesystem(image); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.False); + Assert.That(profile.Capabilities.HasFlag(FilesystemDriverCapabilities.StableNodeIds), Is.True); + Assert.That(profile.Capabilities.HasFlag(FilesystemDriverCapabilities.SparseFiles), Is.True); + + image.Position = 0; + using var session = adapter.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + Assert.That(session.RootNodeId.Value, Is.EqualTo(256UL)); + var dir = session.Lookup(session.RootNodeId, "dir"); + Assert.That(dir, Is.Not.Null); + var file = session.Lookup(dir!.Value, "data.bin"); + Assert.That(file, Is.Not.Null); + Assert.That(file!.Value.Value, Is.GreaterThan(256UL), "file identity must be the native Btrfs inode object id"); + + using var handle = session.OpenFile(file.Value, FileAccess.Read); + var slice = new byte[911]; + var read = handle.Read(7_777, slice); + Assert.That(read, Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(payload.AsSpan(7_777, slice.Length).ToArray())); + + var tiny = session.Lookup(dir.Value, "tiny.txt"); + Assert.That(tiny, Is.Not.Null); + using var tinyHandle = session.OpenFile(tiny!.Value, FileAccess.Read); + var tinyBytes = new byte[4]; + Assert.That(tinyHandle.Read(0, tinyBytes), Is.EqualTo(4)); + Assert.That(tinyBytes, Is.EqualTo("tiny"u8.ToArray())); + } + + [Test, Category("ErrorHandling")] + public void Probe_RejectsCompressedExtentInsteadOfSilentlyShorteningFile() { + var inline = Enumerable.Range(0, 31).Select(i => (byte)(0xA1 + i)).ToArray(); + var writer = new BtrfsWriter(); + writer.AddFile("compressed-marker.bin", inline); + using var built = new MemoryStream(); + writer.WriteTo(built); + var bytes = built.ToArray(); + + var payloadAt = bytes.AsSpan().IndexOf(inline); + Assert.That(payloadAt, Is.GreaterThanOrEqualTo(21), "test payload must be present as one inline EXTENT_DATA value"); + Assert.That(bytes.AsSpan(payloadAt + inline.Length).IndexOf(inline), Is.EqualTo(-1), "payload marker must be unique in the image"); + + // Inline payload starts at file_extent_item + 21; compression is byte 16. + bytes[payloadAt - 5] = 1; // non-zero compression id: current native profile must reject it + using var image = new MemoryStream(bytes, writable: false); + var profile = new BtrfsFilesystemDriverAdapter().ProbeFilesystem(image); + + Assert.That(profile.CanMount, Is.False); + Assert.That(string.Join("; ", profile.Limitations), Does.Contain("compression=")); + } + + [Test, Category("ErrorHandling")] + public void NativeSession_RefusesWritableMount() { + var writer = new BtrfsWriter(); + writer.AddFile("a.bin", "abc"u8.ToArray()); + using var image = new MemoryStream(); + writer.WriteTo(image); + + var adapter = new BtrfsFilesystemDriverAdapter(); + image.Position = 0; + Assert.Throws(() => + adapter.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))); + } +} diff --git a/Compression.Tests/CbmNibble/CbmNibbleFilesystemDriverTests.cs b/Compression.Tests/CbmNibble/CbmNibbleFilesystemDriverTests.cs new file mode 100644 index 000000000..8641bc9e4 --- /dev/null +++ b/Compression.Tests/CbmNibble/CbmNibbleFilesystemDriverTests.cs @@ -0,0 +1,126 @@ +using Compression.Registry; +using FileSystem.CbmNibble; + +namespace Compression.Tests.CbmNibble; + +[TestFixture] +public sealed class CbmNibbleFilesystemDriverTests { + [TestCase(false)] + [TestCase(true)] + [Category("Driver")] + public void CanonicalNibbleImage_SupportsFilesystemRoundTrip(bool nib) { + var original = Enumerable.Range(0, 900).Select(i => (byte)(i * 29)).ToArray(); + var writer = new CbmNibbleWriter(); + writer.AddFile("HELLO", original); + var bytes = nib ? writer.BuildNib() : writer.Build(); + + using var image = new MemoryStream(); + image.Write(bytes); + image.Position = 0; + + IFilesystemDriverProvider filesystem = nib ? new NibFormatDescriptor() : new G64FormatDescriptor(); + var beforeTracks = CbmNibbleReader.Read(image.ToArray(), nib ? "image.nib" : "image.g64"); + var untouchedBefore = beforeTracks.Tracks.Single(track => track.Index == 68).Data.ToArray(); + + var profile = filesystem.ProbeFilesystem(image); + Assert.Multiple(() => { + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.MutationModel, Is.EqualTo(FilesystemMutationModel.Direct)); + }); + + using (var fs = filesystem.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))) { + var root = fs.RootNodeId; + var helloId = fs.Lookup(root, "HELLO"); + Assert.That(helloId.HasValue, Is.True); + using (var hello = fs.OpenFile(helloId!.Value, FileAccess.Read)) { + var probe = new byte[111]; + Assert.That(hello.Read(257, probe), Is.EqualTo(probe.Length)); + Assert.That(probe, Is.EqualTo(original.AsSpan(257, probe.Length).ToArray())); + } + + var newId = fs.CreateFile(root, "MOUNTED"); + using (var handle = fs.OpenFile(newId, FileAccess.ReadWrite)) { + handle.Write(0, "filesystem-driver"u8); + handle.Write(32, "gcr"u8); + handle.SetLength(40); + } + fs.Rename(root, "MOUNTED", root, "RENAMED", replace: false); + fs.DeleteFile(root, "HELLO"); + fs.Flush(); + } + + var afterTracks = CbmNibbleReader.Read(image.ToArray(), nib ? "image.nib" : "image.g64"); + Assert.That(afterTracks.Tracks.Single(track => track.Index == 68).Data, Is.EqualTo(untouchedBefore), + "filesystem writes must not rewrite an unrelated track"); + + image.Position = 0; + using var reopened = filesystem.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + Assert.That(reopened.Lookup(reopened.RootNodeId, "HELLO"), Is.Null); + var renamedId = reopened.Lookup(reopened.RootNodeId, "RENAMED"); + Assert.That(renamedId.HasValue, Is.True); + using var renamed = reopened.OpenFile(renamedId!.Value, FileAccess.Read); + var result = new byte[40]; + Assert.That(renamed.Read(0, result), Is.EqualTo(result.Length)); + Assert.Multiple(() => { + Assert.That(result.AsSpan(0, 17).ToArray(), Is.EqualTo("filesystem-driver"u8.ToArray())); + Assert.That(result.AsSpan(17, 15).ToArray(), Is.EqualTo(new byte[15])); + Assert.That(result.AsSpan(32, 3).ToArray(), Is.EqualTo("gcr"u8.ToArray())); + Assert.That(result.AsSpan(35, 5).ToArray(), Is.EqualTo(new byte[5])); + }); + } + + [Test, Category("Driver"), Category("EdgeCase")] + public void G64SectorProjection_DecodesNonByteAlignedTrackRotation() { + var writer = new CbmNibbleWriter(); + writer.AddFile("HELLO", Enumerable.Range(0, 600).Select(i => (byte)i).ToArray()); + using var image = new MemoryStream(); + image.Write(writer.Build()); + image.Position = 0; + + using (var tracks = CbmNibbleRawTrackDevices.OpenG64(image, writable: true, leaveOpen: true)) { + var info = tracks.EnumerateTracks().Single(track => track.Index == 0); + var raw = new byte[(int)info.Length]; + Assert.That(tracks.ReadTrack(0, raw), Is.EqualTo(raw.Length)); + tracks.WriteTrack(0, RotateBits(raw, 3)); + tracks.Flush(); + } + + image.Position = 0; + var profile = new G64FormatDescriptor().ProbeFilesystem(image); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + } + + [Test, Category("Driver"), Category("EdgeCase")] + public void G64WritableMount_RejectsMeaningfulOddHalfTrack() { + var writer = new CbmNibbleWriter(); + writer.AddFile("HELLO", [1, 2, 3, 4]); + using var image = new MemoryStream(); + image.Write(writer.Build()); + image.Position = 0; + + using (var tracks = CbmNibbleRawTrackDevices.OpenG64(image, writable: true, leaveOpen: true)) { + tracks.WriteTrack(1, Enumerable.Repeat((byte)0xA5, 128).ToArray(), encodingParameter: 3); + tracks.Flush(); + } + + image.Position = 0; + var profile = new G64FormatDescriptor().ProbeFilesystem(image); + Assert.Multiple(() => { + Assert.That(profile.CanMount, Is.True, "standard whole tracks remain readable"); + Assert.That(profile.CanMountWritable, Is.False); + Assert.That(profile.Limitations.Any(x => x.Contains("half-track", StringComparison.OrdinalIgnoreCase)), Is.True); + }); + } + + private static byte[] RotateBits(byte[] source, int shift) { + var totalBits = source.Length * 8; + var result = new byte[source.Length]; + for (var destinationBit = 0; destinationBit < totalBits; ++destinationBit) { + var sourceBit = (destinationBit + shift) % totalBits; + var value = (source[sourceBit >> 3] >> (7 - (sourceBit & 7))) & 1; + result[destinationBit >> 3] |= (byte)(value << (7 - (destinationBit & 7))); + } + return result; + } +} diff --git a/Compression.Tests/CbmNibble/CbmNibbleModifyTests.cs b/Compression.Tests/CbmNibble/CbmNibbleModifyTests.cs new file mode 100644 index 000000000..afcdaddea --- /dev/null +++ b/Compression.Tests/CbmNibble/CbmNibbleModifyTests.cs @@ -0,0 +1,150 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileSystem.CbmNibble; + +namespace Compression.Tests.CbmNibble; + +[TestFixture] +public sealed class CbmNibbleModifyTests { + [Test, Category("RoundTrip")] + public void G64_DirectTracks_AddReplaceRemoveDefragAndPurge() { + var descriptor = new G64FormatDescriptor(); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var a = Enumerable.Range(0, 113).Select(i => (byte)(i * 17)).ToArray(); + var b = Enumerable.Range(0, 227).Select(i => (byte)(255 - i)).ToArray(); + using var image = new MemoryStream(); + descriptor.Create(image, [ + ArchiveInputInfo.InMemory("track_00.bin", a), + ArchiveInputInfo.InMemory("track_03.bin", b), + ], new FormatCreateOptions()); + + AssertTrack(descriptor, image, "track_00.bin", a); + AssertTrack(descriptor, image, "track_03.bin", b); + + var replacement = Enumerable.Repeat((byte)0xA5, 181).ToArray(); + var added = Enumerable.Repeat((byte)0x3C, 97).ToArray(); + descriptor.Add(image, [ + ArchiveInputInfo.InMemory("track_00.bin", replacement), + ArchiveInputInfo.InMemory("track_05.bin", added), + ]); + AssertTrack(descriptor, image, "track_00.bin", replacement); + AssertTrack(descriptor, image, "track_03.bin", b); + AssertTrack(descriptor, image, "track_05.bin", added); + + descriptor.Remove(image, ["track_03.bin"]); + var names = ListNames(descriptor, image); + Assert.That(names, Does.Not.Contain("track_03.bin")); + Assert.That(names, Does.Contain("track_00.bin")); + Assert.That(names, Does.Contain("track_05.bin")); + + descriptor.Defragment(image, new DefragOptions()); + AssertTrack(descriptor, image, "track_00.bin", replacement); + AssertTrack(descriptor, image, "track_05.bin", added); + + descriptor.Purge(image); + names = ListNames(descriptor, image); + Assert.That(names.Where(n => n.StartsWith("track_", StringComparison.Ordinal)), Is.Empty); + Assert.That(names, Does.Contain("metadata.ini")); + } + + [Test, Category("EdgeCase")] + public void G64_VariableSpeedMap_FailsClosedForMutationAndWipe() { + var descriptor = new G64FormatDescriptor(); + using var image = new MemoryStream(); + descriptor.Create(image, [ArchiveInputInfo.InMemory("track_00.bin", new byte[] { 1, 2, 3, 4 })], + new FormatCreateOptions()); + + var bytes = image.ToArray(); + var trackCount = bytes[9]; + var speedTable = 12 + trackCount * 4; + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(speedTable, 4), 0x100); + image.Position = 0; + image.SetLength(0); + image.Write(bytes); + + Assert.That(() => descriptor.Add(image, + [ArchiveInputInfo.InMemory("track_00.bin", new byte[] { 9, 8, 7 })]), + Throws.InstanceOf()); + + var before = image.ToArray(); + Assert.That(descriptor.WipeUnusedSpace(image), Is.EqualTo(0)); + Assert.That(image.ToArray(), Is.EqualTo(before)); + } + + [Test, Category("Maintenance")] + public void G64_Wipe_ClearsOnlyUnreferencedTrailingBytes() { + var descriptor = new G64FormatDescriptor(); + using var image = new MemoryStream(); + var track = Enumerable.Repeat((byte)0x55, 64).ToArray(); + descriptor.Create(image, [ArchiveInputInfo.InMemory("track_00.bin", track)], new FormatCreateOptions()); + var liveLength = image.Length; + image.Position = image.Length; + image.Write(Enumerable.Repeat((byte)0xCC, 128).ToArray()); + + var wiped = descriptor.WipeUnusedSpace(image); + Assert.That(wiped, Is.GreaterThanOrEqualTo(128)); + Assert.That(image.ToArray().AsSpan((int)liveLength, 128).ToArray(), Is.All.EqualTo((byte)0)); + AssertTrack(descriptor, image, "track_00.bin", track); + } + + [Test, Category("RoundTrip")] + public void Nib_DirectTracks_UseFixedSlotsForReplaceRemoveAndPurge() { + var descriptor = new NibFormatDescriptor(); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var a = Enumerable.Repeat((byte)0x55, CbmNibbleReader.NibTrackSize).ToArray(); + var b = Enumerable.Repeat((byte)0xAA, CbmNibbleReader.NibTrackSize).ToArray(); + using var image = new MemoryStream(); + descriptor.Create(image, [ + ArchiveInputInfo.InMemory("track_00.bin", a), + ArchiveInputInfo.InMemory("track_17.bin", b), + ], new FormatCreateOptions()); + + Assert.That(image.Length, Is.EqualTo(CbmNibbleReader.NibExpectedFileSize)); + AssertTrack(descriptor, image, "track_00.bin", a); + AssertTrack(descriptor, image, "track_17.bin", b); + + var replacement = Enumerable.Repeat((byte)0x6D, CbmNibbleReader.NibTrackSize).ToArray(); + descriptor.Add(image, [ArchiveInputInfo.InMemory("track_17.bin", replacement)]); + AssertTrack(descriptor, image, "track_00.bin", a); + AssertTrack(descriptor, image, "track_17.bin", replacement); + + var beforeDefrag = image.ToArray(); + descriptor.Defragment(image, new DefragOptions()); + Assert.That(image.ToArray(), Is.EqualTo(beforeDefrag), "Fixed-slot NIB defrag should be a true no-op."); + + descriptor.Remove(image, ["track_00.bin"]); + Assert.That(ListNames(descriptor, image), Does.Not.Contain("track_00.bin")); + AssertTrack(descriptor, image, "track_17.bin", replacement); + + descriptor.Purge(image); + Assert.That(image.ToArray(), Is.All.EqualTo((byte)0)); + Assert.That(ListNames(descriptor, image).Where(n => n.StartsWith("track_", StringComparison.Ordinal)), Is.Empty); + } + + [Test, Category("EdgeCase")] + public void Nib_RejectsNonSlotSizedTrackReplacementWithoutChangingImage() { + var descriptor = new NibFormatDescriptor(); + using var image = new MemoryStream(); + descriptor.Create(image, [], new FormatCreateOptions()); + var before = image.ToArray(); + + Assert.That(() => descriptor.Add(image, + [ArchiveInputInfo.InMemory("track_01.bin", new byte[123])]), + Throws.InstanceOf()); + Assert.That(image.ToArray(), Is.EqualTo(before)); + } + + private static HashSet ListNames(IArchiveFormatOperations descriptor, MemoryStream image) { + image.Position = 0; + return descriptor.List(image, null).Select(e => e.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + private static void AssertTrack(IArchiveFormatOperations descriptor, MemoryStream image, + string name, byte[] expected) { + image.Position = 0; + var actual = descriptor.ExtractEntryToMemory(image, name, null); + Assert.That(actual, Is.EqualTo(expected), name); + } +} diff --git a/Compression.Tests/CbmNibble/CbmNibbleRawTrackDeviceTests.cs b/Compression.Tests/CbmNibble/CbmNibbleRawTrackDeviceTests.cs new file mode 100644 index 000000000..24a688058 --- /dev/null +++ b/Compression.Tests/CbmNibble/CbmNibbleRawTrackDeviceTests.cs @@ -0,0 +1,75 @@ +using Compression.Registry; +using FileSystem.CbmNibble; + +namespace Compression.Tests.CbmNibble; + +[TestFixture] +public sealed class CbmNibbleRawTrackDeviceTests { + [Test, Category("Driver")] + public void NibTrackDevice_PerformsPositionalTrackWritesWithoutRebuildingOtherSlots() { + using var image = new MemoryStream(new byte[CbmNibbleReader.NibExpectedFileSize], writable: true); + var first = Enumerable.Repeat((byte)0x44, CbmNibbleReader.NibTrackSize).ToArray(); + var second = Enumerable.Repeat((byte)0x99, CbmNibbleReader.NibTrackSize).ToArray(); + + using (var device = CbmNibbleRawTrackDevices.OpenNib(image, writable: true)) { + Assert.That(device.TrackCount, Is.EqualTo(84)); + device.WriteTrack(2, first); + device.WriteTrack(40, second); + device.Flush(); + + var buffer = new byte[CbmNibbleReader.NibTrackSize]; + Assert.That(device.ReadTrack(2, buffer), Is.EqualTo(buffer.Length)); + Assert.That(buffer, Is.EqualTo(first)); + device.ClearTrack(2); + Assert.That(device.ReadTrack(2, buffer), Is.EqualTo(0)); + Assert.That(device.ReadTrack(40, buffer), Is.EqualTo(buffer.Length)); + Assert.That(buffer, Is.EqualTo(second)); + } + } + + [Test, Category("Driver")] + public void G64TrackDevice_PreservesStableTrackIndexesAcrossCommit() { + var initial = CbmNibbleWriter.BuildG64FromTracks([ + new CbmNibbleReader.Track(0, new byte[] { 1, 2, 3 }, 3), + new CbmNibbleReader.Track(4, new byte[] { 7, 8, 9 }, 3), + ], trackCount: 6); + using var image = new MemoryStream(); + image.Write(initial); + image.Position = 0; + + using (var device = CbmNibbleRawTrackDevices.OpenG64(image, writable: true)) { + var replacement = new byte[] { 9, 9, 9, 9, 9 }; + device.WriteTrack(0, replacement); + device.WriteTrack(5, new byte[] { 0x55, 0xAA }, encodingParameter: 3); + device.ClearTrack(4); + device.Flush(); + + var buffer = new byte[64]; + Assert.That(device.ReadTrack(0, buffer), Is.EqualTo(replacement.Length)); + Assert.That(buffer.AsSpan(0, replacement.Length).ToArray(), Is.EqualTo(replacement)); + Assert.That(device.ReadTrack(4, buffer), Is.EqualTo(0)); + Assert.That(device.ReadTrack(5, buffer), Is.EqualTo(2)); + } + + var parsed = CbmNibbleReader.Read(image.ToArray(), "image.g64"); + Assert.That(parsed.TrackCount, Is.EqualTo(6)); + Assert.That(parsed.Tracks.Single(t => t.Index == 0).Data, Is.EqualTo(new byte[] { 9, 9, 9, 9, 9 })); + Assert.That(parsed.Tracks.Single(t => t.Index == 4).Data, Is.Empty); + Assert.That(parsed.Tracks.Single(t => t.Index == 5).Data, Is.EqualTo(new byte[] { 0x55, 0xAA })); + } + + [Test, Category("Driver"), Category("EdgeCase")] + public void G64TrackDevice_RejectsWritableVariableSpeedProfile() { + var imageBytes = CbmNibbleWriter.BuildG64FromTracks([ + new CbmNibbleReader.Track(0, new byte[] { 1 }, 3), + ], trackCount: 1); + // one track => speed table begins at 12 + 4 + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(imageBytes.AsSpan(16, 4), 0x100); + using var image = new MemoryStream(imageBytes, writable: true); + + Assert.That(() => CbmNibbleRawTrackDevices.OpenG64(image, writable: true), + Throws.InstanceOf()); + using var readOnly = CbmNibbleRawTrackDevices.OpenG64(image, writable: false); + Assert.That(readOnly.CanWrite, Is.False); + } +} diff --git a/Compression.Tests/D64/D64FilesystemDriverTests.cs b/Compression.Tests/D64/D64FilesystemDriverTests.cs new file mode 100644 index 000000000..d0e2065b6 --- /dev/null +++ b/Compression.Tests/D64/D64FilesystemDriverTests.cs @@ -0,0 +1,94 @@ +using Compression.Registry; +using FileSystem.D64; + +namespace Compression.Tests.D64; + +[TestFixture] +public sealed class D64FilesystemDriverTests { + [Test, Category("Driver")] + public void WritableSession_PreservesNodeIdentityAndPersistsNamespaceAndData() { + var original = Enumerable.Range(0, 700).Select(i => (byte)(i * 17)).ToArray(); + var writer = new D64Writer(); + writer.AddFile("HELLO", original); + using var image = new MemoryStream(); + image.Write(writer.Build("DRIVER", "42")); + image.Position = 0; + + var descriptor = new D64FormatDescriptor(); + var profile = descriptor.ProbeFilesystem(image); + Assert.Multiple(() => { + Assert.That(profile.CanMount, Is.True); + Assert.That(profile.CanMountWritable, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.MutationModel, Is.EqualTo(FilesystemMutationModel.Direct)); + Assert.That(profile.Capabilities.HasFlag(FilesystemDriverCapabilities.RandomAccess), Is.True); + Assert.That(profile.Capabilities.HasFlag(FilesystemDriverCapabilities.Transactions), Is.False); + }); + + FilesystemNodeId createdId; + using (var fs = descriptor.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))) { + var root = fs.RootNodeId; + var helloId = fs.Lookup(root, "hello"); + Assert.That(helloId.HasValue, Is.True); + using (var hello = fs.OpenFile(helloId!.Value, FileAccess.Read)) { + var slice = new byte[97]; + Assert.That(hello.Read(123, slice), Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(original.AsSpan(123, slice.Length).ToArray())); + } + + createdId = fs.CreateFile(root, "newfile"); + using (var handleA = fs.OpenFile(createdId, FileAccess.ReadWrite)) + using (var handleB = fs.OpenFile(createdId, FileAccess.ReadWrite)) { + handleA.Write(0, "0123456789"u8); + var observed = new byte[4]; + Assert.That(handleB.Read(3, observed), Is.EqualTo(4)); + Assert.That(observed, Is.EqualTo("3456"u8.ToArray())); + handleB.Write(4, "ABCD"u8); + handleA.SetLength(12); + handleA.Flush(); + } + + fs.Rename(root, "NEWFILE", root, "RENAMED", replace: false); + Assert.That(fs.Lookup(root, "RENAMED"), Is.EqualTo(createdId), "rename must not change node identity"); + fs.DeleteFile(root, "HELLO"); + fs.Flush(); + } + + image.Position = 0; + using var reopened = descriptor.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + var reopenedRoot = reopened.RootNodeId; + Assert.Multiple(() => { + Assert.That(reopened.Lookup(reopenedRoot, "HELLO"), Is.Null); + Assert.That(reopened.Lookup(reopenedRoot, "RENAMED").HasValue, Is.True); + }); + var renamedId = reopened.Lookup(reopenedRoot, "RENAMED")!.Value; + using var renamed = reopened.OpenFile(renamedId, FileAccess.Read); + var payload = new byte[12]; + Assert.That(renamed.Read(0, payload), Is.EqualTo(payload.Length)); + Assert.That(payload, Is.EqualTo(new byte[] { + (byte)'0', (byte)'1', (byte)'2', (byte)'3', + (byte)'A', (byte)'B', (byte)'C', (byte)'D', + (byte)'8', (byte)'9', 0, 0, + })); + } + + [Test, Category("Driver"), Category("EdgeCase")] + public void Probe_RefusesWritableMountWhenBamOwnershipIsInconsistent() { + var writer = new D64Writer(); + writer.AddFile("HELLO", [1, 2, 3]); + var bytes = writer.Build(); + + // Track 1 sector 0 is allocated to HELLO by the writer. Lie in the BAM and + // mark it free while retaining a matching free-count byte. + const int bamOffset = 17 * 21 * 256; // start of track 18 + bytes[bamOffset + 4]++; + bytes[bamOffset + 5] |= 0x01; + + using var image = new MemoryStream(bytes, writable: true); + var profile = new D64FormatDescriptor().ProbeFilesystem(image); + Assert.Multiple(() => { + Assert.That(profile.CanMount, Is.True); + Assert.That(profile.CanMountWritable, Is.False); + Assert.That(profile.Limitations.Any(x => x.Contains("BAM", StringComparison.OrdinalIgnoreCase)), Is.True); + }); + } +} diff --git a/Compression.Tests/Directory.Build.targets b/Compression.Tests/Directory.Build.targets new file mode 100644 index 000000000..3f3f3082a --- /dev/null +++ b/Compression.Tests/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + + diff --git a/Compression.Tests/Dmg/DmgModifyTests.cs b/Compression.Tests/Dmg/DmgModifyTests.cs new file mode 100644 index 000000000..110c31021 --- /dev/null +++ b/Compression.Tests/Dmg/DmgModifyTests.cs @@ -0,0 +1,71 @@ +using Compression.Registry; +using FileFormat.Dmg; + +namespace Compression.Tests.Dmg; + +[TestFixture] +public sealed class DmgModifyTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Writer_NonSectorAlignedPartition_RoundTripsExactLength() { + var payload = new byte[517]; + new Random(19).NextBytes(payload); + + var writer = new DmgWriter(); + writer.AddPartition("odd.bin", payload); + using var image = new MemoryStream(); + writer.WriteTo(image); + + image.Position = 0; + using var reader = new DmgReader(image); + Assert.That(reader.Entries, Has.Count.EqualTo(1)); + Assert.That(reader.Entries[0].Size, Is.EqualTo(payload.Length)); + Assert.That(reader.Extract(reader.Entries[0]), Is.EqualTo(payload)); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_AddReplaceRemove_MutatesRawUdifProfile() { + var descriptor = new DmgFormatDescriptor(); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var a = Enumerable.Range(0, 517).Select(i => (byte)(i * 11)).ToArray(); + var b = Enumerable.Range(0, 733).Select(i => (byte)(i * 7)).ToArray(); + var c = Enumerable.Range(0, 91).Select(i => (byte)(255 - i)).ToArray(); + var a2 = Enumerable.Range(0, 1025).Select(i => (byte)(i * 3)).ToArray(); + + using var image = new MemoryStream(); + descriptor.Create(image, [ + ArchiveInputInfo.InMemory("A.BIN", a), + ArchiveInputInfo.InMemory("B.BIN", b), + ], new FormatCreateOptions()); + + var modifier = (IArchiveModifiable)descriptor; + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory("C.BIN", c)]); + AssertPayload(image, "A.BIN", a); + AssertPayload(image, "B.BIN", b); + AssertPayload(image, "C.BIN", c); + + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory("A.BIN", a2)]); + AssertPayload(image, "A.BIN", a2); + AssertPayload(image, "B.BIN", b); + AssertPayload(image, "C.BIN", c); + + image.Position = 0; + modifier.Remove(image, ["B.BIN"]); + image.Position = 0; + using var reader = new DmgReader(image); + Assert.That(reader.Entries.Select(e => e.Name), Is.EquivalentTo(new[] { "A.BIN", "C.BIN" })); + Assert.That(reader.Extract(reader.Entries.Single(e => e.Name == "A.BIN")), Is.EqualTo(a2)); + Assert.That(reader.Extract(reader.Entries.Single(e => e.Name == "C.BIN")), Is.EqualTo(c)); + } + + private static void AssertPayload(MemoryStream image, string name, byte[] expected) { + image.Position = 0; + using var reader = new DmgReader(image); + var entry = reader.Entries.Single(e => string.Equals(e.Name, name, StringComparison.OrdinalIgnoreCase)); + Assert.That(entry.Size, Is.EqualTo(expected.Length)); + Assert.That(reader.Extract(entry), Is.EqualTo(expected)); + } +} diff --git a/Compression.Tests/Dtb/DtbInPlaceModifyTests.cs b/Compression.Tests/Dtb/DtbInPlaceModifyTests.cs deleted file mode 100644 index 898550e38..000000000 --- a/Compression.Tests/Dtb/DtbInPlaceModifyTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -using Compression.Registry; -using FileFormat.Dtb; - -namespace Compression.Tests.Dtb; - -/// -/// Locks the honest demotion of FileFormat.Dtb: the descriptor must NOT -/// advertise and the Description -/// must name the specific spec-level reason that blocks in-place mutation. -/// The companion read tests live in . -/// -/// Why DTB stays R-only: the 40-byte FDT header at offset 0 records -/// totalsize, off_dt_struct, off_dt_strings, -/// size_dt_struct, size_dt_strings. Adding/removing any -/// property mutates the struct or strings block, which cascades through every -/// downstream offset and the four header size/offset fields. Mutating those -/// fields in place is a rebuild, not an in-place splice — so promoting to -/// CanModify would mis-advertise the surface. -/// -[TestFixture] -public class DtbInPlaceModifyTests { - - [Test, Category("HappyPath")] - public void Descriptor_DoesNotAdvertiseCanModify() { - var desc = new DtbFormatDescriptor(); - Assert.That(desc.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); - Assert.That(desc, Is.Not.InstanceOf()); - } - - [Test, Category("HappyPath")] - public void Description_NamesTheBlockingFdtHeaderFields() { - var desc = new DtbFormatDescriptor(); - Assert.That(desc.Description, Does.Contain("totalsize")); - Assert.That(desc.Description, Does.Contain("off_dt_strings")); - Assert.That(desc.Description, Does.Contain("off_dt_struct")); - Assert.That(desc.Description, Does.Contain("rebuild")); - } -} diff --git a/Compression.Tests/Dtb/DtbModifyTests.cs b/Compression.Tests/Dtb/DtbModifyTests.cs new file mode 100644 index 000000000..04332a5aa --- /dev/null +++ b/Compression.Tests/Dtb/DtbModifyTests.cs @@ -0,0 +1,62 @@ +using Compression.Registry; +using FileFormat.Dtb; + +namespace Compression.Tests.Dtb; + +[TestFixture] +public sealed class DtbModifyTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void CreateAddReplaceRemove_PreservesHierarchy() { + var descriptor = new DtbFormatDescriptor(); + var compatible = "vendor,board\0"u8.ToArray(); + var reg = new byte[] { 0, 0, 0, 1, 0, 0, 0, 32 }; + var status = "okay\0"u8.ToArray(); + var replacement = "disabled\0"u8.ToArray(); + + using var image = new MemoryStream(); + descriptor.Create(image, [ + ArchiveInputInfo.InMemory("soc/serial@1000/compatible.bin", compatible), + ArchiveInputInfo.InMemory("soc/serial@1000/reg.bin", reg), + ], new FormatCreateOptions()); + + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + AssertProperty(image, "/soc/serial@1000", "compatible", compatible); + AssertProperty(image, "/soc/serial@1000", "reg", reg); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory("soc/serial@1000/status.bin", status)]); + AssertProperty(image, "/soc/serial@1000", "status", status); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory("soc/serial@1000/status.bin", replacement)]); + AssertProperty(image, "/soc/serial@1000", "status", replacement); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Remove(image, ["soc/serial@1000/reg.bin"]); + var parsed = Parse(image); + Assert.That(parsed.Properties.Any(p => p.NodePath == "/soc/serial@1000" && p.Name == "reg"), Is.False); + Assert.That(parsed.Properties.Any(p => p.NodePath == "/soc/serial@1000" && p.Name == "compatible"), Is.True); + } + + [Test, Category("RoundTrip")] + public void RootTextEntry_ListCreateRoundTrip_MapsBackToRootAndRestoresNulTermination() { + var descriptor = new DtbFormatDescriptor(); + using var image = new MemoryStream(); + descriptor.Create(image, + [ArchiveInputInfo.InMemory("_root/compatible.txt", "vendor,board"u8.ToArray())], + new FormatCreateOptions()); + + var parsed = Parse(image); + var property = parsed.Properties.Single(p => p.NodePath == "/" && p.Name == "compatible"); + Assert.That(property.Data, Is.EqualTo("vendor,board\0"u8.ToArray())); + } + + private static DtbReader.Fdt Parse(MemoryStream image) => DtbReader.Read(image.ToArray()); + + private static void AssertProperty(MemoryStream image, string path, string name, byte[] data) { + var property = Parse(image).Properties.Single(p => p.NodePath == path && p.Name == name); + Assert.That(property.Data, Is.EqualTo(data)); + } +} diff --git a/Compression.Tests/Erofs/ErofsRwTests.cs b/Compression.Tests/Erofs/ErofsRwTests.cs new file mode 100644 index 000000000..990ff6b65 --- /dev/null +++ b/Compression.Tests/Erofs/ErofsRwTests.cs @@ -0,0 +1,110 @@ +using Compression.Registry; +using FileSystem.Erofs; + +namespace Compression.Tests.Erofs; + +[TestFixture] +public sealed class ErofsRwTests { + private static readonly byte[] Alpha = "alpha payload"u8.ToArray(); + private static readonly byte[] Beta = Enumerable.Range(0, 9000).Select(i => (byte)(i * 29 + 3)).ToArray(); + + private static MemoryStream CreateImage(string label = "CWB") { + var descriptor = new ErofsFormatDescriptor(); + var stream = new MemoryStream(); + descriptor.Create(stream, [ + ArchiveInputInfo.InMemory("dir/alpha.txt", Alpha), + ArchiveInputInfo.InMemory("beta.bin", Beta), + ], new FormatCreateOptions { + FormatSpecific = new Dictionary { ["VolumeLabel"] = label }, + }); + stream.Position = 0; + return stream; + } + + [Test] + public void SupportedFlatProfile_AdvertisesRwAndMaintenance() { + var descriptor = new ErofsFormatDescriptor(); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + } + + [Test] + public void Add_ReplacesExistingFile_AndPreservesVolumeLabel() { + using var image = CreateImage("SYSTEM"); + var replacement = "replacement alpha"u8.ToArray(); + var descriptor = new ErofsFormatDescriptor(); + + descriptor.Add(image, [ArchiveInputInfo.InMemory("dir/alpha.txt", replacement)]); + + image.Position = 0; + var reader = new ErofsReader(image); + Assert.That(reader.VolumeName, Is.EqualTo("SYSTEM")); + var alpha = reader.Entries.Single(e => e.Path == "dir/alpha.txt"); + Assert.That(reader.ExtractFile(alpha), Is.EqualTo(replacement)); + var beta = reader.Entries.Single(e => e.Path == "beta.bin"); + Assert.That(reader.ExtractFile(beta), Is.EqualTo(Beta)); + } + + [Test] + public void Remove_DropsFile_AndPreservesOtherPayload() { + using var image = CreateImage(); + var descriptor = new ErofsFormatDescriptor(); + + descriptor.Remove(image, ["dir/alpha.txt"]); + + image.Position = 0; + var reader = new ErofsReader(image); + Assert.That(reader.Entries.Any(e => e.Path == "dir/alpha.txt"), Is.False); + var beta = reader.Entries.Single(e => e.Path == "beta.bin"); + Assert.That(reader.ExtractFile(beta), Is.EqualTo(Beta)); + } + + [Test] + public void Purge_LeavesValidEmptyImage() { + using var image = CreateImage("EMPTYME"); + var descriptor = new ErofsFormatDescriptor(); + + ((IArchivePurgeable)descriptor).Purge(image); + + image.Position = 0; + var reader = new ErofsReader(image); + Assert.That(reader.VolumeName, Is.EqualTo("EMPTYME")); + Assert.That(reader.Entries.Where(e => !e.IsDirectory), Is.Empty); + } + + [Test] + public void Defragment_PreservesPayloadAndEmitsProgress() { + using var image = CreateImage(); + var descriptor = new ErofsFormatDescriptor(); + var phases = new List(); + + descriptor.Defragment(image, new DefragOptions { OnProgress = e => phases.Add(e.Phase) }); + + image.Position = 0; + var reader = new ErofsReader(image); + Assert.That(reader.ExtractFile(reader.Entries.Single(e => e.Path == "dir/alpha.txt")), Is.EqualTo(Alpha)); + Assert.That(reader.ExtractFile(reader.Entries.Single(e => e.Path == "beta.bin")), Is.EqualTo(Beta)); + Assert.That(phases, Does.Contain("scanning").Or.Contain("complete")); + } + + [Test] + public void Wipe_IsConservativeWithoutAllocatorProof() { + using var image = CreateImage(); + var before = image.ToArray(); + var descriptor = new ErofsFormatDescriptor(); + + var wiped = descriptor.WipeUnusedSpace(image); + + Assert.That(wiped, Is.Zero); + Assert.That(image.ToArray(), Is.EqualTo(before)); + var map = descriptor.EnumerateExtents(image).ToArray(); + Assert.That(map, Is.Not.Empty); + Assert.That(map.Any(e => e.Kind == DefragBlockKind.Free), Is.False, + "Unproven EROFS bytes must be reserved, never inferred free."); + } +} diff --git a/Compression.Tests/Ewf/EwfPurgeTests.cs b/Compression.Tests/Ewf/EwfPurgeTests.cs new file mode 100644 index 000000000..3c6f6a147 --- /dev/null +++ b/Compression.Tests/Ewf/EwfPurgeTests.cs @@ -0,0 +1,26 @@ +using Compression.Registry; +using FileFormat.Ewf; + +namespace Compression.Tests.Ewf; + +[TestFixture] +public sealed class EwfPurgeTests { + [Test] + public void Purge_LeavesValidEmptyImageAndGeneratedDiagnostics() { + var media = new byte[EwfWriter.ChunkSize * 2]; + for (var i = 0; i < media.Length; ++i) media[i] = (byte)(i * 31 + 7); + using var image = new MemoryStream(); + image.Write(new EwfWriter { CompressChunks = true }.Build(media)); + image.Position = 0; + + var descriptor = new EwfFormatDescriptor(); + ((IArchivePurgeable)descriptor).Purge(image); + + var parsed = EwfReader.Read(image.ToArray()); + Assert.That(EwfReader.ExtractMedia(parsed), Is.Empty); + var names = descriptor.List(image, null).Select(e => e.Name).ToArray(); + Assert.That(names, Does.Contain("media.raw")); + Assert.That(names, Does.Contain("metadata.ini")); + Assert.That(names.Any(n => n.StartsWith("section_", StringComparison.Ordinal)), Is.True); + } +} diff --git a/Compression.Tests/Ewf/EwfTests.cs b/Compression.Tests/Ewf/EwfTests.cs index 4ecf09409..73d43df64 100644 --- a/Compression.Tests/Ewf/EwfTests.cs +++ b/Compression.Tests/Ewf/EwfTests.cs @@ -1,88 +1,80 @@ using System.Buffers.Binary; using System.Text; +using Compression.Registry; using FileFormat.Ewf; +using FileFormat.Zlib; namespace Compression.Tests.Ewf; [TestFixture] public class EwfTests { - // Build a minimal EWF segment: 13-byte header + one "header" section - // carrying a fabricated acquisition header + a terminal "done" section. private static byte[] BuildEwf(byte[] headerPayload, bool logical = false) { const int headerSize = EwfReader.FileHeaderSize; const int descSize = EwfReader.SectionDescriptorSize; var headerSectionSize = descSize + headerPayload.Length; var doneSectionOffset = headerSize + headerSectionSize; - var total = doneSectionOffset + descSize; // "done" payload empty - + var total = doneSectionOffset + descSize; var buf = new byte[total]; - // File header. (logical ? EwfReader.LvfSignature : EwfReader.EvfSignature).CopyTo(buf, 0); - buf[8] = 0x01; // fields_start - BinaryPrimitives.WriteUInt16LittleEndian(buf.AsSpan(9), 1); // segment 1 - BinaryPrimitives.WriteUInt16LittleEndian(buf.AsSpan(11), 0x0000); // fields_end - - // Section 1: "header" — 16-byte type + 8-byte next + 8-byte size + pad + CRC. - var sec1Offset = headerSize; - WriteSectionType(buf, sec1Offset, "header"); - BinaryPrimitives.WriteUInt64LittleEndian(buf.AsSpan(sec1Offset + 16), (ulong)doneSectionOffset); - BinaryPrimitives.WriteUInt64LittleEndian(buf.AsSpan(sec1Offset + 24), (ulong)headerSectionSize); - BinaryPrimitives.WriteUInt32LittleEndian(buf.AsSpan(sec1Offset + 72), 0xCAFEBABE); // checksum - headerPayload.CopyTo(buf.AsSpan(sec1Offset + descSize)); - - // Section 2: "done" — terminal; next_offset points to itself per convention. + buf[8] = 0x01; + BinaryPrimitives.WriteUInt16LittleEndian(buf.AsSpan(9), 1); + BinaryPrimitives.WriteUInt16LittleEndian(buf.AsSpan(11), 0); + + WriteSectionType(buf, headerSize, "header"); + BinaryPrimitives.WriteUInt64LittleEndian(buf.AsSpan(headerSize + 16), (ulong)doneSectionOffset); + BinaryPrimitives.WriteUInt64LittleEndian(buf.AsSpan(headerSize + 24), (ulong)headerSectionSize); + BinaryPrimitives.WriteUInt32LittleEndian(buf.AsSpan(headerSize + 72), 0xCAFEBABE); + headerPayload.CopyTo(buf.AsSpan(headerSize + descSize)); + WriteSectionType(buf, doneSectionOffset, "done"); BinaryPrimitives.WriteUInt64LittleEndian(buf.AsSpan(doneSectionOffset + 16), (ulong)doneSectionOffset); BinaryPrimitives.WriteUInt64LittleEndian(buf.AsSpan(doneSectionOffset + 24), descSize); BinaryPrimitives.WriteUInt32LittleEndian(buf.AsSpan(doneSectionOffset + 72), 0xDEADBEEF); - return buf; } private static void WriteSectionType(byte[] buf, int offset, string type) { var ascii = Encoding.ASCII.GetBytes(type); Buffer.BlockCopy(ascii, 0, buf, offset, Math.Min(ascii.Length, 16)); - // Remaining bytes in the 16-byte type field stay zero by default. } private static byte[] SampleHeader() { - // Fake "header" text block in the shape libewf emits: version line, key row, - // value row, tab-separated. var text = "1\r\n" + - "c\tn\te\tnotes\tav\tov\tm\tu\tp\tr\r\n" + - "CASE123\tSMITH\tExaminer\tNotes\t20060101\t20060102\tMD5\tUnknown\tp\tr\r\n"; - return Encoding.UTF8.GetBytes(text); + "a\tc\tn\te\tt\tav\tov\tm\tu\tp\tr\r\n" + + "Description\tCASE123\tEVIDENCE7\tExaminer\tNotes\t20060101\t20060102\tMD5\tUnknown\tp\tr\r\n"; + return ZlibStream.Compress(Encoding.UTF8.GetBytes(text)); + } + + private static byte[] Media(int length = 3 * EwfWriter.ChunkSize) { + var data = new byte[length]; + for (var i = 0; i < data.Length; ++i) + data[i] = (byte)((i * 17 + i / 101) & 0xFF); + return data; } [Test, Category("HappyPath")] public void Read_ParsesEvfSignatureAndSections() { var data = BuildEwf(SampleHeader()); var img = EwfReader.Read(data); - Assert.That(img.IsLogical, Is.False); Assert.That(img.SegmentNumber, Is.EqualTo((ushort)1)); Assert.That(img.Sections, Has.Count.EqualTo(2)); Assert.That(img.Sections[0].Type, Is.EqualTo("header")); Assert.That(img.Sections[1].Type, Is.EqualTo("done")); - Assert.That(img.Sections[0].Payload.Length, Is.EqualTo(SampleHeader().Length)); } [Test, Category("HappyPath")] public void Read_RecognisesLvfSignature() { - var data = BuildEwf(SampleHeader(), logical: true); - var img = EwfReader.Read(data); + var img = EwfReader.Read(BuildEwf(SampleHeader(), logical: true)); Assert.That(img.IsLogical, Is.True); } [Test, Category("HappyPath")] public void Descriptor_List_EmitsMetadataAndSectionEntries() { - var data = BuildEwf(SampleHeader()); - using var ms = new MemoryStream(data); - var entries = new EwfFormatDescriptor().List(ms, null); - var names = entries.Select(e => e.Name).ToList(); - + using var ms = new MemoryStream(BuildEwf(SampleHeader())); + var names = new EwfFormatDescriptor().List(ms, null).Select(e => e.Name).ToList(); Assert.That(names, Does.Contain("metadata.ini")); Assert.That(names.Any(n => n.StartsWith("section_00_header", StringComparison.Ordinal)), Is.True); Assert.That(names.Any(n => n.StartsWith("section_01_done", StringComparison.Ordinal)), Is.True); @@ -90,15 +82,12 @@ public void Descriptor_List_EmitsMetadataAndSectionEntries() { [Test, Category("HappyPath"), Category("RoundTrip")] public void Descriptor_Extract_ExtractsMetadataIniWithAcquisitionBlock() { - var data = BuildEwf(SampleHeader()); var tmp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); Directory.CreateDirectory(tmp); try { - using var ms = new MemoryStream(data); + using var ms = new MemoryStream(BuildEwf(SampleHeader())); new EwfFormatDescriptor().Extract(ms, tmp, null, null); - var meta = Path.Combine(tmp, "metadata.ini"); - Assert.That(File.Exists(meta), Is.True); - var text = File.ReadAllText(meta); + var text = File.ReadAllText(Path.Combine(tmp, "metadata.ini")); Assert.That(text, Does.Contain("[ewf]")); Assert.That(text, Does.Contain("section_count = 2")); Assert.That(text, Does.Contain("[acquisition]")); @@ -108,6 +97,90 @@ public void Descriptor_Extract_ExtractsMetadataIniWithAcquisitionBlock() { } } + [TestCase(false), TestCase(true)] + [Category("HappyPath"), Category("RoundTrip")] + public void Writer_Reader_ReconstructsLogicalMedia(bool compress) { + var media = Media(); + var encoded = new EwfWriter { CompressChunks = compress }.Build(media); + var decoded = EwfReader.ExtractMedia(EwfReader.Read(encoded)); + Assert.That(decoded.AsSpan(0, media.Length).ToArray(), Is.EqualTo(media)); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_List_ExposesMutableMediaRaw() { + var media = Media(); + using var image = new MemoryStream(new EwfWriter().Build(media), writable: true); + var descriptor = new EwfFormatDescriptor(); + var mediaEntry = descriptor.List(image, null).Single(e => e.Name == "media.raw"); + Assert.That(mediaEntry.OriginalSize, Is.EqualTo(media.Length)); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_Add_ReplacesMediaRaw() { + var original = Media(EwfWriter.ChunkSize); + var replacement = Media(EwfWriter.ChunkSize * 2); + replacement.AsSpan().Reverse(); + using var image = new MemoryStream(); + image.Write(new EwfWriter { CompressChunks = true }.Build(original)); + image.Position = 0; + + var descriptor = new EwfFormatDescriptor(); + descriptor.Add(image, [ArchiveInputInfo.InMemory("media.raw", replacement)]); + image.Position = 0; + var decoded = EwfReader.ExtractMedia(EwfReader.Read(image.ToArray())); + Assert.That(decoded.AsSpan(0, replacement.Length).ToArray(), Is.EqualTo(replacement)); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_RemoveMedia_LeavesValidEmptyEvf() { + using var image = new MemoryStream(); + image.Write(new EwfWriter().Build(Media(EwfWriter.ChunkSize))); + image.Position = 0; + var descriptor = new EwfFormatDescriptor(); + descriptor.Remove(image, ["media.raw"]); + var parsed = EwfReader.Read(image.ToArray()); + Assert.That(EwfReader.ExtractMedia(parsed), Is.Empty); + Assert.That(parsed.Sections.Any(s => s.Type == "done"), Is.True); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_Defragment_PreservesMediaAndReportsLayout() { + var media = Media(); + using var image = new MemoryStream(); + image.Write(new EwfWriter { CompressChunks = true }.Build(media)); + image.Position = 0; + var phases = new List(); + var descriptor = new EwfFormatDescriptor(); + descriptor.Defragment(image, new DefragOptions { + OnProgress = e => phases.Add(e.Phase), + }); + var decoded = EwfReader.ExtractMedia(EwfReader.Read(image.ToArray())); + Assert.That(decoded.AsSpan(0, media.Length).ToArray(), Is.EqualTo(media)); + Assert.That(phases, Does.Contain("scanning")); + Assert.That(phases, Does.Contain("writing")); + Assert.That(phases, Does.Contain("committing")); + Assert.That(phases, Does.Contain("complete")); + Assert.That(descriptor.EnumerateLayout(image).Any(), Is.True); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_Shrink_ChoosesSmallerValidRepresentation() { + var media = new byte[EwfWriter.ChunkSize * 4]; + Array.Fill(media, (byte)0x41); + var original = new EwfWriter { CompressChunks = false }.Build(media); + using var input = new MemoryStream(original); + using var output = new MemoryStream(); + new EwfFormatDescriptor().Shrink(input, output); + Assert.That(output.Length, Is.LessThan(original.Length)); + var decoded = EwfReader.ExtractMedia(EwfReader.Read(output.ToArray())); + Assert.That(decoded.AsSpan(0, media.Length).ToArray(), Is.EqualTo(media)); + } + [Test, Category("EdgeCase")] public void Read_BadSignature_Throws() { var data = new byte[64]; @@ -117,7 +190,7 @@ public void Read_BadSignature_Throws() { [Test, Category("EdgeCase")] public void Read_TruncatedHeader_Throws() { - var data = new byte[8]; // signature only, no fields_start/segment/fields_end + var data = new byte[8]; EwfReader.EvfSignature.CopyTo(data, 0); Assert.That(() => EwfReader.Read(data), Throws.InstanceOf()); } @@ -125,7 +198,7 @@ public void Read_TruncatedHeader_Throws() { [Test, Category("EdgeCase")] public void Read_BadFieldsStart_Throws() { var data = BuildEwf(SampleHeader()); - data[8] = 0x02; // corrupt fields_start (must be 0x01) + data[8] = 0x02; Assert.That(() => EwfReader.Read(data), Throws.InstanceOf()); } } diff --git a/Compression.Tests/FilesystemDrivers/ApfsFilesystemDriverTests.cs b/Compression.Tests/FilesystemDrivers/ApfsFilesystemDriverTests.cs new file mode 100644 index 000000000..41f08dfc2 --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/ApfsFilesystemDriverTests.cs @@ -0,0 +1,63 @@ +using Compression.Lib; +using Compression.Registry; +using FileSystem.Apfs; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class ApfsFilesystemDriverTests { + [OneTimeSetUp] + public void Init() => FormatRegistration.EnsureInitialized(); + + [Test] + public void RegistryUsesNativeApfsSidecar() { + var coverage = FormatRegistry.GetFilesystemDriverCoverage("Apfs"); + Assert.That(coverage.Binding, Is.EqualTo(FilesystemDriverBindingKind.SidecarNative)); + Assert.That(coverage.HasExtentMap, Is.True); + Assert.That(coverage.HasBlockMover, Is.True); + } + + [Test] + public void NativeWriterProfileUsesObjectIdentityAndDirectPositionalReads() { + var payload = Enumerable.Range(0, 20_000).Select(i => (byte)(i * 13 + 5)).ToArray(); + var writer = new ApfsWriter(); + writer.SetMinImageSize(4 * 1024 * 1024); + writer.AddFile("dir/data.bin", payload); + var image = writer.Build(); + + using var stream = new MemoryStream(image, writable: false); + var profile = FormatRegistry.ProbeFilesystem("Apfs", stream); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.False); + + stream.Position = 0; + using var session = FormatRegistry.OpenFilesystem( + "Apfs", stream, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + var dir = session.Lookup(session.RootNodeId, "dir"); + Assert.That(dir, Is.Not.Null); + var file = session.Lookup(dir!.Value, "data.bin"); + Assert.That(file, Is.Not.Null); + + var stat = session.Stat(file!.Value); + Assert.That(stat.NodeId.Value, Is.GreaterThanOrEqualTo((ulong)ApfsConstants.APFS_MIN_USER_INO_NUM)); + Assert.That(stat.Size, Is.EqualTo(payload.Length)); + + using var handle = session.OpenFile(file.Value, FileAccess.Read); + var slice = new byte[1025]; + var read = handle.Read(12_345, slice); + Assert.That(read, Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(payload.AsSpan(12_345, slice.Length).ToArray())); + } + + [Test] + public void WritableMountStaysFailClosed() { + var writer = new ApfsWriter(); + writer.SetMinImageSize(4 * 1024 * 1024); + writer.AddFile("x", "x"u8.ToArray()); + using var stream = new MemoryStream(writer.Build(), writable: true); + + Assert.Throws(() => + FormatRegistry.OpenFilesystem( + "Apfs", stream, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))); + } +} diff --git a/Compression.Tests/FilesystemDrivers/BlockDeviceAdapterTests.cs b/Compression.Tests/FilesystemDrivers/BlockDeviceAdapterTests.cs new file mode 100644 index 000000000..3a867fd4b --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/BlockDeviceAdapterTests.cs @@ -0,0 +1,55 @@ +using Compression.Registry; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class BlockDeviceAdapterTests { + [Test, Category("Driver")] + public void StreamBlockDevice_ProvidesAlignedPositionalBlocks() { + var bytes = Enumerable.Range(0, 2048).Select(i => (byte)i).ToArray(); + using var image = new MemoryStream(bytes.ToArray(), writable: true); + using var device = new StreamBlockDevice(image, 512, writable: true); + + Span block = stackalloc byte[512]; + Assert.That(device.ReadBlocks(2, block), Is.EqualTo(1)); + Assert.That(block[0], Is.EqualTo(bytes[1024])); + Assert.That(block[511], Is.EqualTo(bytes[1535])); + + block.Fill(0xA5); + device.WriteBlocks(1, block); + Assert.That(image.ToArray().AsSpan(512, 512).ToArray(), Is.All.EqualTo((byte)0xA5)); + Assert.That(image.ToArray().AsSpan(0, 512).ToArray(), Is.EqualTo(bytes.AsSpan(0, 512).ToArray())); + } + + [Test, Category("Driver")] + public void BlockDeviceStream_UnalignedWriteTouchesOnlyNecessaryBlocks() { + var original = Enumerable.Range(0, 2048).Select(i => (byte)(i * 17)).ToArray(); + using var image = new MemoryStream(original.ToArray(), writable: true); + using var device = new StreamBlockDevice(image, 512, writable: true); + using var stream = new BlockDeviceStream(device); + + stream.Position = 510; + stream.Write(new byte[] { 0x11, 0x22, 0x33, 0x44, 0x55 }); + stream.Flush(); + + var actual = image.ToArray(); + Assert.That(actual.AsSpan(0, 510).ToArray(), Is.EqualTo(original.AsSpan(0, 510).ToArray())); + Assert.That(actual.AsSpan(510, 5).ToArray(), Is.EqualTo(new byte[] { 0x11, 0x22, 0x33, 0x44, 0x55 })); + Assert.That(actual.AsSpan(515).ToArray(), Is.EqualTo(original.AsSpan(515).ToArray())); + } + + [Test, Category("Driver")] + public void BlockDeviceStream_ProvidesSeekableByteReadsAcrossBlockBoundaries() { + var bytes = Enumerable.Range(0, 1536).Select(i => (byte)(i ^ 0x5A)).ToArray(); + using var image = new MemoryStream(bytes, writable: false); + using var device = new StreamBlockDevice(image, 512, writable: false); + using var stream = new BlockDeviceStream(device); + + stream.Position = 509; + var read = new byte[11]; + Assert.That(stream.Read(read), Is.EqualTo(read.Length)); + Assert.That(read, Is.EqualTo(bytes.AsSpan(509, 11).ToArray())); + Assert.That(stream.CanWrite, Is.False); + Assert.That(() => stream.WriteByte(1), Throws.InstanceOf()); + } +} diff --git a/Compression.Tests/FilesystemDrivers/ExtFilesystemDriverTests.cs b/Compression.Tests/FilesystemDrivers/ExtFilesystemDriverTests.cs new file mode 100644 index 000000000..bc11f403e --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/ExtFilesystemDriverTests.cs @@ -0,0 +1,63 @@ +using Compression.Lib; +using Compression.Registry; +using FileSystem.Ext; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class ExtFilesystemDriverTests { + [OneTimeSetUp] + public void Init() => FormatRegistration.EnsureInitialized(); + + [Test] + public void RegistryUsesNativeExtSidecar() { + var coverage = FormatRegistry.GetFilesystemDriverCoverage("Ext"); + Assert.That(coverage.Binding, Is.EqualTo(FilesystemDriverBindingKind.SidecarNative)); + Assert.That(coverage.HasNativeReadinessProvider, Is.True); + } + + [Test] + public void NativeSessionProvidesStableInodeIdentityAndPositionalReads() { + var payload = Enumerable.Range(0, 20_000).Select(i => (byte)(i * 31)).ToArray(); + var writer = new ExtWriter(); + writer.AddFile("dir/file.bin", payload); + var image = writer.Build(); + + using var stream = new MemoryStream(image, writable: false); + var profile = FormatRegistry.ProbeFilesystem("Ext", stream); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.False); + + stream.Position = 0; + using var session = FormatRegistry.OpenFilesystem( + "Ext", stream, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + + var dir = session.Lookup(session.RootNodeId, "dir"); + Assert.That(dir, Is.Not.Null); + var file = session.Lookup(dir!.Value, "file.bin"); + Assert.That(file, Is.Not.Null); + + var firstStat = session.Stat(file!.Value); + var secondStat = session.Stat(file.Value); + Assert.That(secondStat.NodeId, Is.EqualTo(firstStat.NodeId)); + Assert.That(firstStat.NodeId.Value, Is.GreaterThan(0)); + Assert.That(firstStat.Size, Is.EqualTo(payload.Length)); + + using var handle = session.OpenFile(file.Value, FileAccess.Read); + var slice = new byte[777]; + var read = handle.Read(12_345, slice); + Assert.That(read, Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(payload.AsSpan(12_345, slice.Length).ToArray())); + } + + [Test] + public void WritableMountStaysFailClosed() { + var writer = new ExtWriter(); + writer.AddFile("x", "data"u8.ToArray()); + using var stream = new MemoryStream(writer.Build(), writable: true); + + Assert.Throws(() => + FormatRegistry.OpenFilesystem( + "Ext", stream, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))); + } +} diff --git a/Compression.Tests/FilesystemDrivers/FatFilesystemDriverTests.cs b/Compression.Tests/FilesystemDrivers/FatFilesystemDriverTests.cs new file mode 100644 index 000000000..f9cc789d1 --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/FatFilesystemDriverTests.cs @@ -0,0 +1,99 @@ +using Compression.Lib; +using Compression.Registry; +using FileSystem.Fat; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class FatFilesystemDriverTests { + [OneTimeSetUp] + public void InitializeRegistry() => FormatRegistration.EnsureInitialized(); + + [Test, Category("Driver")] + public void GeneratedSidecar_IsRegisteredForFat() { + var driver = FormatRegistry.GetFilesystemDriver("Fat"); + Assert.That(driver, Is.TypeOf()); + } + + [Test, Category("Driver")] + public void NativeFatSession_ReadsArbitraryOffsetsWithoutExtractingWholeFile() { + var payload = Enumerable.Range(0, 1700).Select(i => (byte)(i * 29 + 7)).ToArray(); + var writer = new FatWriter(); + writer.SetVolumeSerial(0x11223344); + writer.AddFile("HELLO.BIN", payload); + using var image = new MemoryStream(writer.Build(), writable: false); + + var profile = FormatRegistry.ProbeFilesystem("Fat", image); + Assert.That(profile.CanMount, Is.True); + Assert.That(profile.CanMountWritable, Is.False); + Assert.That(profile.ProfileName, Does.StartWith("FAT")); + + using var session = FormatRegistry.OpenFilesystem( + "Fat", image, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + var node = session.Lookup(session.RootNodeId, "hello.bin"); + Assert.That(node, Is.Not.Null, "FAT lookup should be case-insensitive when unambiguous."); + + using var handle = session.OpenFile(node!.Value, FileAccess.Read); + var buffer = new byte[777]; + Assert.That(handle.Read(503, buffer), Is.EqualTo(buffer.Length)); + Assert.That(buffer, Is.EqualTo(payload.AsSpan(503, buffer.Length).ToArray())); + + var tail = new byte[64]; + Assert.That(handle.Read(payload.Length - 17, tail), Is.EqualTo(17)); + Assert.That(tail.AsSpan(0, 17).ToArray(), Is.EqualTo(payload[^17..])); + Assert.That(handle.Read(payload.Length, tail), Is.Zero); + } + + [Test, Category("Driver")] + public void FatDriver_ComposesWithRandomAccessBlockDevice() { + var payload = Enumerable.Range(0, 900).Select(i => (byte)(255 - i)).ToArray(); + var writer = new FatWriter(); + writer.SetVolumeSerial(0x55667788); + writer.AddFile("BLOCK.DAT", payload); + using var image = new MemoryStream(writer.Build(), writable: false); + using var device = new StreamBlockDevice(image, 512, writable: false, leaveOpen: true); + var adapter = new FatFilesystemDriverAdapter(); + + using var session = adapter.OpenFilesystem( + device, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + var node = session.Lookup(session.RootNodeId, "BLOCK.DAT"); + Assert.That(node, Is.Not.Null); + using var handle = session.OpenFile(node!.Value, FileAccess.Read); + var bytes = new byte[300]; + Assert.That(handle.Read(510, bytes), Is.EqualTo(bytes.Length)); + Assert.That(bytes, Is.EqualTo(payload.AsSpan(510, bytes.Length).ToArray())); + } + + [Test, Category("Driver"), Category("Corruption")] + public void WritableReadiness_FailsClosedWhenFatCopiesDisagree() { + var writer = new FatWriter(); + writer.SetVolumeSerial(0x99AABBCC); + writer.AddFile("A.BIN", Enumerable.Repeat((byte)0xA5, 700).ToArray()); + var bytes = writer.Build(); + + // Default 1.44 MB FAT12 geometry: reserved sector 1, each FAT is 9 sectors. + // Cluster 2's packed FAT12 entry starts at byte offset cluster + cluster/2. + var secondFatStart = (1 + 9) * 512; + bytes[secondFatStart + 3] ^= 0x01; + using var image = new MemoryStream(bytes, writable: false); + + var profile = new FatFilesystemDriverAdapter().ProbeFilesystem(image); + Assert.That(profile.CanMount, Is.False); + Assert.That(profile.Limitations.Any(text => text.Contains("copies disagree", StringComparison.OrdinalIgnoreCase)), Is.True); + } + + [Test, Category("Driver"), Category("Contract")] + public void FatReadiness_SeparatesExistingOfflineMutationFromMountedWriteCompleteness() { + var writer = new FatWriter(); + writer.SetVolumeSerial(0xCAFEBABE); + writer.AddFile("A.TXT", "abc"u8.ToArray()); + using var image = new MemoryStream(writer.Build(), writable: true); + + var report = FormatRegistry.AssessFilesystemDriver("Fat", image, FilesystemDriverTarget.ReadWrite); + Assert.That(report.UsesNativeProvider, Is.True); + Assert.That(report.Derivable, Is.False); + Assert.That(report.AvailableLayers.HasFlag(FilesystemDriverReadinessLayer.AllocationMap), Is.True); + Assert.That(report.AvailableLayers.HasFlag(FilesystemDriverReadinessLayer.WriteData), Is.False); + Assert.That(report.Blockers.Any(text => text.Contains("FatModifier", StringComparison.Ordinal)), Is.True); + } +} diff --git a/Compression.Tests/FilesystemDrivers/FilesystemDriverDerivationTests.cs b/Compression.Tests/FilesystemDrivers/FilesystemDriverDerivationTests.cs new file mode 100644 index 000000000..57aca98f8 --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/FilesystemDriverDerivationTests.cs @@ -0,0 +1,118 @@ +using Compression.Lib; +using Compression.Registry; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class FilesystemDriverDerivationTests { + [OneTimeSetUp] + public void InitializeRegistry() => FormatRegistration.EnsureInitialized(); + + [Test, Category("Driver")] + public void DerivedReadOnlySession_ReconstructsHierarchyAndProvidesPositionalHandles() { + var descriptor = new ProjectionDescriptor(); + using var image = new MemoryStream(new byte[] { 0x42 }, writable: false); + using var session = FilesystemDriverDerivation.Open( + descriptor, image, new FilesystemOpenOptions(ReadOnly: true)); + + Assert.That(session.Profile.CanMount, Is.True); + Assert.That(session.Profile.CanMountWritable, Is.False); + var root = session.RootNodeId; + var rootEntries = session.Enumerate(root); + Assert.That(rootEntries.Select(e => e.Name), Is.EqualTo(new[] { "folder", "link" })); + + var folder = session.Lookup(root, "folder"); + Assert.That(folder, Is.Not.Null); + Assert.That(session.Lookup(root, "FOLDER"), Is.EqualTo(folder), + "fallback lookup may case-fold only when the result is unambiguous"); + var dataNode = session.Lookup(folder!.Value, "data.bin"); + Assert.That(dataNode, Is.Not.Null); + Assert.That(session.Lookup(folder.Value, "data.bin"), Is.EqualTo(dataNode), + "node ids must remain stable for the session lifetime"); + + using var handle = session.OpenFile(dataNode!.Value, FileAccess.Read); + Span slice = stackalloc byte[3]; + Assert.That(handle.Read(2, slice), Is.EqualTo(3)); + Assert.That(slice.ToArray(), Is.EqualTo(new byte[] { 3, 4, 5 })); + Assert.That(handle.Read(99, slice), Is.Zero); + + var link = session.Lookup(root, "link"); + Assert.That(link, Is.Not.Null); + Assert.That(session.Stat(link!.Value).Kind, Is.EqualTo(FilesystemNodeKind.SymbolicLink)); + Assert.That(session.ReadSymbolicLink(link.Value), Is.EqualTo("folder/data.bin")); + + Assert.That(() => session.CreateFile(root, "new.bin"), Throws.InstanceOf()); + Assert.That(() => session.BeginTransaction(), Throws.InstanceOf()); + } + + [Test, Category("Driver"), Category("Contract")] + public void EveryRegisteredFilesystemDescriptor_HasAtLeastReadOnlyDriverDerivationSurface() { + var filesystemDescriptors = FormatRegistry.All + .Where(descriptor => descriptor.GetType().Assembly.GetName().Name + ?.StartsWith("FileSystem.", StringComparison.Ordinal) == true) + .ToArray(); + + Assert.That(filesystemDescriptors, Is.Not.Empty, "No FileSystem.* descriptors were registered."); + var missing = filesystemDescriptors + .Where(descriptor => descriptor is not IFilesystemDriverProvider && descriptor is not IArchiveFormatOperations) + .Select(descriptor => $"{descriptor.Id} ({descriptor.GetType().FullName})") + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + + Assert.That(missing, Is.Empty, + "Every FileSystem.* descriptor must expose either a native filesystem provider or List/OpenEntry so the common read-only driver can be derived. Missing: " + + string.Join(", ", missing)); + } + + [Test, Category("Driver"), Category("Contract")] + public void ArchiveModifyCapability_IsNeverPromotedToMountedWriteSupport() { + var descriptor = new ProjectionDescriptor(); + using var image = new MemoryStream(new byte[] { 0x42 }, writable: true); + + var profile = FilesystemDriverDerivation.Probe(descriptor, image); + Assert.That(profile.CanMount, Is.True); + Assert.That(profile.CanMountWritable, Is.False); + Assert.That(() => FilesystemDriverDerivation.Open( + descriptor, image, new FilesystemOpenOptions(ReadOnly: false)), + Throws.InstanceOf()); + } + + private sealed class ProjectionDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveModifiable { + private static readonly byte[] Data = [1, 2, 3, 4, 5, 6]; + + public string Id => "DriverProjectionTest"; + public string DisplayName => "driver projection test"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanModify | + FormatCapabilities.SupportsMultipleEntries; + public string DefaultExtension => ".drvtest"; + public IReadOnlyList Extensions => [".drvtest"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => []; + public IReadOnlyList Methods => [new("stored", "Stored")]; + public string? TarCompressionFormatId => null; + + public List List(Stream stream, string? password) => [ + new(0, "folder", 0, 0, "directory", true, false, null), + new(1, "folder/data.bin", Data.Length, Data.Length, "stored", false, false, null), + new(2, "link", "folder/data.bin".Length, "folder/data.bin".Length, + "symlink", false, false, null, IsSymlink: true, LinkTarget: "folder/data.bin"), + ]; + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) + => throw new NotSupportedException("The derivation test uses OpenEntry directly."); + + public Stream OpenEntry(Stream archive, string entryName, string? password) { + var bytes = string.Equals(entryName, "folder/data.bin", StringComparison.Ordinal) + ? Data + : []; + return new MemoryStream(bytes, writable: false); + } + + public void Add(Stream archive, IReadOnlyList inputs) + => throw new NotSupportedException(); + public void Remove(Stream archive, string[] entryNames) + => throw new NotSupportedException(); + } +} diff --git a/Compression.Tests/FilesystemDrivers/NtfsFilesystemDriverTests.cs b/Compression.Tests/FilesystemDrivers/NtfsFilesystemDriverTests.cs new file mode 100644 index 000000000..03cce30dd --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/NtfsFilesystemDriverTests.cs @@ -0,0 +1,59 @@ +using Compression.Lib; +using Compression.Registry; +using FileSystem.Ntfs; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class NtfsFilesystemDriverTests { + [OneTimeSetUp] + public void Init() => FormatRegistration.EnsureInitialized(); + + [Test] + public void RegistryUsesNativeNtfsSidecar() { + var coverage = FormatRegistry.GetFilesystemDriverCoverage("Ntfs"); + Assert.That(coverage.Binding, Is.EqualTo(FilesystemDriverBindingKind.SidecarNative)); + Assert.That(coverage.HasExtentMap, Is.True); + Assert.That(coverage.HasBlockMover, Is.True); + } + + [Test] + public void NativeSessionUsesMftIdentityAndSupportsPositionalReads() { + var payload = Enumerable.Range(0, 200 * 1024).Select(i => (byte)(i * 29 + 7)).ToArray(); + var writer = new NtfsWriter(); + writer.AddFile("dir/data.bin", payload); + var image = writer.Build(8 * 1024 * 1024); + + using var stream = new MemoryStream(image, writable: false); + var profile = FormatRegistry.ProbeFilesystem("Ntfs", stream); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.False); + + stream.Position = 0; + using var session = FormatRegistry.OpenFilesystem( + "Ntfs", stream, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + + var dir = session.Lookup(session.RootNodeId, "dir"); + Assert.That(dir, Is.Not.Null); + var file = session.Lookup(dir!.Value, "data.bin"); + Assert.That(file, Is.Not.Null); + Assert.That(file!.Value.Value, Is.GreaterThan(15), "user object id should be its non-reserved MFT record"); + + using var handle = session.OpenFile(file.Value, FileAccess.Read); + var slice = new byte[2049]; + var read = handle.Read(73_333, slice); + Assert.That(read, Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(payload.AsSpan(73_333, slice.Length).ToArray())); + } + + [Test] + public void WritableMountStaysFailClosed() { + var writer = new NtfsWriter(); + writer.AddFile("x.txt", "x"u8.ToArray()); + using var stream = new MemoryStream(writer.Build(), writable: true); + + Assert.Throws(() => + FormatRegistry.OpenFilesystem( + "Ntfs", stream, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))); + } +} diff --git a/Compression.Tests/FilesystemDrivers/XfsFilesystemDriverTests.cs b/Compression.Tests/FilesystemDrivers/XfsFilesystemDriverTests.cs new file mode 100644 index 000000000..d72df34ff --- /dev/null +++ b/Compression.Tests/FilesystemDrivers/XfsFilesystemDriverTests.cs @@ -0,0 +1,61 @@ +using Compression.Lib; +using Compression.Registry; +using FileSystem.Xfs; + +namespace Compression.Tests.FilesystemDrivers; + +[TestFixture] +public sealed class XfsFilesystemDriverTests { + [OneTimeSetUp] + public void Init() => FormatRegistration.EnsureInitialized(); + + [Test] + public void RegistryUsesNativeXfsSidecar() { + var coverage = FormatRegistry.GetFilesystemDriverCoverage("Xfs"); + Assert.That(coverage.Binding, Is.EqualTo(FilesystemDriverBindingKind.SidecarNative)); + Assert.That(coverage.HasExtentMap, Is.True); + Assert.That(coverage.HasBlockMover, Is.True); + } + + [Test] + public void NativeSessionUsesInodeIdentityAndStreamsPositionalReads() { + var payload = Enumerable.Range(0, 80_000).Select(i => (byte)(i * 17 + 11)).ToArray(); + var writer = new XfsWriter(); + writer.AddFile("a/b.bin", payload); + var image = writer.BuildImageBytes(); + + using var stream = new MemoryStream(image, writable: false); + var profile = FormatRegistry.ProbeFilesystem("Xfs", stream); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.False); + + stream.Position = 0; + using var session = FormatRegistry.OpenFilesystem( + "Xfs", stream, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + var a = session.Lookup(session.RootNodeId, "a"); + Assert.That(a, Is.Not.Null); + var file = session.Lookup(a!.Value, "b.bin"); + Assert.That(file, Is.Not.Null); + + var stat = session.Stat(file!.Value); + Assert.That(stat.NodeId.Value, Is.GreaterThan(0)); + Assert.That(stat.Size, Is.EqualTo(payload.Length)); + + using var handle = session.OpenFile(file.Value, FileAccess.Read); + var slice = new byte[1537]; + var read = handle.Read(31_337, slice); + Assert.That(read, Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(payload.AsSpan(31_337, slice.Length).ToArray())); + } + + [Test] + public void WritableMountStaysFailClosed() { + var writer = new XfsWriter(); + writer.AddFile("x", "x"u8.ToArray()); + using var stream = new MemoryStream(writer.BuildImageBytes(), writable: true); + + Assert.Throws(() => + FormatRegistry.OpenFilesystem( + "Xfs", stream, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))); + } +} diff --git a/Compression.Tests/Mounting/DokanRuntimeProbeTests.cs b/Compression.Tests/Mounting/DokanRuntimeProbeTests.cs new file mode 100644 index 000000000..b8cd1eea1 --- /dev/null +++ b/Compression.Tests/Mounting/DokanRuntimeProbeTests.cs @@ -0,0 +1,52 @@ +using Compression.Mounting.Dokan; +using Compression.Registry; + +namespace Compression.Tests.Mounting; + +[TestFixture] +public sealed class DokanRuntimeProbeTests { + [Test] + public void BackendNeverAdvertisesUnimplementedMountModes() { + var backend = new DokanFilesystemMountBackend( + new DokanRuntimeStatus( + IsAvailable: true, + LibraryVersion: 210, + DriverVersion: 210, + LibraryPath: "dokan2.dll", + UnavailableReason: null + ) + ); + + var profile = backend.GetProfile(); + + Assert.Multiple(() => { + Assert.That(profile.Id, Is.EqualTo("dokan")); + Assert.That(profile.IsAvailable, Is.True); + Assert.That(profile.SupportsReadOnly, Is.False); + Assert.That(profile.SupportsReadWrite, Is.False); + Assert.That(profile.RequiredReadCapabilities, Is.EqualTo(FilesystemDriverCapabilities.None)); + Assert.That(profile.RequiredWriteCapabilities, Is.EqualTo(FilesystemDriverCapabilities.None)); + Assert.That( + profile.Limitations.Any(static limitation => limitation.Contains("intentionally disabled", StringComparison.Ordinal)), + Is.True + ); + }); + } + + [Test] + public void RuntimeAvailabilityRequiresBothLibraryAndDriverVersions() { + var status = DokanRuntimeProbe.Probe(); + + Assert.Multiple(() => { + Assert.That(status.IsAvailable, Is.EqualTo( + OperatingSystem.IsWindows() && status.LibraryVersion != 0 && status.DriverVersion != 0 + )); + if (status.IsAvailable) { + Assert.That(status.LibraryPath, Is.Not.Null.And.Not.Empty); + Assert.That(status.UnavailableReason, Is.Null); + } else { + Assert.That(status.UnavailableReason, Is.Not.Null.And.Not.Empty); + } + }); + } +} diff --git a/Compression.Tests/Mounting/FilesystemMountCapabilityResolverTests.cs b/Compression.Tests/Mounting/FilesystemMountCapabilityResolverTests.cs new file mode 100644 index 000000000..cba1f70f5 --- /dev/null +++ b/Compression.Tests/Mounting/FilesystemMountCapabilityResolverTests.cs @@ -0,0 +1,254 @@ +using Compression.Mounting; +using Compression.Registry; + +namespace Compression.Tests.Mounting; + +[TestFixture] +public sealed class FilesystemMountCapabilityResolverTests { + private static readonly FilesystemDriverCapabilities AllCoreCapabilities = + FilesystemMountCapabilityResolver.CoreReadCapabilities | + FilesystemMountCapabilityResolver.CoreWriteCapabilities; + + [Test] + public void DescriptorCanModifyAloneDoesNotGrantWritableMount() { + var descriptorCapabilities = FormatCapabilities.CanModify; + Assert.That(descriptorCapabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var plan = Resolve( + Profile(canMountWritable: false), + MountAccessMode.ReadWrite, + sourceCanWrite: true + ); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Contain(MountSupportReasonCode.FilesystemProfileNotWritable)); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Not.Contain(MountSupportReasonCode.MissingDriverCapabilities)); + }); + } + + [Test] + public void WholeImageRebuildNeverGrantsWritableMount() { + var plan = Resolve( + Profile(mutationModel: FilesystemMutationModel.WholeImageRebuild), + MountAccessMode.ReadWrite, + sourceCanWrite: true + ); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Contain(MountSupportReasonCode.UnsupportedMutationModel)); + }); + } + + [Test] + public void ReadOnlySourceRejectsWritableMount() { + var plan = Resolve(Profile(), MountAccessMode.ReadWrite, sourceCanWrite: false); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Contain(MountSupportReasonCode.SourceIsReadOnly)); + }); + } + + [TestCase(FilesystemDriverCapabilities.WriteData)] + [TestCase(FilesystemDriverCapabilities.Truncate)] + [TestCase(FilesystemDriverCapabilities.CreateFile)] + [TestCase(FilesystemDriverCapabilities.DeleteFile)] + [TestCase(FilesystemDriverCapabilities.CreateDirectory)] + [TestCase(FilesystemDriverCapabilities.RemoveDirectory)] + [TestCase(FilesystemDriverCapabilities.Rename)] + [TestCase(FilesystemDriverCapabilities.Flush)] + public void MissingCoreWritePrimitiveRejectsWritableMount(FilesystemDriverCapabilities missingCapability) { + var plan = Resolve( + Profile(capabilities: AllCoreCapabilities & ~missingCapability), + MountAccessMode.ReadWrite, + sourceCanWrite: true + ); + + var missingReason = plan.Reasons.Single(reason => + reason.Code == MountSupportReasonCode.MissingDriverCapabilities && + reason.MissingCapabilities.HasFlag(missingCapability)); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.MissingCapabilities, Is.EqualTo(missingCapability)); + Assert.That(missingReason.MissingCapabilities, Is.EqualTo(missingCapability)); + }); + } + + [TestCase(FilesystemDriverCapabilities.EnumerateDirectories)] + [TestCase(FilesystemDriverCapabilities.ReadData)] + [TestCase(FilesystemDriverCapabilities.RandomAccess)] + [TestCase(FilesystemDriverCapabilities.StableNodeIds)] + public void MissingCoreReadPrimitiveRejectsReadOnlyMount(FilesystemDriverCapabilities missingCapability) { + var plan = Resolve( + Profile(capabilities: AllCoreCapabilities & ~missingCapability), + MountAccessMode.ReadOnly, + sourceCanWrite: false + ); + + var missingReason = plan.Reasons.Single(reason => reason.Code == MountSupportReasonCode.MissingDriverCapabilities); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.MissingCapabilities, Is.EqualTo(missingCapability)); + Assert.That(missingReason.MissingCapabilities, Is.EqualTo(missingCapability)); + }); + } + + [TestCase(MountAccessMode.ReadOnly)] + [TestCase(MountAccessMode.ReadWrite)] + public void UnavailableBackendRejectsEveryAccessMode(MountAccessMode accessMode) { + var plan = Resolve( + Profile(), + accessMode, + sourceCanWrite: true, + backend: Backend(isAvailable: false) + ); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Contain(MountSupportReasonCode.BackendUnavailable)); + }); + } + + [Test] + public void BackendReadOnlySupportFlagIsRequired() { + var plan = Resolve( + Profile(), + MountAccessMode.ReadOnly, + sourceCanWrite: false, + backend: Backend(supportsReadOnly: false) + ); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Contain(MountSupportReasonCode.BackendDoesNotSupportReadOnly)); + }); + } + + [Test] + public void BackendReadWriteSupportFlagIsRequired() { + var plan = Resolve( + Profile(), + MountAccessMode.ReadWrite, + sourceCanWrite: true, + backend: Backend(supportsReadWrite: false) + ); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.Reasons.Select(static reason => reason.Code), + Does.Contain(MountSupportReasonCode.BackendDoesNotSupportReadWrite)); + }); + } + + [Test] + public void OptionalFilesystemCapabilitiesRemainOptional() { + var optional = + FilesystemDriverCapabilities.HardLinks | + FilesystemDriverCapabilities.SymbolicLinks | + FilesystemDriverCapabilities.SetMetadata | + FilesystemDriverCapabilities.SparseFiles | + FilesystemDriverCapabilities.Transactions; + var profile = Profile(capabilities: AllCoreCapabilities & ~optional); + + var readOnly = Resolve(profile, MountAccessMode.ReadOnly, sourceCanWrite: false); + var readWrite = Resolve(profile, MountAccessMode.ReadWrite, sourceCanWrite: true); + + Assert.Multiple(() => { + Assert.That(readOnly.IsSupported, Is.True); + Assert.That(readWrite.IsSupported, Is.True); + Assert.That(readOnly.RequiredCapabilities & optional, Is.EqualTo(FilesystemDriverCapabilities.None)); + Assert.That(readWrite.RequiredCapabilities & optional, Is.EqualTo(FilesystemDriverCapabilities.None)); + }); + } + + [Test] + public void BackendSpecificCapabilitiesAreReportedExactly() { + var requiredRead = FilesystemDriverCapabilities.SetMetadata; + var requiredWrite = FilesystemDriverCapabilities.SymbolicLinks; + var backend = Backend(requiredReadCapabilities: requiredRead, requiredWriteCapabilities: requiredWrite); + var plan = Resolve(Profile(), MountAccessMode.ReadWrite, sourceCanWrite: true, backend: backend); + + Assert.Multiple(() => { + Assert.That(plan.IsSupported, Is.False); + Assert.That(plan.MissingCapabilities, Is.EqualTo(requiredRead | requiredWrite)); + Assert.That(plan.Reasons.Where(static reason => reason.Code == MountSupportReasonCode.MissingDriverCapabilities) + .Select(static reason => reason.MissingCapabilities), + Is.EquivalentTo(new[] { requiredRead, requiredWrite })); + }); + } + + [Test] + public void RegistryRejectsDuplicateBackendIdsCaseInsensitively() { + var first = new StubBackend(Backend(id: "dokan")); + var duplicate = new StubBackend(Backend(id: "DOKAN")); + + Assert.That( + () => new MountBackendRegistry(new IFilesystemMountBackend[] { first, duplicate }), + Throws.ArgumentException + ); + } + + private static MountPlan Resolve( + FilesystemDriverProfile profile, + MountAccessMode accessMode, + bool sourceCanWrite, + MountBackendProfile? backend = null + ) => FilesystemMountCapabilityResolver.Resolve( + profile, + backend ?? Backend(), + accessMode, + sourceCanWrite + ); + + private static FilesystemDriverProfile Profile( + FilesystemDriverCapabilities? capabilities = null, + FilesystemMutationModel mutationModel = FilesystemMutationModel.Direct, + bool canMount = true, + bool canMountWritable = true + ) => new( + "testfs", + "synthetic", + capabilities ?? AllCoreCapabilities, + mutationModel, + canMount, + canMountWritable, + Array.Empty() + ); + + private static MountBackendProfile Backend( + string id = "test", + bool isAvailable = true, + bool supportsReadOnly = true, + bool supportsReadWrite = true, + FilesystemDriverCapabilities requiredReadCapabilities = FilesystemDriverCapabilities.None, + FilesystemDriverCapabilities requiredWriteCapabilities = FilesystemDriverCapabilities.None + ) => new( + id, + "Synthetic backend", + isAvailable, + supportsReadOnly, + supportsReadWrite, + requiredReadCapabilities, + requiredWriteCapabilities, + Array.Empty() + ); + + private sealed class StubBackend(MountBackendProfile profile) : IFilesystemMountBackend { + public MountBackendProfile GetProfile() => profile; + + public ValueTask MountAsync( + FilesystemMountRequest request, + CancellationToken cancellationToken = default + ) => throw new NotSupportedException(); + } +} diff --git a/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs b/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs index 13e9c5058..490751766 100644 --- a/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs +++ b/Compression.Tests/Operations/ArchiveModifyRoundTripTests.cs @@ -19,15 +19,13 @@ namespace Compression.Tests.Operations; [TestFixture] public class ArchiveModifyRoundTripTests { - // Name-preserving modifiable archive containers (probe names round-trip verbatim). private static readonly string[] NamePreservingModifiableArchives = [ "Ace", "Afs", "Ampk", "AndroidBundle", "Ba2", "Big", "Bsa", "Cbr", "Chm", - "CompactPro", "Deb", "Doc", "Dzip", "FreeArc", "Gar", "Gob", "GodotPck", + "CompactPro", "Deb", "Dmg", "Doc", "Dzip", "FreeArc", "Gar", "Gob", "GodotPck", "Grp", "Hpi", "LzxAmiga", "Mpq", "Msg", "Msi", "Msix", "Narc", "Nds", "Nsa", "Ppt", "Psarc", "Rgss", "Rpa", "Sar", "Sarc", "Slf", "Sqx", "StuffIt", "ThumbsDb", "Tnef", "U8", "Uharc", "Vpk", "Vpp", "VppV2", "Vsdx", "Wad", "Xls", "Xps", "Ypf", "Zpaq", - // previously promoted siblings that share the same contract "SevenZip", "Zip", "Tar", "Rar", "Cab", "Arj", "Zoo", "Arc", "Appx", "Apk", ]; @@ -64,7 +62,6 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { } if (ms.Length == 0) { Assert.Ignore($"{formatId}: create produced no image."); return; } - // Add a third file through the modify path. ms.Position = 0; try { modifiable.Add(ms, [ArchiveInputInfo.InMemory("C.TXT", cData)]); @@ -77,7 +74,6 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { Assert.That(Has(afterAdd, "B.BIN"), Is.True, $"{formatId}: B.BIN lost during Add (after={Join(afterAdd)})"); Assert.That(Has(afterAdd, "C.TXT"), Is.True, $"{formatId}: C.TXT missing after Add (after={Join(afterAdd)})"); - // Remove the second file through the modify path. ms.Position = 0; var bName = afterAdd.First(n => Matches(n, "B.BIN")); modifiable.Remove(ms, [bName]); @@ -86,7 +82,6 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { Assert.That(Has(afterRemove, "A.TXT"), Is.True, $"{formatId}: A.TXT lost during Remove (after={Join(afterRemove)})"); Assert.That(Has(afterRemove, "C.TXT"), Is.True, $"{formatId}: C.TXT lost during Remove (after={Join(afterRemove)})"); - // Survivors must extract byte-identically. var work = Path.Combine(Path.GetTempPath(), "cwb_modrt_" + Guid.NewGuid().ToString("N")[..8]); Directory.CreateDirectory(work); try { @@ -100,7 +95,7 @@ public void CreateAddRemove_SurvivorsStayByteIdentical(string formatId) { Assert.That(File.ReadAllBytes(a!).SequenceEqual(aData), Is.True, $"{formatId}: A.TXT content changed by modify cycle"); Assert.That(File.ReadAllBytes(c!).SequenceEqual(cData), Is.True, $"{formatId}: C.TXT content changed by modify cycle"); } finally { - try { Directory.Delete(work, true); } catch { /* best effort */ } + try { Directory.Delete(work, true); } catch { } } } @@ -113,6 +108,5 @@ private static bool Matches(string name, string user) => string.Equals(Path.GetFileName(name.Replace('\\', '/')), user, StringComparison.OrdinalIgnoreCase); private static bool Has(IEnumerable names, string user) => names.Any(n => Matches(n, user)); - private static string Join(IEnumerable names) => string.Join(",", names.Take(8)); } diff --git a/Compression.Tests/Operations/CapabilityDocumentationTests.cs b/Compression.Tests/Operations/CapabilityDocumentationTests.cs new file mode 100644 index 000000000..80c577eda --- /dev/null +++ b/Compression.Tests/Operations/CapabilityDocumentationTests.cs @@ -0,0 +1,125 @@ +#pragma warning disable CS1591 +using System.Text.RegularExpressions; +using Compression.Registry; + +namespace Compression.Tests.Operations; + +[TestFixture] +public sealed class CapabilityDocumentationTests { + [Test, Category("HappyPath")] + public void OperationCoverageFilesystemMatrixMatchesLiveRegistry() { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var document = File.ReadAllText(FindRepositoryFile("docs", "OPERATION_COVERAGE.md")); + var section = Slice(document, "## Filesystem descriptors", "## N/A notes"); + var documented = ParseFilesystemMatrix(section); + var problems = new List(); + + foreach (var id in FormatRegistry.FilesystemFormatIds) { + var descriptor = FormatRegistry.GetById(id)!; + var ops = FormatRegistry.GetArchiveOps(id); + var expected = ExpectedRow(descriptor, ops); + if (!documented.Remove(id, out var actual)) { + problems.Add($"missing row: {RenderRow(id, expected)}"); + continue; + } + + foreach (var column in expected.Keys) + if (!actual.TryGetValue(column, out var value) || value != expected[column]) + problems.Add($"{id}.{column}: documented={Render(actual.GetValueOrDefault(column))}, live={Render(expected[column])}; expected row: {RenderRow(id, expected)}"); + } + + foreach (var unexpected in documented.Keys.OrderBy(x => x, StringComparer.OrdinalIgnoreCase)) + problems.Add($"unexpected filesystem row not present in live registry: {unexpected}"); + + Assert.That(problems, Is.Empty, + "docs/OPERATION_COVERAGE.md filesystem matrix is stale. Regenerate/update it from the live registry:\n" + + string.Join("\n", problems)); + } + + [Test, Category("HappyPath")] + public void WormProseCannotNameDescriptorsThatAdvertiseCanModify() { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var document = File.ReadAllText(FindRepositoryFile("docs", "OPERATION_COVERAGE.md")); + var section = Slice(document, "### Stays WORM", "## Filesystem descriptors"); + var boldNames = Regex.Matches(section, @"\*\*([^*]+)\*\*") + .SelectMany(m => m.Groups[1].Value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var contradictions = FormatRegistry.All + .Where(d => d.Capabilities.HasFlag(FormatCapabilities.CanModify)) + .Where(d => boldNames.Contains(d.Id) || boldNames.Contains(d.DisplayName)) + .Select(d => $"{d.Id} ({d.DisplayName}) advertises CanModify but is listed under 'Stays WORM'.") + .OrderBy(x => x, StringComparer.Ordinal) + .ToArray(); + + Assert.That(contradictions, Is.Empty, + "Write-capability prose contradicts executable descriptor state:\n" + string.Join("\n", contradictions)); + } + + private static Dictionary ExpectedRow(IFormatDescriptor descriptor, IArchiveFormatOperations? ops) { + var defrag = ops is IArchiveDefragmentable; + var shrink = ops is IArchiveShrinkable; + var purge = ops is IArchivePurgeable; + var wipe = ops is IWipeEmpty or IFilesystemExtentMap or IArchiveLayoutMap; + var optimize = ops is ILayoutOptimizable || descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsOptimize); + return new Dictionary(StringComparer.OrdinalIgnoreCase) { + ["Compact"] = defrag || shrink || optimize, + ["Defrag"] = defrag, + ["Shrink"] = shrink, + ["Purge"] = purge, + ["Wipe"] = wipe, + ["Optimize"] = optimize, + }; + } + + private static Dictionary> ParseFilesystemMatrix(string section) { + var lines = section.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n'); + var headerIndex = Array.FindIndex(lines, line => line.TrimStart().StartsWith("| Format |", StringComparison.Ordinal)); + Assert.That(headerIndex, Is.GreaterThanOrEqualTo(0), "Filesystem capability table header is missing."); + var columns = SplitRow(lines[headerIndex]); + var result = new Dictionary>(StringComparer.OrdinalIgnoreCase); + for (var i = headerIndex + 2; i < lines.Length; ++i) { + if (!lines[i].TrimStart().StartsWith('|')) break; + var cells = SplitRow(lines[i]); + if (cells.Count != columns.Count || cells.Count == 0) continue; + var id = cells[0]; + if (string.IsNullOrWhiteSpace(id)) continue; + var row = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var c = 1; c < cells.Count; ++c) + row[columns[c]] = ParseMarker(cells[c]); + result[id] = row; + } + return result; + } + + private static List SplitRow(string line) + => line.Trim().Trim('|').Split('|').Select(cell => cell.Trim()).ToList(); + + private static bool ParseMarker(string marker) + => marker switch { + "Y" or "Yes" or "✅" => true, + "·" or "—" or "-" or "" => false, + _ => throw new InvalidDataException($"Unknown capability marker '{marker}'."), + }; + + private static string Render(bool value) => value ? "Y" : "·"; + + private static string RenderRow(string id, IReadOnlyDictionary row) + => $"| {id} | {Render(row["Compact"])} | {Render(row["Defrag"])} | {Render(row["Shrink"])} | {Render(row["Purge"])} | {Render(row["Wipe"])} | {Render(row["Optimize"])} |"; + + private static string Slice(string text, string startHeading, string endHeading) { + var start = text.IndexOf(startHeading, StringComparison.Ordinal); + Assert.That(start, Is.GreaterThanOrEqualTo(0), $"Missing heading '{startHeading}'."); + var end = text.IndexOf(endHeading, start + startHeading.Length, StringComparison.Ordinal); + Assert.That(end, Is.GreaterThan(start), $"Missing heading '{endHeading}' after '{startHeading}'."); + return text[start..end]; + } + + private static string FindRepositoryFile(params string[] relativeParts) { + for (var current = new DirectoryInfo(AppContext.BaseDirectory); current != null; current = current.Parent) { + var path = relativeParts.Aggregate(current.FullName, Path.Combine); + if (File.Exists(path)) return path; + } + throw new FileNotFoundException($"Could not locate repository file '{Path.Combine(relativeParts)}' from '{AppContext.BaseDirectory}'."); + } +} diff --git a/Compression.Tests/Operations/FilesystemDriverCoverageTests.cs b/Compression.Tests/Operations/FilesystemDriverCoverageTests.cs new file mode 100644 index 000000000..309c56fc8 --- /dev/null +++ b/Compression.Tests/Operations/FilesystemDriverCoverageTests.cs @@ -0,0 +1,60 @@ +using Compression.Lib; +using Compression.Registry; + +namespace Compression.Tests.Operations; + +[TestFixture] +public sealed class FilesystemDriverCoverageTests { + [OneTimeSetUp] + public void Init() => FormatRegistration.EnsureInitialized(); + + [Test] + public void EveryFileSystemProjectHasACommonDriverPath() { + var coverage = FormatRegistry.GetFilesystemDriverCoverage(); + + Assert.That(coverage, Is.Not.Empty, + "Source generation did not mark any FileSystem.* descriptors."); + + var missing = coverage + .Where(item => !item.HasDriverPath) + .Select(item => item.FormatId) + .OrderBy(id => id, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + Assert.That(missing, Is.Empty, + "Every FileSystem.* descriptor must expose a native provider/sidecar or the safe read-only archive projection."); + } + + [Test] + public void FilesystemCoverageIdsResolveToDescriptors() { + foreach (var id in FormatRegistry.FilesystemFormatIds) { + var descriptor = FormatRegistry.GetById(id); + Assert.That(descriptor, Is.Not.Null, $"Filesystem id '{id}' has no descriptor."); + Assert.That(FormatRegistry.GetFilesystemDriverCoverage(id).FormatId, + Is.EqualTo(descriptor!.Id).IgnoreCase); + } + } + + [Test] + public void NativeSidecarsAreVisibleInCoverage() { + var coverage = FormatRegistry.GetFilesystemDriverCoverage() + .ToDictionary(item => item.FormatId, StringComparer.OrdinalIgnoreCase); + + foreach (var id in FormatRegistry.FilesystemFormatIds) { + var sidecar = FormatRegistry.GetFilesystemDriver(id); + if (sidecar == null) continue; + Assert.That(coverage[id].Binding, Is.EqualTo(FilesystemDriverBindingKind.SidecarNative), + $"Generated sidecar for '{id}' is not the selected common-driver binding."); + Assert.That(coverage[id].HasNativeReadinessProvider, Is.True); + } + } + + [Test] + public void ArchiveModifyNeverCountsAsMountedWriteByItself() { + foreach (var item in FormatRegistry.GetFilesystemDriverCoverage()) { + if (!item.HasArchiveMutation || item.IsNative) continue; + Assert.That(item.Binding, Is.EqualTo(FilesystemDriverBindingKind.ArchiveProjection), + $"'{item.FormatId}' only has archive mutation; it must remain a read-only mounted projection until a native provider exists."); + } + } +} diff --git a/Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs b/Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs new file mode 100644 index 000000000..54e92283f --- /dev/null +++ b/Compression.Tests/Operations/FilesystemRwPromotionRoundTripTests.cs @@ -0,0 +1,149 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace Compression.Tests.Operations; + +[TestFixture] +public sealed class FilesystemRwPromotionRoundTripTests { + [TestCase("CramFs")] + [TestCase("SquashFs")] + [TestCase("Erofs")] + [TestCase("Msa")] + [TestCase("Pfs0")] + [Category("HappyPath"), Category("RoundTrip")] + public void CreateAddRemove_PromotedFormatsKeepSurvivorsByteExact(string formatId) { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var ops = FormatRegistry.GetArchiveOps(formatId); + Assert.That(ops, Is.Not.Null, formatId); + Assert.That(ops, Is.InstanceOf(), formatId); + Assert.That(ops, Is.InstanceOf(), formatId); + + var descriptor = FormatRegistry.All.Single(d => d.Id == formatId); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True, formatId); + + var creator = (IArchiveCreatable)ops!; + var modifier = (IArchiveModifiable)ops!; + var archiveOps = (IArchiveFormatOperations)ops!; + var a = Enumerable.Range(0, 333).Select(i => (byte)(i * 7)).ToArray(); + var b = Enumerable.Range(0, 517).Select(i => (byte)(i * 11)).ToArray(); + var c = Enumerable.Range(0, 129).Select(i => (byte)(255 - i)).ToArray(); + + using var image = new MemoryStream(); + creator.Create(image, [ + ArchiveInputInfo.InMemory("A.TXT", a), + ArchiveInputInfo.InMemory("B.BIN", b), + ], new FormatCreateOptions()); + Assert.That(image.Length, Is.GreaterThan(0), formatId); + + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory("C.DAT", c)]); + Assert.That(ListNames(archiveOps, image), Does.Contain("C.DAT").IgnoreCase, formatId); + + image.Position = 0; + modifier.Remove(image, ["B.BIN"]); + var names = ListNames(archiveOps, image); + Assert.Multiple(() => { + Assert.That(names.Any(n => Matches(n, "A.TXT")), Is.True, $"{formatId}: A.TXT disappeared"); + Assert.That(names.Any(n => Matches(n, "B.BIN")), Is.False, $"{formatId}: B.BIN survived removal"); + Assert.That(names.Any(n => Matches(n, "C.DAT")), Is.True, $"{formatId}: C.DAT disappeared"); + }); + + var work = Path.Combine(Path.GetTempPath(), "cwb_rw_promote_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try { + image.Position = 0; + archiveOps.Extract(image, work, null, null); + Assert.That(FindExtracted(work, "A.TXT"), Is.EqualTo(a), $"{formatId}: A.TXT changed"); + Assert.That(FindExtracted(work, "C.DAT"), Is.EqualTo(c), $"{formatId}: C.DAT changed"); + } finally { + try { Directory.Delete(work, recursive: true); } catch { } + } + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Ewf_ReplaceAndClearSemanticMedia_RoundTrips() { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var ops = FormatRegistry.GetArchiveOps("Ewf")!; + var descriptor = FormatRegistry.All.Single(d => d.Id == "Ewf"); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(ops, Is.InstanceOf()); + Assert.That(ops, Is.InstanceOf()); + + var original = Enumerable.Range(0, 300_123).Select(i => (byte)(i * 13)).ToArray(); + var replacement = Enumerable.Range(0, 130_777).Select(i => (byte)(i * 17)).ToArray(); + using var image = new MemoryStream(); + ((IArchiveCreatable)ops).Create(image, + [ArchiveInputInfo.InMemory("capture.raw", original)], new FormatCreateOptions()); + + Assert.That(ReadNamed((IArchiveFormatOperations)ops, image, "media.raw"), Is.EqualTo(original)); + + image.Position = 0; + ((IArchiveModifiable)ops).Add(image, [ArchiveInputInfo.InMemory("media.raw", replacement)]); + Assert.That(ReadNamed((IArchiveFormatOperations)ops, image, "media.raw"), Is.EqualTo(replacement)); + + image.Position = 0; + ((IArchiveModifiable)ops).Remove(image, ["media.raw"]); + var names = ListNames((IArchiveFormatOperations)ops, image); + Assert.That(names.Any(n => Matches(n, "media.raw")), Is.True, + "An empty EWF still represents a zero-length acquired medium."); + Assert.That(ReadNamed((IArchiveFormatOperations)ops, image, "media.raw"), Is.Empty); + } + + [Test, Category("HappyPath")] + public void Refs_AdvertisesScopedOfflineMutationAndFailsClosedOnDiagnosticImage() { + Compression.Lib.FormatRegistration.EnsureInitialized(); + var ops = FormatRegistry.GetArchiveOps("Refs")!; + var descriptor = FormatRegistry.All.Single(d => d.Id == "Refs"); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(ops, Is.InstanceOf()); + + // A header-only diagnostic image has no live namespace. The modifier must + // reject it before touching bytes rather than treating FULL.refs as content. + var imageBytes = BuildMinimalRefsHeader(); + using var image = new MemoryStream(imageBytes, writable: true); + var before = image.ToArray(); + Assert.Throws(() => + ((IArchiveModifiable)ops).Add(image, [ArchiveInputInfo.InMemory("A.TXT", [1, 2, 3])])); + Assert.That(image.ToArray(), Is.EqualTo(before)); + } + + private static List ListNames(IArchiveFormatOperations ops, MemoryStream image) { + image.Position = 0; + return ops.List(image, null).Where(e => !e.IsDirectory).Select(e => e.Name).ToList(); + } + + private static byte[] ReadNamed(IArchiveFormatOperations ops, MemoryStream image, string name) { + var work = Path.Combine(Path.GetTempPath(), "cwb_rw_extract_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try { + image.Position = 0; + ops.Extract(image, work, null, [name]); + return FindExtracted(work, name); + } finally { + try { Directory.Delete(work, recursive: true); } catch { } + } + } + + private static byte[] FindExtracted(string root, string name) { + var file = Directory.GetFiles(root, "*", SearchOption.AllDirectories) + .FirstOrDefault(f => Matches(Path.GetFileName(f), name)); + Assert.That(file, Is.Not.Null, $"Expected extracted file {name}"); + return File.ReadAllBytes(file!); + } + + private static bool Matches(string actual, string expected) + => string.Equals(Path.GetFileName(actual.Replace('\\', '/')), expected, StringComparison.OrdinalIgnoreCase); + + private static byte[] BuildMinimalRefsHeader() { + var image = new byte[4096]; + System.Text.Encoding.ASCII.GetBytes("ReFS").CopyTo(image.AsSpan(3)); + System.Text.Encoding.ASCII.GetBytes("FSRS").CopyTo(image.AsSpan(0x10)); + System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(0x14, 2), 0x200); + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(0x18, 8), 1024UL); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(0x20, 4), 512); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(0x24, 4), 8); + image[0x28] = 3; + image[0x29] = 14; + return image; + } +} \ No newline at end of file diff --git a/Compression.Tests/Operations/RebuildProgressCancellationTests.cs b/Compression.Tests/Operations/RebuildProgressCancellationTests.cs new file mode 100644 index 000000000..b557c0e31 --- /dev/null +++ b/Compression.Tests/Operations/RebuildProgressCancellationTests.cs @@ -0,0 +1,78 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace Compression.Tests.Operations; + +[TestFixture] +public sealed class RebuildProgressCancellationTests { + [Test] + public void RebuildInPlace_CancelDuringTargetWrite_LeavesOriginalUntouched() { + var original = Enumerable.Range(0, 4096).Select(i => (byte)(i * 31)).ToArray(); + using var archive = new MemoryStream(); + archive.Write(original); + archive.Position = 0; + + var descriptor = new FakeDescriptor(); + using var cts = new CancellationTokenSource(); + var phases = new List(); + + Assert.Throws(() => + RebuildVerb.RebuildInPlace(archive, descriptor, descriptor, + onProgress: e => { + phases.Add(e.Phase); + if (e.Phase == "writing" && e.CurrentWriteOffset > 0) + cts.Cancel(); + }, cancellationToken: cts.Token)); + + Assert.That(phases, Does.Contain("scanning")); + Assert.That(phases, Does.Contain("reading")); + Assert.That(phases, Does.Contain("writing")); + Assert.That(phases, Does.Not.Contain("committing")); + Assert.That(archive.ToArray(), Is.EqualTo(original), + "A cancelled staged rebuild must never overwrite the source stream."); + } + + [Test] + public void RebuildInPlace_ReportsColoredTargetAndCommitPhases() { + using var archive = new MemoryStream(); + archive.Write(new byte[4096]); + archive.Position = 0; + var descriptor = new FakeDescriptor(); + var events = new List(); + + RebuildVerb.RebuildInPlace(archive, descriptor, descriptor, onProgress: events.Add); + + Assert.That(events.Select(e => e.Phase), Does.Contain("writing")); + Assert.That(events.Select(e => e.Phase), Does.Contain("verifying")); + Assert.That(events.Select(e => e.Phase), Does.Contain("staged")); + Assert.That(events.Select(e => e.Phase), Does.Contain("committing")); + Assert.That(events.Select(e => e.Phase), Does.Contain("complete")); + + var targetMap = events.First(e => e.Phase == "writing").BlockMap; + Assert.That(targetMap, Is.Not.Null.And.Not.Empty); + Assert.That(targetMap!.Any(b => b.Kind == DefragBlockKind.Used && b.Classification.HasValue), Is.True, + "Staged archive rebuilds should keep a colored block map visible while writing."); + } + + private sealed class FakeDescriptor : IArchiveFormatOperations, IArchiveCreatable { + private static readonly byte[] Payload = Enumerable.Range(0, 256 * 1024) + .Select(i => (byte)(i * 17)).ToArray(); + + public List List(Stream stream, string? password) + => [new ArchiveEntryInfo(0, "payload.bin", Payload.Length, Payload.Length, + "deflate", false, false, null)]; + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) + => File.WriteAllBytes(Path.Combine(outputDir, "payload.bin"), Payload); + + public Stream OpenEntry(Stream archive, string entryName, string? password) + => new MemoryStream(Payload, writable: false); + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + var data = inputs.Single(i => !i.IsDirectory).ReadContent(); + const int ChunkSize = 16 * 1024; + for (var offset = 0; offset < data.Length; offset += ChunkSize) + output.Write(data, offset, Math.Min(ChunkSize, data.Length - offset)); + } + } +} diff --git a/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs b/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs index f76aac18b..fff90c079 100644 --- a/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs +++ b/Compression.Tests/Operations/WriteCapabilityHonestyTests.cs @@ -7,22 +7,18 @@ namespace Compression.Tests.Operations; /// Honesty guard for the WORM-vs-R/W capability claim. The write scale /// (Unsupported → Read-Only → WORM → R/W) is only meaningful if /// is reserved for formats that genuinely -/// support modifying an existing container. +/// support modifying an existing container/image. /// /// R/W means a working add / replace / remove on an existing instance that yields a -/// valid result. The edit may be byte-preserving in place or may relayout / -/// re-pack the container (moving existing data) — both are honest R/W for a conceptually -/// read-write format. What is NOT honest is advertising -/// with no working modify path at all. Read-only-by-design formats (CramFS, SquashFS) and -/// create-only formats stay WORM () even though a -/// rebuild could synthesise a modified copy — they do not advertise R/W. +/// valid result. The edit may be byte-preserving in place, append replacement state, +/// relayout members, or rebuild the image. Those are implementation choices. What is not +/// honest is advertising with no working edit path. +/// A format that can only create a fresh instance remains WORM. /// /// -/// This test enforces the deterministic half of that rule for every registered format that -/// claims : its runtime ops object must actually -/// implement (otherwise the R/W claim is entirely unbacked — -/// there is no modify path). That the modify works (round-trips) is verified -/// separately by the registry-driven Generic{Purge,Defrag}RoundTripTests. +/// This test enforces the deterministic half of that rule for every registered claimant: +/// its runtime ops object must implement . Behavioural +/// round-trip tests verify that individual modify paths actually work. /// /// [TestFixture] @@ -42,7 +38,7 @@ public void EveryCanModifyClaimIsBackedByAModifyPath(string formatId) { Assert.That(ops, Is.InstanceOf(), $"{formatId} advertises R/W (CanModify) but its ops does not implement IArchiveModifiable — " - + "the claim is unbacked. Implement IArchiveModifiable (in-place or relayout/rebuild) " + + "the claim is unbacked. Implement a working existing-instance edit path " + "or downgrade to WORM (CanCreate only)."); } -} +} \ No newline at end of file diff --git a/Compression.Tests/OrangeFs/OrangeFsWriteTests.cs b/Compression.Tests/OrangeFs/OrangeFsWriteTests.cs new file mode 100644 index 000000000..42cdfcb26 --- /dev/null +++ b/Compression.Tests/OrangeFs/OrangeFsWriteTests.cs @@ -0,0 +1,46 @@ +using Compression.Registry; +using FileSystem.OrangeFs; + +namespace Compression.Tests.OrangeFs; + +[TestFixture] +public sealed class OrangeFsWriteTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void CreateReplaceRemove_RoundTripsObjectPayloadAndPreservesHeaderIdentity() { + var descriptor = new OrangeFsFormatDescriptor(); + var first = Enumerable.Range(0, 97).Select(i => (byte)(i * 3)).ToArray(); + var replacement = Enumerable.Range(0, 513).Select(i => (byte)(i * 7)).ToArray(); + using var image = new MemoryStream(); + + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory("object.bin", first)], new FormatCreateOptions()); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + + var before = image.ToArray()[..12]; + AssertPayload(image, first); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory("object.bin", replacement)]); + Assert.That(image.ToArray()[..12], Is.EqualTo(before)); + AssertPayload(image, replacement); + + image.Position = 0; + ((IArchiveModifiable)descriptor).Remove(image, ["object.bin"]); + Assert.That(image.Length, Is.EqualTo(16)); + AssertPayload(image, []); + } + + private static void AssertPayload(MemoryStream image, byte[] expected) { + image.Position = 0; + using var reader = new OrangeFsReader(image); + var objectEntry = reader.Entries.FirstOrDefault(e => e.Name == "object.bin"); + if (expected.Length == 0) { + Assert.That(objectEntry, Is.Null); + Assert.That(reader.ObjectSize, Is.Zero); + return; + } + Assert.That(objectEntry, Is.Not.Null); + Assert.That(reader.Extract(objectEntry!), Is.EqualTo(expected)); + } +} diff --git a/Compression.Tests/UefiFv/UefiFvWriteTests.cs b/Compression.Tests/UefiFv/UefiFvWriteTests.cs new file mode 100644 index 000000000..64f6dd99d --- /dev/null +++ b/Compression.Tests/UefiFv/UefiFvWriteTests.cs @@ -0,0 +1,75 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.UefiFv; + +namespace Compression.Tests.UefiFv; + +[TestFixture] +public sealed class UefiFvWriteTests { + private const string DriverName = "11223344-5566-7788-99aa-bbccddeeff00_DRIVER.bin"; + private const string RawName = "01234567-89ab-cdef-0123-456789abcdef_RAW.bin"; + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_Create_ProducesChecksummedFixedCapacityFv() { + var payload = Enumerable.Range(0, 333).Select(i => (byte)(i * 13)).ToArray(); + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, payload)], new FormatCreateOptions()); + + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(image.Length, Is.GreaterThan(payload.Length + 64 * 1024)); + + var bytes = image.ToArray(); + var fv = UefiFvReader.Read(bytes); + Assert.That(fv.Files, Has.Count.EqualTo(1)); + Assert.That(fv.Files[0].Contents, Is.EqualTo(payload)); + + uint sum = 0; + for (var i = 0; i < fv.Header.HeaderLength; i += 2) + sum += BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(i, 2)); + Assert.That((ushort)sum, Is.Zero, "FV header checksum must sum to zero"); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_AddReplaceRemove_ReusesErasedSpaceWithoutGrowingVolume() { + var descriptor = new UefiFvFormatDescriptor(); + var first = Enumerable.Range(0, 257).Select(i => (byte)i).ToArray(); + var second = Enumerable.Range(0, 721).Select(i => (byte)(i * 5)).ToArray(); + var replacement = Enumerable.Range(0, 1023).Select(i => (byte)(255 - i)).ToArray(); + + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, first)], new FormatCreateOptions()); + var originalLength = image.Length; + + var modifier = (IArchiveModifiable)descriptor; + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory(RawName, second)]); + Assert.That(image.Length, Is.EqualTo(originalLength)); + AssertFiles(image, (DriverName, first), (RawName, second)); + + image.Position = 0; + modifier.Add(image, [ArchiveInputInfo.InMemory(DriverName, replacement)]); + Assert.That(image.Length, Is.EqualTo(originalLength)); + AssertFiles(image, (DriverName, replacement), (RawName, second)); + + image.Position = 0; + modifier.Remove(image, [RawName]); + Assert.That(image.Length, Is.EqualTo(originalLength)); + AssertFiles(image, (DriverName, replacement)); + } + + private static void AssertFiles(MemoryStream image, params (string Name, byte[] Data)[] expected) { + var bytes = image.ToArray(); + var fv = UefiFvReader.Read(bytes); + var actual = fv.Files.Where(f => f.Type != 0xF0) + .ToDictionary(f => $"{f.Name:D}_{UefiFvReader.ShortTypeTag(f.Type)}.bin", f => f.Contents, + StringComparer.OrdinalIgnoreCase); + Assert.That(actual.Keys, Is.EquivalentTo(expected.Select(e => e.Name))); + foreach (var (name, data) in expected) + Assert.That(actual[name], Is.EqualTo(data), name); + } +} diff --git a/Compression.Tests/Zfs/ZfsFilesystemDriverTests.cs b/Compression.Tests/Zfs/ZfsFilesystemDriverTests.cs new file mode 100644 index 000000000..1ed974b51 --- /dev/null +++ b/Compression.Tests/Zfs/ZfsFilesystemDriverTests.cs @@ -0,0 +1,80 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileSystem.Zfs; + +namespace Compression.Tests.Zfs; + +[TestFixture] +public sealed class ZfsFilesystemDriverTests { + [Test, Category("HappyPath"), Category("RoundTrip")] + public void NativeSession_UsesDnodeIdentityAndSupportsPositionalReads() { + var payload = new byte[48 * 1024 + 137]; + for (var i = 0; i < payload.Length; i++) payload[i] = (byte)((i * 29 + 7) & 0xFF); + + var writer = new ZfsWriter(); + writer.AddFile("dir/data.bin", payload); + using var image = new MemoryStream(); + writer.WriteTo(image, 8L * 1024 * 1024); + + var adapter = new ZfsFilesystemDriverAdapter(); + image.Position = 0; + var profile = adapter.ProbeFilesystem(image); + Assert.That(profile.CanMount, Is.True, string.Join("; ", profile.Limitations)); + Assert.That(profile.CanMountWritable, Is.False); + Assert.That(profile.Capabilities.HasFlag(FilesystemDriverCapabilities.StableNodeIds), Is.True); + Assert.That(profile.Capabilities.HasFlag(FilesystemDriverCapabilities.RandomAccess), Is.True); + + image.Position = 0; + using var session = adapter.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: true, LeaveOpen: true)); + var dirId = session.Lookup(session.RootNodeId, "dir"); + Assert.That(dirId, Is.Not.Null); + var fileId = session.Lookup(dirId!.Value, "data.bin"); + Assert.That(fileId, Is.Not.Null); + Assert.That(fileId!.Value.Value, Is.GreaterThan(0UL), "regular-file node must carry the native dataset dnode object id"); + + using var handle = session.OpenFile(fileId.Value, FileAccess.Read); + Assert.That(handle.Length, Is.EqualTo(payload.Length)); + var slice = new byte[733]; + var read = handle.Read(12_345, slice); + Assert.That(read, Is.EqualTo(slice.Length)); + Assert.That(slice, Is.EqualTo(payload.AsSpan(12_345, slice.Length).ToArray())); + } + + [Test, Category("ErrorHandling")] + public void Probe_RejectsUnsupportedPoolVersion() { + var writer = new ZfsWriter(); + writer.AddFile("a.bin", "abc"u8.ToArray()); + using var built = new MemoryStream(); + writer.WriteTo(built, 8L * 1024 * 1024); + var bytes = built.ToArray(); + + var changed = 0; + for (var slot = ZfsConstants.UberblockArrayOffset; + slot + ZfsConstants.UberblockSize <= ZfsConstants.LabelSize; + slot += ZfsConstants.UberblockSize) { + if (BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(slot, 8)) != ZfsConstants.UberblockMagic) + continue; + BinaryPrimitives.WriteUInt64LittleEndian(bytes.AsSpan(slot + 8, 8), ZfsConstants.PoolVersion + 1); + changed++; + } + Assert.That(changed, Is.GreaterThan(0), "writer must emit at least one valid L0 uberblock"); + + using var image = new MemoryStream(bytes, writable: false); + var profile = new ZfsFilesystemDriverAdapter().ProbeFilesystem(image); + Assert.That(profile.CanMount, Is.False); + Assert.That(string.Join("; ", profile.Limitations), Does.Contain("pool version")); + } + + [Test, Category("ErrorHandling")] + public void NativeSession_RefusesWritableMount() { + var writer = new ZfsWriter(); + writer.AddFile("a.bin", "abc"u8.ToArray()); + using var image = new MemoryStream(); + writer.WriteTo(image, 8L * 1024 * 1024); + + var adapter = new ZfsFilesystemDriverAdapter(); + image.Position = 0; + Assert.Throws(() => + adapter.OpenFilesystem(image, new FilesystemOpenOptions(ReadOnly: false, LeaveOpen: true))); + } +} diff --git a/Compression.UI/App.xaml.cs b/Compression.UI/App.xaml.cs index 8b09005d1..477fd11da 100644 --- a/Compression.UI/App.xaml.cs +++ b/Compression.UI/App.xaml.cs @@ -13,6 +13,12 @@ protected override void OnStartup(System.Windows.StartupEventArgs e) { if (System.Environment.GetEnvironmentVariable("COMPRESSIONWORKBENCH_WINE") == "1") System.Windows.Media.RenderOptions.ProcessRenderMode = System.Windows.Interop.RenderMode.SoftwareOnly; + // Documentation CI asks the real application to render its own visual tree. + // Handle that before background registry warm-up and normal startup so the + // captures are deterministic and never depend on desktop automation. + if (ScreenshotMode.TryRun(this, e.Args)) + return; + // Warm the format registry on a background thread so the first user-driven // CanExecute / right-click that calls FormatDetector.DetectByExtension or // FormatRegistry.GetArchiveOps doesn't pay the ~180-descriptor registration @@ -173,4 +179,4 @@ private void HandleExtractArchive(string archivePath) { Shutdown(); } -} +} \ No newline at end of file diff --git a/Compression.UI/Compression.UI.csproj b/Compression.UI/Compression.UI.csproj index cd161d555..60e6d7426 100644 --- a/Compression.UI/Compression.UI.csproj +++ b/Compression.UI/Compression.UI.csproj @@ -31,6 +31,7 @@ + diff --git a/Compression.UI/ScreenshotMode.cs b/Compression.UI/ScreenshotMode.cs new file mode 100644 index 000000000..105911110 --- /dev/null +++ b/Compression.UI/ScreenshotMode.cs @@ -0,0 +1,154 @@ +using System.IO; +using System.Windows; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Threading; +using Compression.Lib; + +namespace Compression.UI; + +/// +/// Deterministic, interaction-free screenshot generation for documentation CI. +/// The application renders its own WPF visual tree instead of relying on desktop +/// automation, so captures do not depend on focus, monitor geometry, or runner timing. +/// +internal static class ScreenshotMode { + private const double CaptureWidth = 1200; + private const double CaptureHeight = 760; + + public static bool TryRun(Application application, string[] args) { + if (args.Length == 0 || !string.Equals(args[0], "--screenshots", StringComparison.OrdinalIgnoreCase)) + return false; + + application.ShutdownMode = ShutdownMode.OnExplicitShutdown; + var outputDirectory = Path.GetFullPath( + args.Length > 1 ? args[1] : Path.Combine("docs", "screenshots")); + + try { + Directory.CreateDirectory(outputDirectory); + Console.WriteLine($"Screenshot output: {outputDirectory}"); + + var fixtureRoot = Path.Combine(Path.GetTempPath(), "CompressionWorkbench-Screenshots"); + RecreateDirectory(fixtureRoot); + + try { + Console.WriteLine("Creating deterministic archive fixture..."); + var archivePath = CreateArchiveFixture(fixtureRoot); + + Console.WriteLine("Capturing archive browser..."); + var mainWindow = new MainWindow { + Width = CaptureWidth, + Height = CaptureHeight, + }; + mainWindow.OpenArchive(archivePath); + Capture(mainWindow, Path.Combine(outputDirectory, "archive-browser.png")); + + Console.WriteLine("Capturing analysis window..."); + Capture(new Views.AnalysisWindow { + Width = CaptureWidth, + Height = CaptureHeight, + }, Path.Combine(outputDirectory, "analysis.png")); + + Console.WriteLine("Capturing maintenance window..."); + Capture(new Views.DefragmentWindow { + Width = CaptureWidth, + Height = CaptureHeight, + }, Path.Combine(outputDirectory, "maintenance.png")); + } + finally { + try { Directory.Delete(fixtureRoot, recursive: true); } + catch { /* CI fixture cleanup is best-effort. */ } + } + + Console.WriteLine("Screenshot generation completed successfully."); + application.Shutdown(0); + } + catch (Exception ex) { + var diagnostic = $"Screenshot generation failed:{Environment.NewLine}{ex}"; + Console.Error.WriteLine(diagnostic); + System.Diagnostics.Trace.WriteLine(diagnostic); + try { + Directory.CreateDirectory(outputDirectory); + File.WriteAllText(Path.Combine(outputDirectory, "screenshot-error.txt"), diagnostic); + } + catch { /* The original exception is the useful failure. */ } + application.Shutdown(1); + } + + return true; + } + + private static string CreateArchiveFixture(string fixtureRoot) { + var inputRoot = Path.Combine(fixtureRoot, "CompressionWorkbench-demo"); + var docs = Path.Combine(inputRoot, "docs"); + var source = Path.Combine(inputRoot, "src"); + Directory.CreateDirectory(docs); + Directory.CreateDirectory(source); + + var readme = Path.Combine(inputRoot, "README.txt"); + var vision = Path.Combine(docs, "vision.txt"); + var codec = Path.Combine(source, "Codec.cs"); + var payloadPath = Path.Combine(inputRoot, "payload.bin"); + + File.WriteAllText(readme, + "CompressionWorkbench demo archive\n\nEvery payload is generated locally by screenshot CI.\n"); + File.WriteAllText(vision, + "One tool for codecs, archives, pseudo-archives, filesystems, analysis and maintenance.\n"); + File.WriteAllText(codec, + "namespace Demo;\n\ninternal static class Codec { public const string Method = \"Deflate\"; }\n"); + + var payload = new byte[4096]; + for (var i = 0; i < payload.Length; i++) + payload[i] = (byte)((i * 37 + i / 7) & 0xff); + File.WriteAllBytes(payloadPath, payload); + + // Archive listings include modification timestamps. Pin them so repeated CI + // captures are byte-stable when the UI itself has not changed. + var timestamp = new DateTime(2024, 1, 2, 12, 34, 56, DateTimeKind.Utc); + foreach (var path in new[] { readme, vision, codec, payloadPath }) + File.SetLastWriteTimeUtc(path, timestamp); + + var archivePath = Path.Combine(fixtureRoot, "CompressionWorkbench-demo.zip"); + var inputs = ArchiveInput.Resolve([inputRoot]); + ArchiveOperations.Create(archivePath, inputs, new CompressionOptions()); + return archivePath; + } + + private static void Capture(Window window, string outputPath) { + window.WindowStartupLocation = WindowStartupLocation.Manual; + window.Left = -10000; + window.Top = -10000; + window.ShowInTaskbar = false; + window.ShowActivated = false; + window.Show(); + + DrainDispatcher(window.Dispatcher); + window.UpdateLayout(); + + var width = Math.Max(1, (int)Math.Ceiling(window.ActualWidth)); + var height = Math.Max(1, (int)Math.Ceiling(window.ActualHeight)); + Console.WriteLine($"Rendering {Path.GetFileName(outputPath)} at {width}x{height}..."); + var bitmap = new RenderTargetBitmap(width, height, 96, 96, PixelFormats.Pbgra32); + bitmap.Render(window); + + var encoder = new PngBitmapEncoder(); + encoder.Frames.Add(BitmapFrame.Create(bitmap)); + using (var output = File.Create(outputPath)) + encoder.Save(output); + + window.Close(); + DrainDispatcher(window.Dispatcher); + } + + private static void DrainDispatcher(Dispatcher dispatcher) { + var frame = new DispatcherFrame(); + dispatcher.BeginInvoke(DispatcherPriority.ApplicationIdle, new Action(() => frame.Continue = false)); + Dispatcher.PushFrame(frame); + } + + private static void RecreateDirectory(string path) { + if (Directory.Exists(path)) + Directory.Delete(path, recursive: true); + Directory.CreateDirectory(path); + } +} diff --git a/Compression.UI/Views/DefragmentWindow.RebuildProgress.cs b/Compression.UI/Views/DefragmentWindow.RebuildProgress.cs new file mode 100644 index 000000000..68bea5959 --- /dev/null +++ b/Compression.UI/Views/DefragmentWindow.RebuildProgress.cs @@ -0,0 +1,522 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Windows; +using System.Windows.Controls; +using Button = System.Windows.Controls.Button; +using Compression.Lib; +using Compression.Registry; + +namespace Compression.UI.Views; + +/// +/// Universal live block-map/cancellation layer for maintenance operations that +/// rebuild or re-group a container instead of physically moving blocks in place. +/// Kept as a partial so it can reuse the existing maintenance window controls and +/// helpers without duplicating the large preview/rendering implementation. +/// +public partial class DefragmentWindow { + private bool _rebuildProgressHooked; + private Button? _maintenanceCancelButton; + private CancellationTokenSource? _maintenanceCancellation; + private bool _maintenanceIsStaged; + private bool _maintenanceCommitStarted; + private string? _maintenanceOperationName; + + protected override void OnContentRendered(EventArgs e) { + base.OnContentRendered(e); + if (this._rebuildProgressHooked) return; + this._rebuildProgressHooked = true; + + // Replace only the main Run dispatch. Shrink/Purge/Wipe/Compact keep their + // existing handlers; rebuild-backed Defrag/Optimize now share this richer UI. + RunBtn.Click -= OnRun; + RunBtn.Click += OnRunWithBlockProgress; + InsertMaintenanceCancelButton(); + + // The original loader sees IArchiveDefragmentable before archive-repack + // support. When the caller explicitly asked for Optimize, correct that + // ambiguity here so ZIP/7z can expose their repack UI instead of looking like + // filesystem-only defraggers. + if (this._requestedVerb == MaintenanceVerb.Optimize && this._formatId is { Length: > 0 } id) { + var descriptor = FormatRegistry.GetById(id); + var ops = FormatRegistry.GetArchiveOps(id); + if (descriptor?.Category is FormatCategory.Archive or FormatCategory.CompoundTar + && ops is IArchiveCreatable) { + this._isArchiveMode = true; + this._archiveOps = ops; + this._isSevenZipFormat = string.Equals(id, "SevenZip", StringComparison.Ordinal); + FsModesGroup.Visibility = Visibility.Collapsed; + ArchiveRepackGroup.Visibility = Visibility.Visible; + SmartSolidRepackCheck.Visibility = this._isSevenZipFormat ? Visibility.Visible : Visibility.Collapsed; + RunBtn.Content = "Optimize"; + RunBtn.IsEnabled = true; + SupportLbl.Text = "Archive re-layout/repack with live staged-target visualization."; + SupportLbl.Foreground = System.Windows.Media.Brushes.DarkGreen; + if (LayoutStatusLbl != null) + LayoutStatusLbl.Text = "Source + staged-target address spaces share the chart for progress; offsets are projected, not physical equivalence."; + } + } + } + + protected override void OnClosing(CancelEventArgs e) { + base.OnClosing(e); + if (e.Cancel || this._maintenanceCancellation == null) return; + + e.Cancel = true; + RequestMaintenanceCancellation(confirmNativeInPlace: true); + } + + private void InsertMaintenanceCancelButton() { + if (this._maintenanceCancelButton != null || RunBtn.Parent is not StackPanel panel) + return; + + var close = panel.Children.OfType