From 44983dbe4ba90bddf0c3bd7401cf29ceadd7d0de Mon Sep 17 00:00:00 2001 From: bitgorust Date: Sun, 16 Aug 2026 20:03:52 +0200 Subject: [PATCH 1/7] docs: propose durable artifact lifecycle --- .../change.json | 31 ++ .../delta.md | 217 ++++++++++++++ .../design.md | 69 +++++ .../evidence.md | 5 + .../proposal.md | 52 ++++ .../tasks.md | 12 + .../artifact-lifecycle-surfaces/change.json | 33 +++ .../artifact-lifecycle-surfaces/delta.md | 266 ++++++++++++++++++ .../artifact-lifecycle-surfaces/design.md | 89 ++++++ .../artifact-lifecycle-surfaces/evidence.md | 5 + .../artifact-lifecycle-surfaces/proposal.md | 55 ++++ .../artifact-lifecycle-surfaces/tasks.md | 12 + .../change.json | 28 ++ .../artifact-publication-transaction/delta.md | 145 ++++++++++ .../design.md | 71 +++++ .../evidence.md | 5 + .../proposal.md | 47 ++++ .../artifact-publication-transaction/tasks.md | 13 + .../artifact-state-cas-limits/change.json | 29 ++ .../artifact-state-cas-limits/delta.md | 171 +++++++++++ .../artifact-state-cas-limits/design.md | 75 +++++ .../artifact-state-cas-limits/evidence.md | 5 + .../artifact-state-cas-limits/proposal.md | 47 ++++ .../artifact-state-cas-limits/tasks.md | 11 + 24 files changed, 1493 insertions(+) create mode 100644 specs/changes/artifact-identity-schema-migration/change.json create mode 100644 specs/changes/artifact-identity-schema-migration/delta.md create mode 100644 specs/changes/artifact-identity-schema-migration/design.md create mode 100644 specs/changes/artifact-identity-schema-migration/evidence.md create mode 100644 specs/changes/artifact-identity-schema-migration/proposal.md create mode 100644 specs/changes/artifact-identity-schema-migration/tasks.md create mode 100644 specs/changes/artifact-lifecycle-surfaces/change.json create mode 100644 specs/changes/artifact-lifecycle-surfaces/delta.md create mode 100644 specs/changes/artifact-lifecycle-surfaces/design.md create mode 100644 specs/changes/artifact-lifecycle-surfaces/evidence.md create mode 100644 specs/changes/artifact-lifecycle-surfaces/proposal.md create mode 100644 specs/changes/artifact-lifecycle-surfaces/tasks.md create mode 100644 specs/changes/artifact-publication-transaction/change.json create mode 100644 specs/changes/artifact-publication-transaction/delta.md create mode 100644 specs/changes/artifact-publication-transaction/design.md create mode 100644 specs/changes/artifact-publication-transaction/evidence.md create mode 100644 specs/changes/artifact-publication-transaction/proposal.md create mode 100644 specs/changes/artifact-publication-transaction/tasks.md create mode 100644 specs/changes/artifact-state-cas-limits/change.json create mode 100644 specs/changes/artifact-state-cas-limits/delta.md create mode 100644 specs/changes/artifact-state-cas-limits/design.md create mode 100644 specs/changes/artifact-state-cas-limits/evidence.md create mode 100644 specs/changes/artifact-state-cas-limits/proposal.md create mode 100644 specs/changes/artifact-state-cas-limits/tasks.md diff --git a/specs/changes/artifact-identity-schema-migration/change.json b/specs/changes/artifact-identity-schema-migration/change.json new file mode 100644 index 0000000..ed505c5 --- /dev/null +++ b/specs/changes/artifact-identity-schema-migration/change.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": 1, + "id": "artifact-identity-schema-migration", + "title": "Introduce durable artifact identity and schema migration", + "lane": "high-risk", + "status": "draft", + "affectedRequirements": [ + "LIFE-01", + "LIFE-02", + "LIFE-07", + "OPS-03", + "OPS-05", + "OPS-07", + "COMPAT-03", + "COMPAT-04", + "QUAL-02" + ], + "currentSpecs": [], + "currentSpecsUpdated": false, + "approval": { + "by": "", + "at": "" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-16", + "archivedAt": null +} diff --git a/specs/changes/artifact-identity-schema-migration/delta.md b/specs/changes/artifact-identity-schema-migration/delta.md new file mode 100644 index 0000000..1b1138b --- /dev/null +++ b/specs/changes/artifact-identity-schema-migration/delta.md @@ -0,0 +1,217 @@ +# Specification delta: Introduce durable artifact identity and schema migration + +## MODIFIED + +### Requirement: LIFE-01 + +Every artifact has a generated opaque ID independent of title, slug, path, content hash, and +deployment URL. A unique human-readable slug is a mutable reference; renaming it preserves the +ID, revisions, state association, and deployment references. + +#### Scenario: Normal behavior + +- **Given:** an existing artifact is renamed +- **When:** the new slug is committed +- **Then:** the artifact ID and history remain unchanged and the new stable path resolves to it + +#### Scenario: Failure or refusal + +- **Given:** a requested slug belongs to another active artifact +- **When:** rename or migration validates the change +- **Then:** it refuses without changing either artifact + +#### Scenario: Relevant boundary + +- **Given:** two legacy entries have the same recoverable content or title +- **When:** they migrate +- **Then:** each receives its own opaque ID unless legacy evidence explicitly proves one identity + +### Requirement: LIFE-02 + +Every successful create, update, rename-with-content, and restore produces one immutable, +monotonically numbered revision record and retained portable-page bytes; history is not +conditional on a flag. + +#### Scenario: Normal behavior + +- **Given:** an artifact at revision 2 +- **When:** an update succeeds +- **Then:** revision 3 is retained and revisions 1 and 2 remain byte-for-byte unchanged + +#### Scenario: Failure or refusal + +- **Given:** a write fails before commit +- **When:** recovery inspects history +- **Then:** no successful revision number is skipped or partially materialized + +#### Scenario: Relevant boundary + +- **Given:** a legacy unversioned page has only one recoverable byte sequence +- **When:** it migrates +- **Then:** exactly one revision is recorded and no earlier content is invented + +### Requirement: LIFE-07 + +Schema-versioned metadata records ID, slug, title, icon, description, timestamps, head, +immutable revision metadata, byte size, content hash, provenance, author when known, and +deployment references, with exact validation and repair diagnostics. + +#### Scenario: Normal behavior + +- **Given:** a valid schema-2 manifest +- **When:** it is read +- **Then:** all artifact and revision fields validate and the selected head matches retained bytes + +#### Scenario: Failure or refusal + +- **Given:** duplicate IDs/slugs, an invalid head, unsafe reference, or mismatched hash +- **When:** the manifest is read or migrated +- **Then:** a typed error or repair report is returned instead of an empty manifest + +#### Scenario: Relevant boundary + +- **Given:** author or deployment information was never known +- **When:** metadata is produced +- **Then:** absence is represented explicitly without fabricated values + +### Requirement: OPS-03 + +Local lifecycle migration creates a verified exact backup, exposes integrity results, and +supports a tested restore of the prior selected store before a new schema is default-enabled. + +#### Scenario: Normal behavior + +- **Given:** a valid legacy store +- **When:** migration completes and its backup is restored in a drill +- **Then:** the restored old bytes and selection match the pre-migration inventory + +#### Scenario: Failure or refusal + +- **Given:** backup creation or verification fails +- **When:** migration is requested +- **Then:** selection remains on the old store and the failure names the next safe action + +#### Scenario: Relevant boundary + +- **Given:** a backup contains private artifact content +- **When:** evidence is retained +- **Then:** only hashes and synthetic fixture results are recorded, not the content + +### Requirement: OPS-05 + +Schema migration has explicit preflight, staged preparation, verification, selection, +post-change verification, and rollback; failed rollout preserves or restores the last known +good compatible state. + +#### Scenario: Normal behavior + +- **Given:** preflight and staged verification pass +- **When:** schema 2 is selected +- **Then:** post-change verification proves all artifact heads and revisions readable + +#### Scenario: Failure or refusal + +- **Given:** interruption or validation failure at any migration boundary +- **When:** recovery runs +- **Then:** it selects a complete old or complete new store and reports which one + +#### Scenario: Relevant boundary + +- **Given:** the filesystem platform lacks required migration evidence +- **When:** default enablement is evaluated +- **Then:** schema 2 remains disabled on that platform + +### Requirement: OPS-07 + +Inspection, migration, repair, and rollback are idempotent or resumable and emit bounded +machine-readable progress/results without requiring undocumented direct store edits. + +#### Scenario: Normal behavior + +- **Given:** an interrupted prepared migration +- **When:** the command is rerun +- **Then:** it resumes or safely restarts from recorded state and reaches the same result + +#### Scenario: Failure or refusal + +- **Given:** the recorded operation token conflicts with current store identity +- **When:** resume is attempted +- **Then:** it refuses mutation and provides a bounded repair report + +#### Scenario: Relevant boundary + +- **Given:** a completed migration is invoked again +- **When:** inputs are unchanged +- **Then:** it reports the prior completion without creating another backup or identity + +### Requirement: COMPAT-03 + +Manifest, revision, state-export, and provider-migration records carry integer schema +versions. Migrations are forward-only, backed up, idempotent, and fault-tested from every +released prior shape; unknown future versions fail without mutation. + +#### Scenario: Normal behavior + +- **Given:** any released legacy fixture +- **When:** it migrates to schema 2 +- **Then:** all recoverable content and metadata validate in the new schema + +#### Scenario: Failure or refusal + +- **Given:** a manifest with a schema version newer than the runtime understands +- **When:** any read-write operation opens it +- **Then:** the operation fails before writing or selecting a replacement + +#### Scenario: Relevant boundary + +- **Given:** historical Cloudflare state uses a shared-KV key shape +- **When:** the offline provider migration fixture runs +- **Then:** records are mapped to explicit site/artifact scope or reported ambiguous without provider mutation + +### Requirement: COMPAT-04 + +Upgrade and rollback preserve artifact IDs, slug references, revision selection, state +association, and deployment references; ambiguous legacy mappings are reported instead of +cross-wired. + +#### Scenario: Normal behavior + +- **Given:** a legacy artifact with local state and deployment metadata +- **When:** upgrade and rollback round-trip +- **Then:** every association returns to the same artifact and selected revision + +#### Scenario: Failure or refusal + +- **Given:** two legacy records cannot be safely associated with one state namespace +- **When:** migration preflight runs +- **Then:** it refuses that association and names the ambiguous records + +#### Scenario: Relevant boundary + +- **Given:** an artifact slug changed after migration +- **When:** a later package upgrade runs +- **Then:** it follows opaque identity rather than attaching data by the old title or slug + +### Requirement: QUAL-02 + +Identity, schema validation, migration, backup, repair, and rollback behavior has deterministic +unit, property, fixture, and fault tests independent of network, wall-clock ordering, and +developer-specific paths. + +#### Scenario: Normal behavior + +- **Given:** the synthetic lifecycle fixture corpus +- **When:** the deterministic suite runs +- **Then:** all normal migration and identity properties pass with fixed observable inputs + +#### Scenario: Failure or refusal + +- **Given:** a migration path has no failure or rollback test +- **When:** implementation verification runs +- **Then:** the packet and Phase 1 gate fail + +#### Scenario: Relevant boundary + +- **Given:** a real supported filesystem result is unavailable +- **When:** unit tests pass elsewhere +- **Then:** the platform remains unverified and is not inferred from synthetic evidence diff --git a/specs/changes/artifact-identity-schema-migration/design.md b/specs/changes/artifact-identity-schema-migration/design.md new file mode 100644 index 0000000..b2d474c --- /dev/null +++ b/specs/changes/artifact-identity-schema-migration/design.md @@ -0,0 +1,69 @@ +# Design: Introduce durable artifact identity and schema migration + +Required for high-risk changes. + +## Context and constraints + +Slug-keyed manifest entries currently combine identity, presentation path, mutable head, and +partial history. Reads catch every error and return an empty manifest, which converts damage +or an unknown schema into apparent absence. The new contract must preserve directly openable +portable HTML, avoid a new dependency, tolerate interruption, and keep old data recoverable. +It must also allow later transactions and exports to use one canonical identity. No platform +may be claimed safe from unit tests on a different filesystem. + +## Chosen design + +Use manifest schema version 2. `ArtifactRecord` is keyed by a random UUID artifact ID and +contains the current unique slug, presentation metadata, created/updated timestamps, selected +head number, ordered `RevisionRecord`s, and deployment references. Each revision records its +monotonic number, immutable content hash, byte size, timestamp, provenance, author when known, +and stored portable-page reference. A validated slug index resolves paths without making the +slug identity. Duplicate IDs, duplicate active slugs, revision gaps/duplicates, mismatched +heads, invalid timestamps, unsafe references, and metadata/file hash disagreements are typed +errors rather than empty-store fallbacks. + +Migration first inventories the old manifest and files without writing. It assigns one opaque +ID per recoverable legacy artifact, imports every existing numbered page in order, and imports +the stable head only when its bytes are not already represented. It never creates a revision +for an advertised file that does not exist; the repair report names missing, orphaned, +duplicate, corrupt, and irrecoverable entries. A canonical migration library also maps the +historical shared Cloudflare KV key shapes to site-scoped export records for later authorized +provider migration; deterministic fixtures exercise this without contacting a provider. + +## Alternatives + +Rejected: retaining slug as identity, because rename and URL changes would keep detaching +history. Rejected: deriving IDs from content, path, or title, because all can change and hashes +can reveal relationships. Rejected: treating malformed input as empty, because it destroys +the distinction between no data and damaged data. Rejected: in-place schema rewriting, +because a crash could destroy the only recoverable copy. Rejected: inferring missing revision +content from metadata, because that fabricates history. + +## Trust, privacy, and failure boundaries + +Manifest bytes, paths, metadata, timestamps, hashes, and legacy provider keys are untrusted. +Parsing is size-bounded and schema-exact; resolved file references remain inside the artifact +root. Migration writes neither provider state nor audience-visible data. Repair reports omit +page bodies, credentials, and raw provider values. A failed validation, ambiguous mapping, +unknown future version, backup error, or hash mismatch leaves the selected store unchanged. + +## Migration, rollout, and rollback + +The migrator has inspect, prepare, verify, select, and rollback phases. Prepare writes a +unique staged generation plus an exact backup and fsyncs the files/directories supported by +the platform adapter. Verify reopens and hashes the staged store. Selection is delegated to +the approved publication transaction primitive so interruption resolves to the complete old +or new store. Re-running inspect/prepare is idempotent; completed migrations report their +existing result. Rollback validates the backup before selection. Schema 2 remains opt-in on +write platforms without the roadmap's lock/migration/fault evidence. + +## Formal-method decision + +- Decision: property model plus exhaustive state-machine testing over bounded legacy stores + and interruption points. +- Property and rationale: migration preserves every recoverable byte exactly once, never + invents a revision, keeps artifact identity stable after selection, is idempotent, rejects + future schemas without mutation, and selects either the complete old or complete new store. +- Model/evidence path: add a dependency-free model under `test/model/` and compare generated + operation traces with filesystem migration tests; retain exact supported-platform fault + reports under `docs/evidence/lifecycle/`. diff --git a/specs/changes/artifact-identity-schema-migration/evidence.md b/specs/changes/artifact-identity-schema-migration/evidence.md new file mode 100644 index 0000000..321d8e0 --- /dev/null +++ b/specs/changes/artifact-identity-schema-migration/evidence.md @@ -0,0 +1,5 @@ +# Evidence: Introduce durable artifact identity and schema migration + +Evidence is pending approval and implementation. Archive validation will add one section for +each affected requirement with exact test/model/manual links, supported-platform results, and +visible failures or exclusions. No provider or platform result will be inferred. diff --git a/specs/changes/artifact-identity-schema-migration/proposal.md b/specs/changes/artifact-identity-schema-migration/proposal.md new file mode 100644 index 0000000..1b3e641 --- /dev/null +++ b/specs/changes/artifact-identity-schema-migration/proposal.md @@ -0,0 +1,52 @@ +# Proposal: Introduce durable artifact identity and schema migration + +## Outcome + +Give every local artifact a stable opaque identity and unconditional immutable revision +history, with a versioned metadata schema that upgrades all released local shapes without +losing or inventing content. A migration either produces a verified new store plus backup, or +leaves the old bytes selected and emits an actionable repair report. + +## Context + +The current manifest is unversioned, keys artifacts by slug, stores only revision numbers, and +silently replaces missing, malformed, or future-shaped manifests with an empty manifest. +History is optional, title-derived slug changes create a new identity, legacy files can be +omitted from metadata, and no backup/repair/rollback protocol exists. Phase 1 must freeze an +identity and metadata contract before rendering, packaging, collaboration, and export build on +it. The Phase 0 support matrix has no certified write-platform cells, so the new schema cannot +be default-enabled until the required filesystem evidence exists. + +## Scope + +- In scope: schema-versioned `ArtifactRecord` and `RevisionRecord` data; random opaque IDs; + stable unique slugs as mutable references; immutable monotonically numbered revisions; + complete metadata and provenance; validation; migration dry-run, backup, repair, resume, + verification, rollback, and legacy fixtures for every released manifest/state shape, + including historical Cloudflare shared-KV keys. +- Out of scope: the publication locking/commit algorithm (owned by + `artifact-publication-transaction`); state CAS and quotas (owned by + `artifact-state-cas-limits`); user-facing lifecycle commands and plugin arguments (owned by + `artifact-lifecycle-surfaces`); real provider mutation; and promotion of an untested OS or + filesystem to supported. + +## Risks and rollback + +- Risk: an upgrade could detach a stable page from its identity, invent history for missing + files, overwrite an unknown future schema, cross-wire old Cloudflare state, or leave a + partially selected schema after interruption. Platform-specific rename and durability + behavior could make an otherwise passing migration unsafe. +- Rollback: preserve exact pre-migration bytes in a transaction-scoped backup, keep selection + on the old schema until copy/validate completes, and provide an idempotent rollback that + restores the verified backup and records the result. Unknown future schemas and ambiguous + legacy data fail before mutation. New-schema default enablement remains off for any + unverified write platform. + +## Validation plan + +Validation reviews representative create, rename, revise, repair, upgrade, and rollback +journeys against `LIFE-01`, `LIFE-02`, and `LIFE-07`. Verification uses table-driven fixtures +for every released shape, round-trip and idempotence properties, missing/corrupt/future-schema +cases, backup byte comparison, interruption at every migration boundary, and an explicit +legacy Cloudflare key mapping fixture. Exact supported-filesystem runs remain required before +default enablement and Phase 1 completion. diff --git a/specs/changes/artifact-identity-schema-migration/tasks.md b/specs/changes/artifact-identity-schema-migration/tasks.md new file mode 100644 index 0000000..ee66e69 --- /dev/null +++ b/specs/changes/artifact-identity-schema-migration/tasks.md @@ -0,0 +1,12 @@ +# Tasks: Introduce durable artifact identity and schema migration + +- [ ] Confirm proposal validation and human approval. +- [ ] Add the artifact/revision schemas, exact validators, typed errors, and opaque-ID/slug index. +- [ ] Add every released local manifest/state fixture and historical shared-KV key fixture. +- [ ] Implement bounded inspect, dry-run repair report, backup, staged migration, resume, + verification, and rollback without enabling unverified platforms by default. +- [ ] Add deterministic identity, migration, idempotence, backup/restore, and fault/property tests. +- [ ] Retain exact supported-write-platform results and explicit unavailable cells. +- [ ] Record validation and verification evidence, including failures and exclusions. +- [ ] Add or update `specs/current/artifact-lifecycle.spec.md` and reconcile roadmap/traceability. +- [ ] Run repository validation and archive the packet. diff --git a/specs/changes/artifact-lifecycle-surfaces/change.json b/specs/changes/artifact-lifecycle-surfaces/change.json new file mode 100644 index 0000000..148a1c8 --- /dev/null +++ b/specs/changes/artifact-lifecycle-surfaces/change.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "id": "artifact-lifecycle-surfaces", + "title": "Expose complete artifact lifecycle operations", + "lane": "high-risk", + "status": "draft", + "affectedRequirements": [ + "LIFE-03", + "LIFE-05", + "LIFE-06", + "UX-01", + "UX-02", + "UX-04", + "UX-06", + "SEC-02", + "COMPAT-05", + "COMPAT-07", + "QUAL-02" + ], + "currentSpecs": [], + "currentSpecsUpdated": false, + "approval": { + "by": "", + "at": "" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-16", + "archivedAt": null +} diff --git a/specs/changes/artifact-lifecycle-surfaces/delta.md b/specs/changes/artifact-lifecycle-surfaces/delta.md new file mode 100644 index 0000000..6e741f3 --- /dev/null +++ b/specs/changes/artifact-lifecycle-surfaces/delta.md @@ -0,0 +1,266 @@ +# Specification delta: Expose complete artifact lifecycle operations + +## MODIFIED + +### Requirement: LIFE-03 + +Stable paths and registered URLs resolve to the selected head revision. Restoring an earlier +revision creates one new auditable head revision with restored-from provenance and preserves +all prior history. + +#### Scenario: Normal behavior + +- **Given:** artifact revision 3 and retained revision 1 +- **When:** revision 1 is restored against expected head 3 +- **Then:** revision 4 becomes head with revision-1 bytes and explicit restore provenance + +#### Scenario: Failure or refusal + +- **Given:** the expected head changed or requested revision is absent +- **When:** restore is attempted +- **Then:** it refuses without moving the head or changing history + +#### Scenario: Relevant boundary + +- **Given:** an old path or URL is not a registered reference +- **When:** resolution is attempted +- **Then:** it is not guessed from title or similar slug + +### Requirement: LIFE-05 + +Update resolves an exact ID, contained path, registered URL, or deprecated exact slug and +accepts expected revision/hash. A stale refusal carries bounded current identity, head, +metadata, and merge content or an immutable pinned content reference for the same session. + +#### Scenario: Normal behavior + +- **Given:** an exact artifact reference and matching expected revision +- **When:** update commits +- **Then:** one new revision is selected under the same opaque identity + +#### Scenario: Failure or refusal + +- **Given:** the expected revision/hash is stale +- **When:** update is attempted +- **Then:** no bytes change and the bounded result identifies current merge input and retry token + +#### Scenario: Relevant boundary + +- **Given:** a title-derived slug already exists but no artifact/precondition was supplied +- **When:** legacy-style publish runs +- **Then:** it returns a conflict instead of overwriting the existing artifact + +### Requirement: LIFE-06 + +CLI and plugin expose list, read/status, restore, archive/unarchive, export, and import with +consistent identities/results. Archive is previewed, explicitly confirmed, transactional, and +recoverable; irreversible delete is unavailable. + +#### Scenario: Normal behavior + +- **Given:** an active artifact with history and local state +- **When:** archive is previewed and confirmed with its current token +- **Then:** it leaves active listings while all data remains recoverable by opaque ID + +#### Scenario: Failure or refusal + +- **Given:** confirmation is missing, stale, or scoped to another artifact/head +- **When:** archive is requested +- **Then:** it refuses and reports that the artifact remains active + +#### Scenario: Relevant boundary + +- **Given:** an archived artifact's slug is now used by another active artifact +- **When:** unarchive is requested +- **Then:** it requires an explicit non-conflicting slug and preserves both identities + +### Requirement: UX-01 + +Documented CLI/plugin create, revise, read, restore, export, archive, and unarchive paths show +artifact ID, slug, head revision/hash, visibility, and target capability at each decision. + +#### Scenario: Normal behavior + +- **Given:** a user requests artifact status +- **When:** CLI or plugin returns it +- **Then:** the same identity, head, visibility, capability, and deployment references appear + +#### Scenario: Failure or refusal + +- **Given:** a reference resolves no artifact +- **When:** a lifecycle operation runs +- **Then:** output explains accepted reference forms and makes no mutation + +#### Scenario: Relevant boundary + +- **Given:** a local artifact also has public-static deployment references +- **When:** status is shown +- **Then:** local and public-static capabilities are distinguished rather than labeled private/live + +### Requirement: UX-02 + +Empty, stale, validation, quota, denied, archived, incompatible, and partial-recovery lifecycle +states say what happened, what remained unchanged, and the next safe action in bounded output. + +#### Scenario: Normal behavior + +- **Given:** a stale update +- **When:** the refusal is displayed +- **Then:** it identifies unchanged current head and gives merge/retry information + +#### Scenario: Failure or refusal + +- **Given:** a corrupt bundle or store cannot be safely read +- **When:** an operation is requested +- **Then:** it does not present empty success and points to bounded repair/status output + +#### Scenario: Relevant boundary + +- **Given:** recovery completed with exclusions +- **When:** status is displayed +- **Then:** exclusions remain visible beside the usable selected state + +### Requirement: UX-04 + +Archive previews exact artifact/head, state/revision scope, bytes, deployment references, and +recovery behavior; execution requires a one-use confirmation bound to that scope. Irreversible +local deletion remains unavailable. + +#### Scenario: Normal behavior + +- **Given:** a current archive preview +- **When:** its exact token is confirmed +- **Then:** only the named artifact/head and associated local data are archived recoverably + +#### Scenario: Failure or refusal + +- **Given:** scope changed after preview +- **When:** the old token is submitted +- **Then:** archive refuses and requires a new preview + +#### Scenario: Relevant boundary + +- **Given:** deployment references point to external retained copies +- **When:** local archive is previewed +- **Then:** output states those external copies are not deleted + +### Requirement: UX-06 + +Export produces a documented schema-versioned checksummed directory bundle with portable page, +metadata, all revisions, sources when available, comments, decisions, and supported documents. +Import validates the entire bundle before atomic mutation and reports unsupported content. + +#### Scenario: Normal behavior + +- **Given:** a representable artifact bundle +- **When:** export then import completes +- **Then:** identity, revisions, selected head, metadata, state, and documents round-trip + +#### Scenario: Failure or refusal + +- **Given:** a corrupt, oversized, escaping, or future-schema bundle +- **When:** import preflight runs +- **Then:** it fails without creating or modifying an artifact + +#### Scenario: Relevant boundary + +- **Given:** an export contains authoring source or local mutable state +- **When:** public deployment staging runs +- **Then:** those internal bundle/store areas remain excluded unless separately authorized + +### Requirement: SEC-02 + +Lifecycle references, paths, URLs, bundle entries, metadata, confirmation tokens, operation +IDs, and results are validated and bounded before authority or filesystem access; resolved +paths remain within the intended root and ambiguous references fail closed. + +#### Scenario: Normal behavior + +- **Given:** a valid contained path or registered URL +- **When:** it resolves +- **Then:** exactly one artifact ID is returned within configured result limits + +#### Scenario: Failure or refusal + +- **Given:** encoded traversal, symlink escape, unsupported scheme, duplicate match, or foreign URL +- **When:** resolution/import is attempted +- **Then:** it is rejected before reading or writing outside the artifact root + +#### Scenario: Relevant boundary + +- **Given:** a stale payload refers to a large immutable source +- **When:** it is returned +- **Then:** inline output remains bounded and the pinned path cannot escape or change revisions + +### Requirement: COMPAT-05 + +Lifecycle CLI/tool arguments and versioned result/bundle schemas follow SemVer. Existing +`version`, `expectedHash`, `latest`, `state`, restore spelling, and exact bare-slug behavior are +retained as documented compatibility aliases for at least one supported minor; removals +require notice and a migration path. + +#### Scenario: Normal behavior + +- **Given:** an existing supported caller uses a retained legacy spelling +- **When:** the new minor runs it +- **Then:** equivalent safe behavior occurs with bounded deprecation guidance + +#### Scenario: Failure or refusal + +- **Given:** a legacy update omits the precondition for an existing artifact +- **When:** compatibility handling runs +- **Then:** it refuses overwrite and explains the new safe call instead of silently weakening CAS + +#### Scenario: Relevant boundary + +- **Given:** `version:false` is supplied +- **When:** publication succeeds +- **Then:** immutable history is still created and the argument is reported deprecated + +### Requirement: COMPAT-07 + +Export/import preserves every representable artifact identity, revision, timestamp/time zone, +authorship/provenance field, deployment policy/reference, comment, decision, and supported +document with checksummed validation. + +#### Scenario: Normal behavior + +- **Given:** an artifact containing every supported field +- **When:** it round-trips through a bundle +- **Then:** semantic equality and immutable revision hashes are preserved + +#### Scenario: Failure or refusal + +- **Given:** a bundle field or revision cannot be represented safely +- **When:** import validates it +- **Then:** the whole import is refused with an exact unsupported-content report + +#### Scenario: Relevant boundary + +- **Given:** target storage already contains the same ID with divergent history +- **When:** import preflight runs +- **Then:** it requires an explicit supported collision policy and never cross-wires histories + +### Requirement: QUAL-02 + +Reference resolution, lifecycle operations, stale payloads, permissions, archive recovery, and +export/import have deterministic unit and CLI/plugin end-to-end tests, including compatibility +and hostile boundaries. + +#### Scenario: Normal behavior + +- **Given:** the lifecycle journey and bundle fixture corpus +- **When:** CLI and plugin suites run +- **Then:** every operation produces the same domain state and bounded result + +#### Scenario: Failure or refusal + +- **Given:** a public lifecycle path or changed argument lacks a test +- **When:** packet verification runs +- **Then:** implementation and Phase 1 acceptance fail + +#### Scenario: Relevant boundary + +- **Given:** OpenCode host/platform evidence is unavailable +- **When:** worktree tests pass +- **Then:** no packed-host or supported-platform claim is inferred diff --git a/specs/changes/artifact-lifecycle-surfaces/design.md b/specs/changes/artifact-lifecycle-surfaces/design.md new file mode 100644 index 0000000..779305b --- /dev/null +++ b/specs/changes/artifact-lifecycle-surfaces/design.md @@ -0,0 +1,89 @@ +# Design: Expose complete artifact lifecycle operations + +Required for high-risk changes. + +## Context and constraints + +Lifecycle semantics must be shared without expanding the deployment `Publisher` interface, +which currently owns only `publish`. References arrive from users/agents and are untrusted; +paths and URLs can be ambiguous or hostile. OpenCode tool arguments are public API and require +explicit approval. The package is pre-1.0 but still follows SemVer and notice rules. Exported +portable pages must survive package removal, while local state and authoring sources must not +silently enter public-static deployment trees. + +## Chosen design + +Add a local `ArtifactLifecycleStore` over the approved identity, transaction, and mutable-state +primitives. `ArtifactRef` accepts an opaque ID, a contained stable/revision HTML path, or an +exact URL recorded in deployment references. A bare slug remains an exact deprecated lookup +through the unique slug index for current CLI compatibility; fuzzy title/slug guesses and +unregistered URLs are refused. Results use a bounded versioned domain shape shared by CLI and +plugin formatters. + +Create omits `artifact`; update supplies `artifact` plus `expectedRevision` or `expectedHash`. +If a legacy title-derived create collides with an existing slug without a precondition, it +returns a conflict rather than overwriting. `version` remains accepted but history is always +created. Each revision retains the portable HTML and, when publication originates from the +renderer, its input format and exact authoring source in a non-public history area. A stale +result includes ID/slug/head/hash/metadata and current source inline up to 256 KiB; larger +sources return a transaction-pinned immutable source path plus bounded beginning/end preview, +so the same session can read and merge without rediscovering identity. + +Restore resolves a historical revision and commits its bytes/source as a new head revision +whose provenance names the restored-from revision; no pointer rewinding or history deletion +occurs. Archive preflight returns ID, slug, revision/state counts, bytes, deployment references, +and recovery behavior plus a one-use transaction-bound confirmation token. Confirmed archive +moves the complete logical artifact into the internal archive namespace and removes active +references/gallery entries atomically; unarchive resolves slug conflicts explicitly. + +Export is a schema-versioned directory bundle containing a checksummed manifest, portable +pages, all revisions, metadata/provenance, authoring sources when present, comments, decisions, +and supported documents. It is staged and verified before selection. Import validates schema, +paths, hashes, sizes, identity collisions, and representability without mutation, then commits +atomically with an explicit collision policy; unknown future schemas fail. Public deployment +adapters continue excluding source, mutable-state, archive, transaction, and backup areas. + +CLI adds `list`, `status`, `read`, `archive`, `unarchive`, `export`, and `import`, while keeping +`latest`, `state`, and the old restore spelling as aliases. Plugin adds approved `artifact` and +`expectedRevision` publish arguments, CAS arguments to mutable tools, and one +`artifact_lifecycle` operation tool. Archive requires an `artifact_archive` `ctx.ask` scoped to +the exact opaque ID and confirmation token. The base `Publisher` interface does not change. + +## Alternatives + +Rejected: fuzzy title/slug matching, because it can mutate the wrong artifact. Rejected: +rewinding the head pointer for restore, because it erases the audit meaning of revisions. +Rejected: immediate delete, because recoverable archive is safer and the target does not yet +require irreversible local deletion. Rejected: ZIP/tar or a new archive dependency, because a +checksummed directory bundle is inspectable and portable without new code authority. Rejected: +one overloaded `artifact_publish` operation enum, because read/lifecycle authority and +publication input would become harder to review. + +## Trust, privacy, and failure boundaries + +Artifact refs, paths, URLs, bundles, metadata, source, state, confirmation tokens, and operation +IDs are untrusted. Resolution requires exact normalized containment and rejects symlink escape, +encoded traversal, unsupported schemes, duplicate matches, and foreign deployment URLs. +Results cap lists, source previews, state/doc previews, errors, and diagnostics; secrets are +scanned before export across an audience boundary. Archive authority is distinct from publish, +tokens are single-use and bound to current head/scope, and failures state that nothing changed. + +## Migration, rollout, and rollback + +Implement only after all four Goal 2 packets are approved. Add read/list/status first, then +stale-protected create/update, restore, archive/unarchive, and export/import on the transaction +store. Keep compatibility aliases with deprecation output through at least one supported +minor; changed update semantics ship only in a new minor. Rollback disables new mutations, +recovers any transaction, selects the verified prior schema/store, and leaves portable HTML +readable. Export/import and archive remain unavailable if full-state verification fails. + +## Formal-method decision + +- Decision: reference-resolution and lifecycle state-machine property model. +- Property and rationale: an accepted reference resolves exactly one artifact; stale updates + never write; restore adds exactly one revision; archive is reversible and preserves all + associated data; import is all-or-nothing; export/import round-trip every representable + field; and confirmation tokens cannot authorize a different head or artifact. +- Model/evidence path: dependency-free lifecycle traces under `test/model/`, CLI/plugin E2E and + hostile bundle/reference fixtures, plus retained browser/manual workflow evidence under + `docs/evidence/lifecycle/`. diff --git a/specs/changes/artifact-lifecycle-surfaces/evidence.md b/specs/changes/artifact-lifecycle-surfaces/evidence.md new file mode 100644 index 0000000..7f54d4d --- /dev/null +++ b/specs/changes/artifact-lifecycle-surfaces/evidence.md @@ -0,0 +1,5 @@ +# Evidence: Expose complete artifact lifecycle operations + +Evidence is pending approval and implementation. Archive validation will add one section for +each affected requirement with exact test/model/manual links, CLI/plugin journeys, hostile +reference and bundle results, compatibility observations, and visible failures or exclusions. diff --git a/specs/changes/artifact-lifecycle-surfaces/proposal.md b/specs/changes/artifact-lifecycle-surfaces/proposal.md new file mode 100644 index 0000000..4893af8 --- /dev/null +++ b/specs/changes/artifact-lifecycle-surfaces/proposal.md @@ -0,0 +1,55 @@ +# Proposal: Expose complete artifact lifecycle operations + +## Outcome + +Expose one coherent local lifecycle through CLI and OpenCode: exact artifact references, +create/update with stale protection, list/status/read, auditable revision restore, recoverable +archive/unarchive, and validated versioned export/import. Every result identifies artifact, +head, hash, visibility/capability, unchanged state on failure, and the next safe action. + +## Context + +The CLI currently renders by title-derived slug, restores by bare slug/version, reports only +the latest artifact, and reads decision state. The plugin exposes publish plus separate DB, +state, and comment tools but no artifact list/read/status/restore/archive/export/import. Update +identity is inferred from title, history is optional, stale metadata is incomplete, archive is +absent, and no portable lifecycle bundle exists. Goal 2 must freeze these contracts for Goals +3, 4, and 6. + +## Scope + +- In scope: exact ID/contained path/registered URL reference resolution plus deprecated exact + bare-slug compatibility; required expected revision/hash for updates; stable structured + domain results; full CLI and plugin lifecycle operations; auditable restore as a new + revision; archive preview/confirmation/unarchive; schema-versioned directory export/import; + retained authoring source for stale merge; and SemVer/deprecation documentation. +- Public API proposed for approval: add `artifact` and `expectedRevision` to + `artifact_publish`; retain `version` as a deprecated no-op and `expectedHash` as a supported + compatibility precondition; add `expectedRevision`/`operationId` where state/comment/DB + mutations require CAS; add an `artifact_lifecycle` tool with bounded operations and an + `artifact_archive` permission checkpoint. The `Publisher` interface is not changed. +- Out of scope: hosted lifecycle, audience/deployment permissions, structured OpenCode host + result integration and reopen fallback (Goal 4), event collaboration (Goal 6), irreversible + deletion, release, real deployment, and provider-side import/export. + +## Risks and rollback + +- Risk: reference parsing could target the wrong artifact or escape the root; restore could + destroy history; archive could orphan state; export could omit private/local data or leak it + to public staging; import could partially mutate; and public tool-argument changes could + break existing agents or scripts. +- Rollback: keep existing CLI spellings and tool arguments as documented compatibility + aliases for at least one supported minor; use a new pre-1.0 minor for changed update + semantics; implement archive as a reversible transaction; validate an entire import before + mutation; and roll back selected schema/store using the approved lifecycle transaction. + If lifecycle surfaces cannot safely initialize, existing portable HTML stays readable and + all mutations fail closed. + +## Validation plan + +Validation follows documented create, revise, stale merge, rename, restore, archive/unarchive, +export, and import journeys through both CLI and plugin, including preview and failure text. +Verification covers exact/fuzzy/hostile references, stale payload bounds, immutable restore, +archive recovery, export round trips, corrupt/unknown bundles, old argument compatibility, +permission denial, and packed CLI/plugin behavior. Human approval of the listed public tool +arguments and permission name is required before implementation. diff --git a/specs/changes/artifact-lifecycle-surfaces/tasks.md b/specs/changes/artifact-lifecycle-surfaces/tasks.md new file mode 100644 index 0000000..8cf5b4a --- /dev/null +++ b/specs/changes/artifact-lifecycle-surfaces/tasks.md @@ -0,0 +1,12 @@ +# Tasks: Expose complete artifact lifecycle operations + +- [ ] Confirm proposal validation and human approval. +- [ ] Implement exact artifact reference parsing/resolution and bounded domain results. +- [ ] Retain authoring source and implement stale-protected create/update compatibility behavior. +- [ ] Implement list/read/status and auditable restore on the lifecycle store. +- [ ] Implement archive preview/token/confirmation, archive, and unarchive transactions. +- [ ] Implement checksummed schema-versioned directory export/import with full preflight. +- [ ] Add the approved CLI commands, plugin arguments/tool, CAS arguments, and archive permission. +- [ ] Add deterministic CLI/plugin/reference/permission/archive/bundle/compatibility tests and docs. +- [ ] Record validation/verification evidence and update current specs, roadmap, and traceability. +- [ ] Run repository validation and archive the packet. diff --git a/specs/changes/artifact-publication-transaction/change.json b/specs/changes/artifact-publication-transaction/change.json new file mode 100644 index 0000000..f05c8e4 --- /dev/null +++ b/specs/changes/artifact-publication-transaction/change.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "id": "artifact-publication-transaction", + "title": "Make artifact publication crash-safe across processes", + "lane": "high-risk", + "status": "draft", + "affectedRequirements": [ + "LIFE-04", + "SEC-07", + "OPS-04", + "OPS-05", + "QUAL-02", + "QUAL-06" + ], + "currentSpecs": [], + "currentSpecsUpdated": false, + "approval": { + "by": "", + "at": "" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-16", + "archivedAt": null +} diff --git a/specs/changes/artifact-publication-transaction/delta.md b/specs/changes/artifact-publication-transaction/delta.md new file mode 100644 index 0000000..5cbd2aa --- /dev/null +++ b/specs/changes/artifact-publication-transaction/delta.md @@ -0,0 +1,145 @@ +# Specification delta: Make artifact publication crash-safe across processes + +## MODIFIED + +### Requirement: LIFE-04 + +Create, update, restore, manifest mutation, revision retention, and gallery generation execute +under one fenced inter-process transaction. Recovery before managed access yields a complete +old or complete new logical state and never reports a mixed commit as successful. + +#### Scenario: Normal behavior + +- **Given:** two processes update the same expected head concurrently +- **When:** both attempt to publish +- **Then:** exactly one commits and the other receives a stale result with no partial writes + +#### Scenario: Failure or refusal + +- **Given:** interruption at any stage, journal, replacement, or cleanup boundary +- **When:** a new process opens the store +- **Then:** recovery selects and verifies the complete old or complete new transaction + +#### Scenario: Relevant boundary + +- **Given:** two processes publish different artifacts concurrently +- **When:** both complete +- **Then:** the manifest and gallery contain both commits without a lost entry + +### Requirement: SEC-07 + +Filesystem lifecycle writes use bounded serialization with expected-head checks, fencing, and +idempotent recovery. Lock waits, staged bytes, target counts, retries, and diagnostics have +enforced limits; a retry cannot duplicate a committed revision. + +#### Scenario: Normal behavior + +- **Given:** a writer holds the valid fencing token and the expected head matches +- **When:** it commits within limits +- **Then:** one revision is created and the lock is released after verification + +#### Scenario: Failure or refusal + +- **Given:** a stale owner resumes after lock takeover +- **When:** it reaches a commit boundary +- **Then:** fencing validation refuses its commit without altering the selected state + +#### Scenario: Relevant boundary + +- **Given:** a caller retries after losing the success response +- **When:** recovery finds the operation already committed +- **Then:** it returns the existing commit result instead of creating another revision + +### Requirement: OPS-04 + +Lock timeout, cancellation, process crash, corrupt stage, and interrupted cleanup produce a +typed degraded state that names what remains selected and the next safe recovery action. +Last-known safe reads continue only after integrity is verified. + +#### Scenario: Normal behavior + +- **Given:** cleanup was interrupted after a verified commit +- **When:** the store reopens +- **Then:** the committed state remains readable and cleanup resumes idempotently + +#### Scenario: Failure or refusal + +- **Given:** neither old nor staged targets can be fully verified +- **When:** recovery runs +- **Then:** mutations fail closed and diagnostics identify the bounded repair scope + +#### Scenario: Relevant boundary + +- **Given:** a caller cancels while waiting for another writer +- **When:** cancellation is observed +- **Then:** no transaction is started and the selected state remains unchanged + +### Requirement: OPS-05 + +Publication transaction rollout has preflight, staged opt-in, post-commit verification, and a +tested rollback to the prior compatible store. Failed rollout never silently enables the new +commit path. + +#### Scenario: Normal behavior + +- **Given:** transaction preflight and platform fault tests pass +- **When:** the new path is enabled +- **Then:** post-change verification confirms the selected head, manifest, revision, and gallery + +#### Scenario: Failure or refusal + +- **Given:** post-change verification fails +- **When:** rollout recovery executes +- **Then:** it restores the verified last-known-good store and reports the failed candidate + +#### Scenario: Relevant boundary + +- **Given:** a target filesystem has not run the required fault suite +- **When:** enablement is evaluated +- **Then:** the old compatible path remains selected on that platform + +### Requirement: QUAL-02 + +Transaction, locking, fencing, stale checks, recovery, and fault boundaries have deterministic +unit, property, multi-process, and filesystem tests that do not rely on timing luck. + +#### Scenario: Normal behavior + +- **Given:** deterministic worker barriers and fault indices +- **When:** the lifecycle suite runs +- **Then:** every race and write boundary produces the modeled result + +#### Scenario: Failure or refusal + +- **Given:** a commit boundary lacks a deterministic failure test +- **When:** packet verification runs +- **Then:** implementation and archive validation are not accepted + +#### Scenario: Relevant boundary + +- **Given:** the same test repeats under scheduling variation +- **When:** results are compared +- **Then:** correctness depends on explicit barriers/state, not wall-clock ordering + +### Requirement: QUAL-06 + +Adversarial lifecycle tests cover stale/replayed writes, lock exhaustion, path/symlink attacks, +resource exhaustion, split-brain takeover, process crash, journal corruption, and recovery. + +#### Scenario: Normal behavior + +- **Given:** the complete adversarial transaction corpus +- **When:** it runs on a supported write platform +- **Then:** every attack is contained and a complete selected state remains + +#### Scenario: Failure or refusal + +- **Given:** a crafted journal or target path escapes the artifact root +- **When:** recovery validates it +- **Then:** recovery refuses the input without reading or writing outside the root + +#### Scenario: Relevant boundary + +- **Given:** repeated writers exceed wait or recovery limits +- **When:** overload is reached +- **Then:** excess work is rejected without corrupting existing artifacts or leaking content diff --git a/specs/changes/artifact-publication-transaction/design.md b/specs/changes/artifact-publication-transaction/design.md new file mode 100644 index 0000000..34dc3f7 --- /dev/null +++ b/specs/changes/artifact-publication-transaction/design.md @@ -0,0 +1,71 @@ +# Design: Make artifact publication crash-safe across processes + +Required for high-risk changes. + +## Context and constraints + +The transaction spans several portable files, must preserve direct HTML access, and cannot +assume one process. Node provides atomic creation/rename primitives but no portable atomic +multi-file rename. The design therefore needs a recoverable commit protocol, a clear managed +visibility boundary, and filesystem-specific evidence. It must not add a dependency without a +separate approval, weaken the final-byte cap, or publish local lock/journal data. + +## Chosen design + +Add a lifecycle-store transaction primitive used by every managed read and mutation. An +atomic lock directory contains a unique owner token, process diagnostics, heartbeat, and +monotonic fencing generation. Waits are bounded and abortable. A takeover is allowed only +after the owner is provably gone or its lease is expired; the new fencing generation prevents +a resumed stale writer from committing. Ownership is revalidated before each visible step. + +Within the lock, recovery runs first. A mutation reads and validates the selected state, +performs its expected-head check, renders all target bytes, and writes a unique same-filesystem +stage. It fsyncs supported files/directories, records hashes and old/new targets in a durable +journal, then atomically replaces individual targets with verified backups retained. The +journal records prepared, committing, committed, and cleaned states. The stable page is not +reported or returned until manifest, revision, gallery, and stable bytes reopen consistently. +All managed reads and server startup recover an unfinished journal before resolving a head; +public-static staging excludes lock, journal, backup, and temporary paths. + +Recovery rolls forward only when every staged target matches the journal and the transaction +has a valid commit decision; otherwise it restores every verified old target. It is +idempotent and uses the fencing token. A corrupt or ambiguous journal fails closed with a +bounded repair result instead of guessing. The fault injector is an explicit internal +adapter, not timing-dependent test behavior. + +## Alternatives + +Rejected: the current in-memory promise queue, because it cannot serialize processes. +Rejected: a lock file without fencing, because a suspended stale writer can resume after +takeover. Rejected: direct sequential writes without a journal, because no deterministic +recovery decision exists. Rejected: cross-filesystem temporary paths, because rename loses +atomicity. Rejected: adding SQLite or a locking package at proposal time, because the target +can first be met with reviewed platform adapters and no new dependency. + +## Trust, privacy, and failure boundaries + +Paths, journal bytes, lock metadata, expected heads, and staged files are untrusted. Every +resolved target stays beneath the artifact root; symlinks and unexpected file types fail +closed. Journals and diagnostics contain names, hashes, sizes, and operation IDs but no page +bodies or credentials. Lock waits, transaction bytes, target count, recovery attempts, and +diagnostic output are bounded. Cancellation before commit leaves the old state; cancellation +after a commit decision completes recovery before returning. + +## Migration, rollout, and rollback + +Land the transaction primitive and fault/model tests behind the schema-2 opt-in path. Run +upgrade and crash recovery on each proposed write platform before default enablement. Existing +schema-1 publishing remains the selected fallback until its migration commits. Rollback first +recovers any journal, selects the verified old schema/store, and retains the failed transaction +report. A platform that cannot demonstrate the required atomic-create, rename, and durability +properties remains unverified and schema 2 stays disabled there. + +## Formal-method decision + +- Decision: explicit transaction state machine with exhaustive bounded trace exploration. +- Property and rationale: at most one writer commits for a given expected head; different + artifacts do not lose manifest entries; a stale fencing token never commits; every injected + interruption recovers to the full old or full new logical state; and recovery is idempotent. +- Model/evidence path: dependency-free model traces under `test/model/`, multi-process worker + fixtures under `test/fixtures/`, and exact filesystem fault reports under + `docs/evidence/lifecycle/`. diff --git a/specs/changes/artifact-publication-transaction/evidence.md b/specs/changes/artifact-publication-transaction/evidence.md new file mode 100644 index 0000000..b92d803 --- /dev/null +++ b/specs/changes/artifact-publication-transaction/evidence.md @@ -0,0 +1,5 @@ +# Evidence: Make artifact publication crash-safe across processes + +Evidence is pending approval and implementation. Archive validation will add one section for +each affected requirement with exact test/model/manual links, fault indices, process-race +results, supported-filesystem observations, and visible failures or exclusions. diff --git a/specs/changes/artifact-publication-transaction/proposal.md b/specs/changes/artifact-publication-transaction/proposal.md new file mode 100644 index 0000000..d4f4450 --- /dev/null +++ b/specs/changes/artifact-publication-transaction/proposal.md @@ -0,0 +1,47 @@ +# Proposal: Make artifact publication crash-safe across processes + +## Outcome + +Make create, update, restore, manifest mutation, revision retention, and gallery generation one +recoverable filesystem transaction across processes. Concurrent writers either serialize or +receive a typed stale refusal; interruption recovers a complete old or new logical state. + +## Context + +`FilePublisher` currently serializes only promises in one JavaScript process and writes the +stable page, optional version file, manifest, and gallery directly in sequence. Two processes +can both pass the stale check and lose updates, while a crash can expose mixed files. Restore +also performs multiple in-place writes. Phase 1 requires exact race and fault-injection proof +before later goals depend on the lifecycle store. + +## Scope + +- In scope: a dependency-free inter-process lock with fencing; bounded wait/cancellation; + same-filesystem staging; durable transaction journal; atomic replacement and backups; + startup/read recovery; typed stale, lock, commit, and recovery results; injectable fault + points; and multi-process/model tests for same and different artifacts. +- Out of scope: identity/schema content (owned by `artifact-identity-schema-migration`), + mutable comments/state/database transactions (owned by `artifact-state-cas-limits`), public + lifecycle arguments (owned by `artifact-lifecycle-surfaces`), network filesystems without + verified semantics, and any real deployment or provider mutation. + +## Risks and rollback + +- Risk: stale-lock takeover could create split-brain writers; a journal could be reordered or + corrupted; an interrupted replacement could lose both copies; different-artifact writers + could overwrite one another's manifest entry; or an OS/filesystem could violate assumed + rename/durability semantics. +- Rollback: keep the existing store selected until an approved migration and platform gate + pass. Every replacement retains a transaction-scoped old copy until the committed state is + reopened and verified. Recovery uses a durable journal to roll forward a verified prepared + generation or roll back from verified old copies; ambiguous/corrupt recovery refuses new + writes and reports exact repair scope. + +## Validation plan + +Validation reviews stale-edit, concurrent publish, interrupted publish, and recovery output +for clarity and safe next actions. Verification spawns independent Node processes against one +directory, proves exactly one winner for the same expected head and no lost manifest entries +for different artifacts, injects failure before and after every filesystem boundary, and +compares implementation traces with the bounded transaction model. Supported write-platform +evidence remains mandatory for Phase 1. diff --git a/specs/changes/artifact-publication-transaction/tasks.md b/specs/changes/artifact-publication-transaction/tasks.md new file mode 100644 index 0000000..0702816 --- /dev/null +++ b/specs/changes/artifact-publication-transaction/tasks.md @@ -0,0 +1,13 @@ +# Tasks: Make artifact publication crash-safe across processes + +- [ ] Confirm proposal validation and human approval. +- [ ] Implement the bounded fenced inter-process lock and typed lock outcomes. +- [ ] Implement same-filesystem staging, durable journal, verified backups, commit, recovery, + cleanup, and public-staging exclusions. +- [ ] Route create, update, restore, and managed reads through recovery and the transaction. +- [ ] Add deterministic fault injection and the bounded transaction state model. +- [ ] Add independent-process same-head/different-artifact races, stale takeover, retry, path, + corruption, cancellation, and resource-limit tests. +- [ ] Retain exact supported-filesystem results and explicit unavailable cells. +- [ ] Record validation/verification evidence and update `specs/current/artifact-lifecycle.spec.md`. +- [ ] Run repository validation and archive the packet. diff --git a/specs/changes/artifact-state-cas-limits/change.json b/specs/changes/artifact-state-cas-limits/change.json new file mode 100644 index 0000000..358987d --- /dev/null +++ b/specs/changes/artifact-state-cas-limits/change.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "id": "artifact-state-cas-limits", + "title": "Make mutable artifact state atomic and bounded", + "lane": "high-risk", + "status": "draft", + "affectedRequirements": [ + "LOCAL-04", + "SEC-07", + "OPS-04", + "PERF-05", + "COMPAT-03", + "QUAL-02", + "QUAL-06" + ], + "currentSpecs": [], + "currentSpecsUpdated": false, + "approval": { + "by": "", + "at": "" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-16", + "archivedAt": null +} diff --git a/specs/changes/artifact-state-cas-limits/delta.md b/specs/changes/artifact-state-cas-limits/delta.md new file mode 100644 index 0000000..68a9d19 --- /dev/null +++ b/specs/changes/artifact-state-cas-limits/delta.md @@ -0,0 +1,171 @@ +# Specification delta: Make mutable artifact state atomic and bounded + +## MODIFIED + +### Requirement: LOCAL-04 + +Local decisions, comments, and mini-database stores use versioned envelopes and atomic +expected-revision mutations across clients and processes. Document, collection, thread, body, +and request-rate limits are enforced consistently through HTTP and plugin surfaces. + +#### Scenario: Normal behavior + +- **Given:** two clients update different documents from the same collection revision +- **When:** their operations serialize +- **Then:** both documents remain and each response reports the committed store revision + +#### Scenario: Failure or refusal + +- **Given:** two clients replace the same state from one expected revision +- **When:** both submit +- **Then:** one commits and one receives a bounded current value/revision without mutation + +#### Scenario: Relevant boundary + +- **Given:** a public-static or legacy Cloudflare KV surface +- **When:** local concurrency capability is evaluated +- **Then:** it cannot inherit the local CAS claim or accept an unsupported strong-state label + +### Requirement: SEC-07 + +Every mutable local write is serialized and CAS-checked with bounded body, shape, count, byte, +rate, wait, retry, and response limits. Replayed operation IDs return the prior result and do +not duplicate effects. + +#### Scenario: Normal behavior + +- **Given:** a valid expected revision and operation ID within limits +- **When:** a mutation commits +- **Then:** exactly one new store revision is selected + +#### Scenario: Failure or refusal + +- **Given:** stale, oversized, malformed, rate-exceeded, or future-schema input +- **When:** mutation is attempted +- **Then:** it fails before commit and reports what remained unchanged + +#### Scenario: Relevant boundary + +- **Given:** a success response is lost and the operation is retried +- **When:** the same operation ID reaches the store +- **Then:** the original bounded result is returned without another revision + +### Requirement: OPS-04 + +Stale, quota, overload, corrupt-store, timeout, cancellation, and migration states return typed +degraded results with the selected safe revision and next action. Reads continue only from a +validated last-known-safe envelope. + +#### Scenario: Normal behavior + +- **Given:** a stale client receives the current bounded envelope +- **When:** it merges and retries against that revision +- **Then:** the new mutation can commit without rediscovering artifact identity + +#### Scenario: Failure or refusal + +- **Given:** current state cannot be validated +- **When:** a read or write is requested +- **Then:** writes fail closed and reads do not present corrupt data as empty state + +#### Scenario: Relevant boundary + +- **Given:** the mutation rate reaches its warning threshold but not its hard limit +- **When:** another mutation succeeds +- **Then:** the response includes an actionable warning without weakening consistency + +### Requirement: PERF-05 + +Mutable state publishes and enforces defaults for encoded bytes, answer/thread/document counts, +field sizes, collection size, mutation rate, and bounded operator override ranges with warning +thresholds before hard rejection. + +#### Scenario: Normal behavior + +- **Given:** a store remains below every configured hard limit +- **When:** its mutation commits +- **Then:** measured usage and remaining capacity are available in the bounded result + +#### Scenario: Failure or refusal + +- **Given:** final encoded state would exceed any hard limit +- **When:** mutation is evaluated +- **Then:** it is rejected without changing the existing store + +#### Scenario: Relevant boundary + +- **Given:** an operator requests a value above the absolute override ceiling +- **When:** configuration loads +- **Then:** startup/preflight refuses the invalid configuration + +### Requirement: COMPAT-03 + +Decision, comment, and document stores carry integer schema versions and migrate under backup +from every released local shape. Unknown future versions fail without mutation; legacy +Cloudflare keys are mapped only through explicit provider-migration records. + +#### Scenario: Normal behavior + +- **Given:** a released local JSON state shape +- **When:** migration completes +- **Then:** its payload is preserved under artifact identity with revision and hash metadata + +#### Scenario: Failure or refusal + +- **Given:** an unknown future state schema +- **When:** the runtime opens it +- **Then:** reads/writes refuse rather than treating it as an empty store + +#### Scenario: Relevant boundary + +- **Given:** a historical shared-KV key cannot be assigned unambiguously +- **When:** offline mapping runs +- **Then:** it emits a repair item and performs no provider mutation + +### Requirement: QUAL-02 + +CAS, serialization, migration, validation, and limit behavior is covered by deterministic +unit, property, multi-process, HTTP, plugin, and browser tests independent of network and +developer-specific state. + +#### Scenario: Normal behavior + +- **Given:** the local state fixture corpus and deterministic barriers +- **When:** the suite runs +- **Then:** every normal mutation and limit boundary matches the model + +#### Scenario: Failure or refusal + +- **Given:** a public mutation path bypasses the CAS store or lacks a test +- **When:** verification runs +- **Then:** the packet and Phase 1 gate fail + +#### Scenario: Relevant boundary + +- **Given:** provider access is unavailable +- **When:** local and fake migration tests pass +- **Then:** no real provider or hosted consistency result is claimed + +### Requirement: QUAL-06 + +Adversarial state tests cover stale/replayed writes, malformed schemas/encodings, path names, +deep/large payloads, thread/document exhaustion, mutation-rate overload, process crash, and +cross-artifact association attempts. + +#### Scenario: Normal behavior + +- **Given:** the complete adversarial local-state corpus +- **When:** it executes +- **Then:** isolation, limits, atomicity, and recoverability remain intact + +#### Scenario: Failure or refusal + +- **Given:** a crafted artifact/store name or legacy record targets another artifact +- **When:** it is validated +- **Then:** access is refused before filesystem mutation and no foreign payload is returned + +#### Scenario: Relevant boundary + +- **Given:** controlled overload beyond rate and capacity limits +- **When:** excess writes arrive +- **Then:** they are rejected while existing state remains readable and bounded diff --git a/specs/changes/artifact-state-cas-limits/design.md b/specs/changes/artifact-state-cas-limits/design.md new file mode 100644 index 0000000..25534b3 --- /dev/null +++ b/specs/changes/artifact-state-cas-limits/design.md @@ -0,0 +1,75 @@ +# Design: Make mutable artifact state atomic and bounded + +Required for high-risk changes. + +## Context and constraints + +Three local mutable-store families are exposed both through loopback HTTP and plugin tools. +Their current JSON shapes lack schema versions and revisions. Correctness must hold across +server/plugin processes, not merely request handlers in one process. Limits must apply after +JSON encoding and before writes, conflict responses must be useful but bounded, and state must +follow opaque artifact identity after migration. The portable on-disk page remains network +independent and public-static targets cannot inherit local mutable-state claims. + +## Chosen design + +Store schema 2 uses an envelope containing integer `schemaVersion`, opaque artifact ID, store +kind/key, monotonic revision, SHA-256 content hash, normalized payload, and updated timestamp. +Every mutation supplies an expected revision/hash or create-only precondition. The store is +re-read inside the fenced lifecycle transaction; mismatch returns the current revision/hash +and a bounded current payload sufficient to merge. Mutations to distinct document IDs are +applied to the latest collection inside the lock, so one process cannot replace another's +unrelated document. A repeated operation ID returns its prior result. + +Default limits are: decision document 64 KiB, 256 answers, 256-byte keys, and 4 KiB values; +comment store 256 KiB, 200 threads, 128-byte IDs, 8 KiB quotes, and 16 KiB comment text; +database document 256 KiB, collection 1,000 documents and 16 MiB encoded; and 120 mutations per +artifact/store per rolling minute with a warning at 80 percent. Operator configuration may +lower limits or raise them only up to fixed ceilings of four times each default (1,000 +mutations/minute is the absolute rate ceiling). Reads, writes, response previews, lock wait, +and retry count remain independently bounded. Limit accounting uses final UTF-8 JSON bytes. + +HTTP uses ETag/`If-Match` and `If-None-Match`; structured bodies also carry an operation ID. +The served bridge tracks revisions and shows a reload/merge state on conflict. Plugin +operations expose equivalent expected revision and bounded structured results after the +lifecycle-surfaces packet approves their public shape. Legacy local JSON migrates under +backup; historical Cloudflare records can be exported/mapped but remain unverified and are +never labeled strongly consistent. + +## Alternatives + +Rejected: retaining last-write-wins replacement, because it loses concurrent work. Rejected: +process-local mutexes, because plugin and server processes are independent. Rejected: merging +arbitrary JSON automatically, because conflicts are domain-dependent and can silently corrupt +intent. Rejected: unbounded live payloads on conflict, because they create privacy and +resource risks. Rejected: claiming Cloudflare KV satisfies CAS, because eventual consistency +does not meet the contract. + +## Trust, privacy, and failure boundaries + +Request bodies, document values, IDs, expected tokens, envelopes, and legacy files are +untrusted. Names and artifact identity are validated before filesystem access; payload shape, +depth, scalar lengths, encoded bytes, counts, and rate are checked before locking where safe +and again before commit. Diagnostics redact content by default and cap merge previews. +Malformed, future-schema, stale, quota, timeout, replay-conflict, and corrupt-store cases fail +without mutation. Local rate keys do not contain viewer identity or page content. + +## Migration, rollout, and rollback + +Introduce schema-2 readers, validators, and dry-run migration before enabling writes. Migrate +one store under the lifecycle transaction, verify bytes/hash/identity, then select it; retain +the exact old file until rollback expiry is explicitly recorded. Update HTTP, bridge, and +plugin paths together so no compatibility surface bypasses CAS. On rollback, disable writes, +recover outstanding transactions, verify and select the backup, and report revisions that +cannot be represented. Hosted KV remains a separate unavailable migration target until its +later architecture is approved. + +## Formal-method decision + +- Decision: CAS state-machine/property model with exhaustive bounded interleavings. +- Property and rationale: one expected revision commits at most once; stale mutations never + write; distinct-document concurrent mutations are both retained; revisions increase by one; + migration preserves payload; limits are monotonic and enforced before commit; and retry by + operation ID does not duplicate effects. +- Model/evidence path: add dependency-free traces under `test/model/`, independent-process + worker tests, and local browser/limit evidence under `docs/evidence/lifecycle/`. diff --git a/specs/changes/artifact-state-cas-limits/evidence.md b/specs/changes/artifact-state-cas-limits/evidence.md new file mode 100644 index 0000000..e537da2 --- /dev/null +++ b/specs/changes/artifact-state-cas-limits/evidence.md @@ -0,0 +1,5 @@ +# Evidence: Make mutable artifact state atomic and bounded + +Evidence is pending approval and implementation. Archive validation will add one section for +each affected requirement with exact test/model/manual links, configured limits, race and +overload results, browser states, and visible failures or exclusions. diff --git a/specs/changes/artifact-state-cas-limits/proposal.md b/specs/changes/artifact-state-cas-limits/proposal.md new file mode 100644 index 0000000..78e1caa --- /dev/null +++ b/specs/changes/artifact-state-cas-limits/proposal.md @@ -0,0 +1,47 @@ +# Proposal: Make mutable artifact state atomic and bounded + +## Outcome + +Make local decisions, comments, and mini-database mutations atomic and concurrency safe, with +explicit schema/revision CAS semantics, bounded live conflict payloads, and enforced document, +collection, thread, body, and mutation-rate limits. + +## Context + +The local server and plugin currently perform unprotected read-modify-write operations on JSON +files. Concurrent clients/processes can lose answers, threads, or documents. Most parse errors +become empty stores, writes are in place, collections have no count/total-byte limit, and no +request-rate limit exists. Cloudflare's historical KV handlers use the same shape but cannot +provide strong CAS; their data requires explicit migration treatment rather than being cited +as Phase 1 proof. + +## Scope + +- In scope: versioned decision/comment/document envelopes; per-store monotonic revisions and + content hashes; compare-and-swap and create-only preconditions; atomic mutation via the + approved lifecycle transaction primitive; bounded conflict responses; strict validation; + configurable defaults and hard override ceilings; warning/hard-limit diagnostics; local + HTTP/plugin parity; legacy-shape migration; and process/client race tests. +- Out of scope: hosted strong-state architecture, treating Cloudflare KV as CAS-capable, + datasource execution, live event/reconnect behavior, connector state, provider mutation, + and the final public plugin-tool argument design (owned by `artifact-lifecycle-surfaces`). + +## Risks and rollback + +- Risk: a compatibility fallback could silently bypass CAS; concurrent writes to different + documents could still lose changes; oversized conflict payloads could leak or exhaust + resources; schema migration could detach state from artifact identity; or rate limiting + could make local authoring unusable. +- Rollback: back up each legacy store before schema selection, keep old shapes read-only until + verified migration, and roll back through the lifecycle transaction. Conflict or limit + failures never mutate. The old unsafe write path is not a rollback option after schema 2 is + selected; disabling mutable service routes is the safe fallback. + +## Validation plan + +Validation exercises two-client edit/merge/refusal journeys and checks that errors state the +live revision, unchanged data, limit, and safe retry. Verification uses deterministic +barriers across processes, property traces for CAS and distinct-document updates, migration +fixtures, malformed/oversized inputs, warning and hard-limit boundaries, and rate/overload +tests. Browser evidence is retained for changed decision/comment conflict states; no hosted +or provider result is inferred. diff --git a/specs/changes/artifact-state-cas-limits/tasks.md b/specs/changes/artifact-state-cas-limits/tasks.md new file mode 100644 index 0000000..d1cacd5 --- /dev/null +++ b/specs/changes/artifact-state-cas-limits/tasks.md @@ -0,0 +1,11 @@ +# Tasks: Make mutable artifact state atomic and bounded + +- [ ] Confirm proposal validation and human approval. +- [ ] Implement schema-2 state/comment/collection envelopes, validators, typed errors, and migration. +- [ ] Implement atomic CAS/replay handling on the approved lifecycle transaction primitive. +- [ ] Enforce documented byte/count/field/rate defaults, warnings, and override ceilings. +- [ ] Route loopback HTTP, served bridge, and approved plugin operations through one store API. +- [ ] Add model/property, process/client race, migration, malformed, quota, replay, and overload tests. +- [ ] Retain browser evidence for decision/comment conflict and limit states. +- [ ] Record validation/verification evidence and update current lifecycle/local-service specs. +- [ ] Run repository validation and archive the packet. From f8289bf3b9c1da3886c66bea551549c3a9f5f133 Mon Sep 17 00:00:00 2001 From: bitgorust Date: Sun, 16 Aug 2026 22:32:23 +0200 Subject: [PATCH 2/7] docs: approve durable artifact lifecycle --- .../changes/artifact-identity-schema-migration/change.json | 6 +++--- specs/changes/artifact-lifecycle-surfaces/change.json | 6 +++--- specs/changes/artifact-publication-transaction/change.json | 6 +++--- specs/changes/artifact-state-cas-limits/change.json | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/specs/changes/artifact-identity-schema-migration/change.json b/specs/changes/artifact-identity-schema-migration/change.json index ed505c5..bc5fe5f 100644 --- a/specs/changes/artifact-identity-schema-migration/change.json +++ b/specs/changes/artifact-identity-schema-migration/change.json @@ -3,7 +3,7 @@ "id": "artifact-identity-schema-migration", "title": "Introduce durable artifact identity and schema migration", "lane": "high-risk", - "status": "draft", + "status": "approved", "affectedRequirements": [ "LIFE-01", "LIFE-02", @@ -18,8 +18,8 @@ "currentSpecs": [], "currentSpecsUpdated": false, "approval": { - "by": "", - "at": "" + "by": "aaron.zeng", + "at": "2026-08-16T20:32:04Z" }, "withdrawal": { "by": "", diff --git a/specs/changes/artifact-lifecycle-surfaces/change.json b/specs/changes/artifact-lifecycle-surfaces/change.json index 148a1c8..054844b 100644 --- a/specs/changes/artifact-lifecycle-surfaces/change.json +++ b/specs/changes/artifact-lifecycle-surfaces/change.json @@ -3,7 +3,7 @@ "id": "artifact-lifecycle-surfaces", "title": "Expose complete artifact lifecycle operations", "lane": "high-risk", - "status": "draft", + "status": "approved", "affectedRequirements": [ "LIFE-03", "LIFE-05", @@ -20,8 +20,8 @@ "currentSpecs": [], "currentSpecsUpdated": false, "approval": { - "by": "", - "at": "" + "by": "aaron.zeng", + "at": "2026-08-16T20:32:04Z" }, "withdrawal": { "by": "", diff --git a/specs/changes/artifact-publication-transaction/change.json b/specs/changes/artifact-publication-transaction/change.json index f05c8e4..3118b9a 100644 --- a/specs/changes/artifact-publication-transaction/change.json +++ b/specs/changes/artifact-publication-transaction/change.json @@ -3,7 +3,7 @@ "id": "artifact-publication-transaction", "title": "Make artifact publication crash-safe across processes", "lane": "high-risk", - "status": "draft", + "status": "approved", "affectedRequirements": [ "LIFE-04", "SEC-07", @@ -15,8 +15,8 @@ "currentSpecs": [], "currentSpecsUpdated": false, "approval": { - "by": "", - "at": "" + "by": "aaron.zeng", + "at": "2026-08-16T20:32:04Z" }, "withdrawal": { "by": "", diff --git a/specs/changes/artifact-state-cas-limits/change.json b/specs/changes/artifact-state-cas-limits/change.json index 358987d..ad2e2d8 100644 --- a/specs/changes/artifact-state-cas-limits/change.json +++ b/specs/changes/artifact-state-cas-limits/change.json @@ -3,7 +3,7 @@ "id": "artifact-state-cas-limits", "title": "Make mutable artifact state atomic and bounded", "lane": "high-risk", - "status": "draft", + "status": "approved", "affectedRequirements": [ "LOCAL-04", "SEC-07", @@ -16,8 +16,8 @@ "currentSpecs": [], "currentSpecsUpdated": false, "approval": { - "by": "", - "at": "" + "by": "aaron.zeng", + "at": "2026-08-16T20:32:04Z" }, "withdrawal": { "by": "", From a3e2f06f79a1125b300bddf8b527431e6add0738 Mon Sep 17 00:00:00 2001 From: bitgorust Date: Sun, 16 Aug 2026 22:43:42 +0200 Subject: [PATCH 3/7] feat: make local publication crash-safe --- .../change.json | 2 +- .../artifact-publication-transaction/tasks.md | 10 +- src/file-transaction.ts | 697 ++++++++++++++++++ src/github-pages.ts | 2 +- src/publisher.ts | 93 ++- test/file-transaction.test.ts | 252 +++++++ test/fixtures/file-transaction-worker.ts | 90 +++ test/github-pages.test.ts | 3 + test/model/file-transaction-model.ts | 24 + 9 files changed, 1126 insertions(+), 47 deletions(-) create mode 100644 src/file-transaction.ts create mode 100644 test/file-transaction.test.ts create mode 100644 test/fixtures/file-transaction-worker.ts create mode 100644 test/model/file-transaction-model.ts diff --git a/specs/changes/artifact-publication-transaction/change.json b/specs/changes/artifact-publication-transaction/change.json index 3118b9a..e536fa0 100644 --- a/specs/changes/artifact-publication-transaction/change.json +++ b/specs/changes/artifact-publication-transaction/change.json @@ -3,7 +3,7 @@ "id": "artifact-publication-transaction", "title": "Make artifact publication crash-safe across processes", "lane": "high-risk", - "status": "approved", + "status": "implementing", "affectedRequirements": [ "LIFE-04", "SEC-07", diff --git a/specs/changes/artifact-publication-transaction/tasks.md b/specs/changes/artifact-publication-transaction/tasks.md index 0702816..b5151ed 100644 --- a/specs/changes/artifact-publication-transaction/tasks.md +++ b/specs/changes/artifact-publication-transaction/tasks.md @@ -1,11 +1,11 @@ # Tasks: Make artifact publication crash-safe across processes -- [ ] Confirm proposal validation and human approval. -- [ ] Implement the bounded fenced inter-process lock and typed lock outcomes. -- [ ] Implement same-filesystem staging, durable journal, verified backups, commit, recovery, +- [x] Confirm proposal validation and human approval. +- [x] Implement the bounded fenced inter-process lock and typed lock outcomes. +- [x] Implement same-filesystem staging, durable journal, verified backups, commit, recovery, cleanup, and public-staging exclusions. -- [ ] Route create, update, restore, and managed reads through recovery and the transaction. -- [ ] Add deterministic fault injection and the bounded transaction state model. +- [x] Route create, update, restore, and managed reads through recovery and the transaction. +- [x] Add deterministic fault injection and the bounded transaction state model. - [ ] Add independent-process same-head/different-artifact races, stale takeover, retry, path, corruption, cancellation, and resource-limit tests. - [ ] Retain exact supported-filesystem results and explicit unavailable cells. diff --git a/src/file-transaction.ts b/src/file-transaction.ts new file mode 100644 index 0000000..f6928d4 --- /dev/null +++ b/src/file-transaction.ts @@ -0,0 +1,697 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + lstat, + mkdir, + open, + readFile, + readdir, + rename, + rm, + stat, +} from "node:fs/promises"; +import { dirname, join, resolve, sep } from "node:path"; + +export const TRANSACTION_DIRECTORY = ".transactions"; + +const LOCK_DIRECTORY = "lock"; +const OWNER_FILE = "owner.json"; +const FENCE_FILE = "fence"; +const JOURNAL_FILE = "journal.json"; +const JOURNAL_SCHEMA_VERSION = 1; +const DEFAULT_LOCK_TIMEOUT_MS = 5_000; +const DEFAULT_POLL_INTERVAL_MS = 20; +const OWNER_INITIALIZATION_GRACE_MS = 1_000; +const MAX_TRANSACTION_TARGETS = 64; +const MAX_TRANSACTION_BYTES = 64 * 1024 * 1024; +const TRANSACTION_ID_RE = /^\d+-\d+-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const SHA256_RE = /^[0-9a-f]{64}$/; + +export type TransactionFaultPoint = + | "stage-created" + | "target-staged" + | "journal-prepared" + | "commit-decided" + | "target-backed-up" + | "target-replaced" + | "commit-verified" + | "journal-committed"; + +export interface FileTransactionOptions { + signal?: AbortSignal; + lockTimeoutMs?: number; + pollIntervalMs?: number; + fault?: (point: TransactionFaultPoint, target?: string) => void; +} + +export interface FileTransactionContext { + commit(files: ReadonlyMap): Promise; +} + +interface LockOwner { + schemaVersion: 1; + pid: number; + token: string; + fence: number; + createdAt: string; +} + +type JournalState = "prepared" | "committing" | "committed"; + +interface JournalTarget { + path: string; + existed: boolean; + oldHash: string | null; + newHash: string; + bytes: number; +} + +interface TransactionJournal { + schemaVersion: 1; + id: string; + fence: number; + state: JournalState; + targets: JournalTarget[]; +} + +interface HeldLock { + token: string; + fence: number; + release(): Promise; +} + +export class TransactionLockTimeoutError extends Error { + readonly root: string; + + constructor(root: string) { + super(`timed out waiting for the artifact transaction lock in ${root}`); + this.name = "TransactionLockTimeoutError"; + this.root = root; + } +} + +export class TransactionRecoveryError extends Error { + readonly transactionId: string; + + constructor(transactionId: string, message: string) { + super(`transaction ${transactionId} cannot be recovered safely: ${message}`); + this.name = "TransactionRecoveryError"; + this.transactionId = transactionId; + } +} + +export class TransactionCommitError extends Error { + readonly transactionId: string; + readonly selectedState = "old" as const; + readonly cause: unknown; + + constructor(transactionId: string, cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause); + super(`transaction ${transactionId} did not commit; the old state remains selected: ${detail}`); + this.name = "TransactionCommitError"; + this.transactionId = transactionId; + this.cause = cause; + } +} + +function hashBytes(value: Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function errnoCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) return undefined; + const code = (error as { code?: unknown }).code; + return typeof code === "string" ? code : undefined; +} + +async function exists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + if (errnoCode(error) === "ENOENT") return false; + throw error; + } +} + +function parseInteger(value: string): number | undefined { + if (!/^\d+$/.test(value.trim())) return undefined; + const parsed = Number(value.trim()); + return Number.isSafeInteger(parsed) ? parsed : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseLockOwner(value: unknown): LockOwner | undefined { + if (!isRecord(value)) return undefined; + if ( + value["schemaVersion"] !== 1 || + typeof value["pid"] !== "number" || + !Number.isSafeInteger(value["pid"]) || + typeof value["token"] !== "string" || + typeof value["fence"] !== "number" || + !Number.isSafeInteger(value["fence"]) || + typeof value["createdAt"] !== "string" + ) { + return undefined; + } + return { + schemaVersion: 1, + pid: value["pid"], + token: value["token"], + fence: value["fence"], + createdAt: value["createdAt"], + }; +} + +function parseJournal(value: unknown): TransactionJournal | undefined { + if (!isRecord(value)) return undefined; + if ( + value["schemaVersion"] !== JOURNAL_SCHEMA_VERSION || + typeof value["id"] !== "string" || + !TRANSACTION_ID_RE.test(value["id"]) || + typeof value["fence"] !== "number" || + !Number.isSafeInteger(value["fence"]) || + (value["state"] !== "prepared" && + value["state"] !== "committing" && + value["state"] !== "committed") || + !Array.isArray(value["targets"]) + ) { + return undefined; + } + const targets: JournalTarget[] = []; + for (const target of value["targets"]) { + if ( + !isRecord(target) || + typeof target["path"] !== "string" || + typeof target["existed"] !== "boolean" || + (target["oldHash"] !== null && typeof target["oldHash"] !== "string") || + typeof target["newHash"] !== "string" || + !SHA256_RE.test(target["newHash"]) || + typeof target["bytes"] !== "number" || + !Number.isSafeInteger(target["bytes"]) || + target["bytes"] < 0 || + (target["existed"] && + (typeof target["oldHash"] !== "string" || !SHA256_RE.test(target["oldHash"]))) || + (!target["existed"] && target["oldHash"] !== null) + ) { + return undefined; + } + targets.push({ + path: target["path"], + existed: target["existed"], + oldHash: target["oldHash"], + newHash: target["newHash"], + bytes: target["bytes"], + }); + } + if (targets.length === 0 || targets.length > MAX_TRANSACTION_TARGETS) return undefined; + if (new Set(targets.map((target) => target.path)).size !== targets.length) return undefined; + if (targets.reduce((sum, target) => sum + target.bytes, 0) > MAX_TRANSACTION_BYTES) { + return undefined; + } + return { + schemaVersion: 1, + id: value["id"], + fence: value["fence"], + state: value["state"], + targets, + }; +} + +function safeSegments(relativePath: string): string[] { + if ( + relativePath === "" || + relativePath.includes("\\") || + relativePath.startsWith("/") || + relativePath.endsWith("/") + ) { + throw new Error(`unsafe transaction target ${JSON.stringify(relativePath)}`); + } + const segments = relativePath.split("/"); + if ( + segments.some( + (segment) => + segment === "" || segment === "." || segment === ".." || segment === TRANSACTION_DIRECTORY, + ) + ) { + throw new Error(`unsafe transaction target ${JSON.stringify(relativePath)}`); + } + return segments; +} + +async function containedPath(root: string, relativePath: string): Promise { + const segments = safeSegments(relativePath); + const rootPath = resolve(root); + let current = rootPath; + for (let index = 0; index < segments.length - 1; index++) { + current = join(current, segments[index]); + try { + const info = await lstat(current); + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`transaction target parent is not a real directory: ${relativePath}`); + } + } catch (error) { + if (errnoCode(error) !== "ENOENT") throw error; + break; + } + } + const target = join(rootPath, ...segments); + if (target !== rootPath && !target.startsWith(`${rootPath}${sep}`)) { + throw new Error(`transaction target escapes its root: ${relativePath}`); + } + if (await exists(target)) { + const info = await lstat(target); + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error(`transaction target is not a regular file: ${relativePath}`); + } + } + return target; +} + +async function syncFile(path: string): Promise { + const handle = await open(path, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function syncDirectory(path: string): Promise { + const handle = await open(path, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function writeDurable(path: string, value: Uint8Array): Promise { + await mkdir(dirname(path), { recursive: true }); + const handle = await open(path, "wx"); + try { + await handle.writeFile(value); + await handle.sync(); + } finally { + await handle.close(); + } + await syncDirectory(dirname(path)); +} + +async function replaceDurable(path: string, value: string): Promise { + const temporary = `${path}.${randomUUID()}.tmp`; + await writeDurable(temporary, Buffer.from(value, "utf8")); + await rename(temporary, path); + await syncDirectory(dirname(path)); +} + +async function readJson(path: string): Promise { + return JSON.parse(await readFile(path, "utf8")) as unknown; +} + +function processIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return errnoCode(error) === "EPERM"; + } +} + +function abortError(): Error { + const error = new Error("artifact transaction was cancelled"); + error.name = "AbortError"; + return error; +} + +async function wait(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) throw abortError(); + await new Promise((resolveWait, rejectWait) => { + const onAbort = () => { + clearTimeout(timer); + rejectWait(abortError()); + }; + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolveWait(); + }, ms); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +async function stealDeadLock(lockPath: string): Promise { + let owner: LockOwner | undefined; + try { + owner = parseLockOwner(await readJson(join(lockPath, OWNER_FILE))); + if (!owner) throw new Error("artifact transaction lock owner metadata is invalid"); + } catch (error) { + if (errnoCode(error) !== "ENOENT") throw error; + } + if (owner && processIsAlive(owner.pid)) return false; + if (!owner) { + const info = await stat(lockPath); + if (Date.now() - info.mtimeMs < OWNER_INITIALIZATION_GRACE_MS) return false; + } + const stalePath = `${lockPath}.stale-${randomUUID()}`; + try { + await rename(lockPath, stalePath); + } catch (error) { + if (errnoCode(error) === "ENOENT") return true; + throw error; + } + await rm(stalePath, { recursive: true, force: true }); + return true; +} + +async function acquireLock(root: string, options: FileTransactionOptions): Promise { + await mkdir(root, { recursive: true }); + const transactionsPath = join(root, TRANSACTION_DIRECTORY); + await mkdir(transactionsPath, { recursive: true }); + const lockPath = join(transactionsPath, LOCK_DIRECTORY); + const deadline = Date.now() + (options.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS); + const pollInterval = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + + for (;;) { + if (options.signal?.aborted) throw abortError(); + try { + await mkdir(lockPath); + const priorFence = await readFile(join(transactionsPath, FENCE_FILE), "utf8").then( + (value) => parseInteger(value) ?? 0, + (error: unknown) => { + if (errnoCode(error) === "ENOENT") return 0; + throw error; + }, + ); + const fence = priorFence + 1; + await replaceDurable(join(transactionsPath, FENCE_FILE), `${fence}\n`); + const token = randomUUID(); + const owner: LockOwner = { + schemaVersion: 1, + pid: process.pid, + token, + fence, + createdAt: new Date().toISOString(), + }; + await replaceDurable(join(lockPath, OWNER_FILE), `${JSON.stringify(owner)}\n`); + await syncDirectory(lockPath); + let released = false; + return { + token, + fence, + async release() { + if (released) return; + released = true; + let current: LockOwner | undefined; + try { + current = parseLockOwner(await readJson(join(lockPath, OWNER_FILE))); + } catch (error) { + if (errnoCode(error) !== "ENOENT") throw error; + } + if (current?.token === token) { + await rm(lockPath, { recursive: true, force: true }); + await syncDirectory(transactionsPath); + } + }, + }; + } catch (error) { + if (errnoCode(error) !== "EEXIST") throw error; + } + + if (await stealDeadLock(lockPath)) continue; + if (Date.now() >= deadline) throw new TransactionLockTimeoutError(root); + await wait(pollInterval, options.signal); + } +} + +async function assertLock(root: string, lock: HeldLock): Promise { + const transactionsPath = join(root, TRANSACTION_DIRECTORY); + const owner = parseLockOwner(await readJson(join(transactionsPath, LOCK_DIRECTORY, OWNER_FILE))); + const fence = parseInteger(await readFile(join(transactionsPath, FENCE_FILE), "utf8")); + if (!owner || owner.token !== lock.token || owner.fence !== lock.fence || fence !== lock.fence) { + throw new Error("artifact transaction lock ownership changed before commit"); + } +} + +async function writeJournal(transactionPath: string, journal: TransactionJournal): Promise { + await replaceDurable(join(transactionPath, JOURNAL_FILE), `${JSON.stringify(journal, null, 2)}\n`); + await syncDirectory(transactionPath); +} + +async function targetHash(path: string): Promise { + try { + return hashBytes(await readFile(path)); + } catch (error) { + if (errnoCode(error) === "ENOENT") return undefined; + throw error; + } +} + +async function restorePrepared(root: string, transactionPath: string, journal: TransactionJournal): Promise { + for (const target of journal.targets) { + const destination = await containedPath(root, target.path); + const backup = join(transactionPath, "old", ...safeSegments(target.path)); + const current = await targetHash(destination); + if (current === target.newHash) { + if (!target.existed) { + await rm(destination, { force: true }); + } else if (await exists(backup)) { + await mkdir(dirname(destination), { recursive: true }); + await rename(backup, destination); + await syncDirectory(dirname(backup)); + await syncDirectory(dirname(destination)); + } else { + throw new TransactionRecoveryError(journal.id, `old bytes missing for ${target.path}`); + } + } else if (target.existed && current !== target.oldHash) { + if (await exists(backup)) { + await mkdir(dirname(destination), { recursive: true }); + await rename(backup, destination); + await syncDirectory(dirname(backup)); + await syncDirectory(dirname(destination)); + } else { + throw new TransactionRecoveryError(journal.id, `unexpected old bytes for ${target.path}`); + } + } + const restored = await targetHash(destination); + if ((target.existed && restored !== target.oldHash) || (!target.existed && restored !== undefined)) { + throw new TransactionRecoveryError(journal.id, `rollback verification failed for ${target.path}`); + } + } + await rm(transactionPath, { recursive: true, force: true }); + await syncDirectory(join(root, TRANSACTION_DIRECTORY)); +} + +async function rollForward(root: string, transactionPath: string, journal: TransactionJournal): Promise { + for (const target of journal.targets) { + const destination = await containedPath(root, target.path); + const staged = join(transactionPath, "new", ...safeSegments(target.path)); + const backup = join(transactionPath, "old", ...safeSegments(target.path)); + const current = await targetHash(destination); + if (current === target.newHash) continue; + if (!(await exists(staged)) || (await targetHash(staged)) !== target.newHash) { + throw new TransactionRecoveryError(journal.id, `staged bytes missing for ${target.path}`); + } + if (current !== undefined) { + if (!target.existed || current !== target.oldHash) { + throw new TransactionRecoveryError(journal.id, `unexpected selected bytes for ${target.path}`); + } + await mkdir(dirname(backup), { recursive: true }); + if (!(await exists(backup))) await rename(destination, backup); + else await rm(destination, { force: true }); + await syncDirectory(dirname(backup)); + await syncDirectory(dirname(destination)); + } + await mkdir(dirname(destination), { recursive: true }); + await rename(staged, destination); + await syncFile(destination); + await syncDirectory(dirname(staged)); + await syncDirectory(dirname(destination)); + } + for (const target of journal.targets) { + const destination = await containedPath(root, target.path); + if ((await targetHash(destination)) !== target.newHash) { + throw new TransactionRecoveryError(journal.id, `commit verification failed for ${target.path}`); + } + } + const committed: TransactionJournal = { ...journal, state: "committed" }; + await writeJournal(transactionPath, committed); + await rm(transactionPath, { recursive: true, force: true }); + await syncDirectory(join(root, TRANSACTION_DIRECTORY)); +} + +async function recoverHeld(root: string): Promise { + const transactionsPath = join(root, TRANSACTION_DIRECTORY); + const entries = await readdir(transactionsPath, { withFileTypes: true }); + const candidates = entries + .filter( + (entry) => + entry.isDirectory() && entry.name !== LOCK_DIRECTORY && !entry.name.startsWith(`${LOCK_DIRECTORY}.stale-`), + ) + .map((entry) => entry.name) + .sort(); + for (const name of candidates) { + if (!TRANSACTION_ID_RE.test(name)) { + throw new TransactionRecoveryError(name, "unexpected directory in the transaction store"); + } + const transactionPath = join(transactionsPath, name); + let journal: TransactionJournal | undefined; + try { + journal = parseJournal(await readJson(join(transactionPath, JOURNAL_FILE))); + } catch (error) { + if (errnoCode(error) === "ENOENT") { + await rm(transactionPath, { recursive: true, force: true }); + continue; + } + throw new TransactionRecoveryError(name, "journal is unreadable"); + } + if (!journal || journal.id !== name) { + throw new TransactionRecoveryError(name, "journal is malformed or mismatched"); + } + for (const target of journal.targets) safeSegments(target.path); + if (journal.state === "prepared") await restorePrepared(root, transactionPath, journal); + else await rollForward(root, transactionPath, journal); + } +} + +async function commitHeld( + root: string, + lock: HeldLock, + files: ReadonlyMap, + options: FileTransactionOptions, +): Promise { + if (files.size === 0) throw new Error("artifact transaction has no target files"); + if (files.size > MAX_TRANSACTION_TARGETS) { + throw new Error(`artifact transaction has ${files.size} targets; limit is ${MAX_TRANSACTION_TARGETS}`); + } + const normalized = [...files.entries()] + .map(([path, value]) => [path, typeof value === "string" ? Buffer.from(value, "utf8") : Buffer.from(value)] as const) + .sort(([left], [right]) => left.localeCompare(right)); + const totalBytes = normalized.reduce((sum, [, value]) => sum + value.byteLength, 0); + if (totalBytes > MAX_TRANSACTION_BYTES) { + throw new Error(`artifact transaction has ${totalBytes} bytes; limit is ${MAX_TRANSACTION_BYTES}`); + } + const unique = new Set(normalized.map(([path]) => path)); + if (unique.size !== normalized.length) throw new Error("artifact transaction contains duplicate targets"); + for (const [path] of normalized) await containedPath(root, path); + await assertLock(root, lock); + + const id = `${Date.now()}-${process.pid}-${randomUUID()}`; + const transactionPath = join(root, TRANSACTION_DIRECTORY, id); + await mkdir(join(transactionPath, "new"), { recursive: true }); + await mkdir(join(transactionPath, "old"), { recursive: true }); + options.fault?.("stage-created"); + + const targets: JournalTarget[] = []; + for (const [path, value] of normalized) { + const destination = await containedPath(root, path); + const current = await readFile(destination).then( + (bytes) => bytes, + (error: unknown) => { + if (errnoCode(error) === "ENOENT") return undefined; + throw error; + }, + ); + await writeDurable(join(transactionPath, "new", ...safeSegments(path)), value); + targets.push({ + path, + existed: current !== undefined, + oldHash: current === undefined ? null : hashBytes(current), + newHash: hashBytes(value), + bytes: value.byteLength, + }); + options.fault?.("target-staged", path); + } + + let journal: TransactionJournal = { + schemaVersion: JOURNAL_SCHEMA_VERSION, + id, + fence: lock.fence, + state: "prepared", + targets, + }; + try { + await writeJournal(transactionPath, journal); + options.fault?.("journal-prepared"); + await assertLock(root, lock); + journal = { ...journal, state: "committing" }; + await writeJournal(transactionPath, journal); + options.fault?.("commit-decided"); + + for (const target of targets) { + await assertLock(root, lock); + const destination = await containedPath(root, target.path); + const backup = join(transactionPath, "old", ...safeSegments(target.path)); + const staged = join(transactionPath, "new", ...safeSegments(target.path)); + if (target.existed) { + await mkdir(dirname(backup), { recursive: true }); + await rename(destination, backup); + await syncDirectory(dirname(backup)); + await syncDirectory(dirname(destination)); + options.fault?.("target-backed-up", target.path); + } + await mkdir(dirname(destination), { recursive: true }); + await rename(staged, destination); + await syncFile(destination); + await syncDirectory(dirname(staged)); + await syncDirectory(dirname(destination)); + options.fault?.("target-replaced", target.path); + } + + for (const target of targets) { + const destination = await containedPath(root, target.path); + if ((await targetHash(destination)) !== target.newHash) { + throw new TransactionRecoveryError(id, `verification failed for ${target.path}`); + } + } + options.fault?.("commit-verified"); + journal = { ...journal, state: "committed" }; + await writeJournal(transactionPath, journal); + options.fault?.("journal-committed"); + await rm(transactionPath, { recursive: true, force: true }); + await syncDirectory(join(root, TRANSACTION_DIRECTORY)); + } catch (error) { + await recoverHeld(root); + const selectedNew = + targets.length > 0 && + (await Promise.all( + targets.map(async (target) => { + const destination = await containedPath(root, target.path); + return (await targetHash(destination)) === target.newHash; + }), + )).every(Boolean); + if (selectedNew) return; + throw new TransactionCommitError(id, error); + } +} + +export async function runFileTransaction( + root: string, + operation: (context: FileTransactionContext) => Promise, + options: FileTransactionOptions = {}, +): Promise { + const resolvedRoot = resolve(root); + const lock = await acquireLock(resolvedRoot, options); + try { + await recoverHeld(resolvedRoot); + let committed = false; + const result = await operation({ + async commit(files) { + if (committed) throw new Error("artifact transaction already committed"); + committed = true; + await commitHeld(resolvedRoot, lock, files, options); + }, + }); + return result; + } finally { + await lock.release(); + } +} + +export async function recoverFileTransactions( + root: string, + options: FileTransactionOptions = {}, +): Promise { + await runFileTransaction(root, async () => {}, options); +} diff --git a/src/github-pages.ts b/src/github-pages.ts index 38b9a50..b9b7857 100644 --- a/src/github-pages.ts +++ b/src/github-pages.ts @@ -30,7 +30,7 @@ export function pagesBaseUrl(repo: string): string { return `https://${owner}.github.io/${name}/`; } -const SKIP_ENTRIES = new Set([".git", ".state", ".db", ".datasources"]); +const SKIP_ENTRIES = new Set([".git", ".state", ".db", ".datasources", ".transactions"]); export async function copyArtifacts(fromDir: string, toDir: string): Promise { await mkdir(toDir, { recursive: true }); diff --git a/src/publisher.ts b/src/publisher.ts index 4c720ba..69694e8 100644 --- a/src/publisher.ts +++ b/src/publisher.ts @@ -1,7 +1,12 @@ -import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { readFile, readdir } from "node:fs/promises"; import { createHash } from "node:crypto"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { renderGallery } from "./gallery.ts"; +import { + recoverFileTransactions, + runFileTransaction, + type FileTransactionContext, +} from "./file-transaction.ts"; import { ArtifactTooLargeError, DEFAULT_MAX_BYTES, @@ -69,24 +74,6 @@ export interface Publisher { const MANIFEST_FILE = "manifest.json"; const GALLERY_FILE = "index.html"; -const directoryWrites = new Map>(); - -async function serializeDirectory(dir: string, operation: () => Promise): Promise { - const key = resolve(dir); - const previous = directoryWrites.get(key) ?? Promise.resolve(); - let release: () => void = () => {}; - const current = new Promise((done) => { - release = done; - }); - directoryWrites.set(key, current); - await previous; - try { - return await operation(); - } finally { - release(); - if (directoryWrites.get(key) === current) directoryWrites.delete(key); - } -} async function readManifest(dir: string): Promise { try { @@ -118,11 +105,13 @@ export class FilePublisher implements Publisher { } async publish(input: PublishInput): Promise { - return serializeDirectory(this.dir, () => this.publishSerialized(input)); + return runFileTransaction(this.dir, (transaction) => this.publishSerialized(input, transaction)); } - private async publishSerialized(input: PublishInput): Promise { - await mkdir(this.dir, { recursive: true }); + private async publishSerialized( + input: PublishInput, + transaction: FileTransactionContext, + ): Promise { const manifest = await readManifest(this.dir); const existing = manifest.artifacts[input.slug]; const now = new Date().toISOString(); @@ -175,30 +164,44 @@ export class FilePublisher implements Publisher { hash, }; - const stable = join(this.dir, `${input.slug}.html`); + const stableName = `${input.slug}.html`; + const stable = join(this.dir, stableName); + const files = new Map(); if (input.version) { if (existing && existing.versions.includes(existing.current)) { - const currentArchive = join(this.dir, `${input.slug}.v${existing.current}.html`); - try { - await access(currentArchive); - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; - await writeFile(currentArchive, await readFile(stable)); - } + const currentArchiveName = `${input.slug}.v${existing.current}.html`; + const currentArchive = join(this.dir, currentArchiveName); + const alreadyArchived = await readFile(currentArchive).then( + () => true, + (error: unknown) => { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + return false; + } + throw error; + }, + ); + if (!alreadyArchived) files.set(currentArchiveName, await readFile(stable)); } - await writeFile(join(this.dir, `${input.slug}.v${nextVersion}.html`), html, "utf8"); + files.set(`${input.slug}.v${nextVersion}.html`, html); } - await writeFile(stable, html, "utf8"); + files.set(stableName, html); manifest.artifacts[input.slug] = meta; - await writeFile(join(this.dir, MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + files.set(MANIFEST_FILE, `${JSON.stringify(manifest, null, 2)}\n`); const gallery = join(this.dir, GALLERY_FILE); - await writeFile(gallery, renderGallery(manifest), "utf8"); + files.set(GALLERY_FILE, renderGallery(manifest)); + await transaction.commit(files); return { path: stable, version: nextVersion, gallery, hash }; } async latest(): Promise { + await recoverFileTransactions(this.dir); const manifest = await readManifest(this.dir); return Object.values(manifest.artifacts).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt), @@ -206,10 +209,16 @@ export class FilePublisher implements Publisher { } async restore(slug: string, version: number): Promise { - return serializeDirectory(this.dir, () => this.restoreSerialized(slug, version)); + return runFileTransaction(this.dir, (transaction) => + this.restoreSerialized(slug, version, transaction), + ); } - private async restoreSerialized(slug: string, version: number): Promise { + private async restoreSerialized( + slug: string, + version: number, + transaction: FileTransactionContext, + ): Promise { const manifest = await readManifest(this.dir); const meta = manifest.artifacts[slug]; if (!meta) throw new Error(`unknown artifact: ${slug}`); @@ -219,15 +228,19 @@ export class FilePublisher implements Publisher { const stable = join(this.dir, `${slug}.html`); const content = await readFile(join(this.dir, `${slug}.v${version}.html`), "utf8"); - await writeFile(stable, content, "utf8"); meta.current = version; meta.updatedAt = new Date().toISOString(); meta.hash = contentHash(content); manifest.artifacts[slug] = meta; - await writeFile(join(this.dir, MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); const gallery = join(this.dir, GALLERY_FILE); - await writeFile(gallery, renderGallery(manifest), "utf8"); + await transaction.commit( + new Map([ + [`${slug}.html`, content], + [MANIFEST_FILE, `${JSON.stringify(manifest, null, 2)}\n`], + [GALLERY_FILE, renderGallery(manifest)], + ]), + ); return { path: stable, version, gallery, hash: meta.hash }; } diff --git a/test/file-transaction.test.ts b/test/file-transaction.test.ts new file mode 100644 index 0000000..731647e --- /dev/null +++ b/test/file-transaction.test.ts @@ -0,0 +1,252 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { + access, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + recoverFileTransactions, + runFileTransaction, + TransactionCommitError, + TransactionLockTimeoutError, + TransactionRecoveryError, + TRANSACTION_DIRECTORY, +} from "../src/file-transaction.ts"; +import { FilePublisher, type Manifest } from "../src/publisher.ts"; +import { + ALL_TRANSACTION_FAULT_POINTS, + modeledOutcome, +} from "./model/file-transaction-model.ts"; + +const FIXTURE = join(dirname(fileURLToPath(import.meta.url)), "fixtures", "file-transaction-worker.ts"); + +interface WorkerResult { + code: number | null; + stdout: string; + stderr: string; +} + +async function withTempDir(run: (dir: string) => Promise): Promise { + const dir = await mkdtemp(join(tmpdir(), "file-transaction-")); + try { + await run(dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +function startWorker(args: string[]): { + child: ChildProcessWithoutNullStreams; + result: Promise; +} { + const child = spawn(process.execPath, [FIXTURE, ...args], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const result = new Promise((resolveResult, rejectResult) => { + child.on("error", rejectResult); + child.on("exit", (code) => resolveResult({ code, stdout, stderr })); + }); + return { child, result }; +} + +async function waitForFiles(paths: string[]): Promise { + const deadline = Date.now() + 5_000; + for (;;) { + const found = await Promise.all( + paths.map((path) => access(path).then(() => true, () => false)), + ); + if (found.every(Boolean)) return; + if (Date.now() >= deadline) throw new Error(`workers did not reach barrier: ${paths.join(", ")}`); + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } +} + +test("independent processes allow one same-head winner", async () => { + await withTempDir(async (dir) => { + const first = await new FilePublisher(dir).publish({ slug: "report", html: "one" }); + const readyA = join(dir, "ready-a"); + const readyB = join(dir, "ready-b"); + const go = join(dir, "go"); + const workerA = startWorker(["publish", dir, "report", "two", first.hash, readyA, go]); + const workerB = startWorker(["publish", dir, "report", "three", first.hash, readyB, go]); + await waitForFiles([readyA, readyB]); + await writeFile(go, "go", "utf8"); + const results = await Promise.all([workerA.result, workerB.result]); + assert.deepEqual(results.map((result) => result.code), [0, 0]); + const statuses = results.map((result) => { + assert.equal(result.stderr, ""); + return (JSON.parse(result.stdout) as { status: string }).status; + }); + assert.deepEqual(statuses.sort(), ["committed", "stale"]); + }); +}); + +test("independent processes do not lose different-artifact manifest entries", async () => { + await withTempDir(async (dir) => { + const readyA = join(dir, "ready-a"); + const readyB = join(dir, "ready-b"); + const go = join(dir, "go"); + const workerA = startWorker(["publish", dir, "alpha", "one", "-", readyA, go]); + const workerB = startWorker(["publish", dir, "beta", "two", "-", readyB, go]); + await waitForFiles([readyA, readyB]); + await writeFile(go, "go", "utf8"); + const results = await Promise.all([workerA.result, workerB.result]); + for (const result of results) { + assert.equal(result.code, 0, result.stderr); + assert.equal((JSON.parse(result.stdout) as { status: string }).status, "committed"); + } + const manifest = JSON.parse(await readFile(join(dir, "manifest.json"), "utf8")) as Manifest; + assert.deepEqual(Object.keys(manifest.artifacts).sort(), ["alpha", "beta"]); + }); +}); + +test("every crash boundary recovers the modeled complete old or new transaction", { timeout: 20_000 }, async () => { + const cases = ALL_TRANSACTION_FAULT_POINTS.flatMap((point) => + point === "target-staged" || point === "target-backed-up" || point === "target-replaced" + ? [`${point}@a.txt`, `${point}@b.txt`] + : [point], + ); + for (const faultSpec of cases) { + const point = faultSpec.split("@")[0] as (typeof ALL_TRANSACTION_FAULT_POINTS)[number]; + await withTempDir(async (dir) => { + await writeFile(join(dir, "a.txt"), "old-a", "utf8"); + await writeFile(join(dir, "b.txt"), "old-b", "utf8"); + const worker = startWorker(["crash", dir, faultSpec]); + const result = await worker.result; + assert.equal(result.code, 86, `${faultSpec}: ${result.stderr}`); + await recoverFileTransactions(dir); + const outcome = modeledOutcome(point); + assert.equal( + await readFile(join(dir, "a.txt"), "utf8"), + outcome === "old" ? "old-a" : "new-a", + faultSpec, + ); + assert.equal( + await readFile(join(dir, "b.txt"), "utf8"), + outcome === "old" ? "old-b" : "new-b", + faultSpec, + ); + assert.deepEqual(await readdir(join(dir, TRANSACTION_DIRECTORY)), ["fence"]); + }); + } +}); + +test("a live owner cannot be stolen and lock wait is bounded", async () => { + await withTempDir(async (dir) => { + const ready = join(dir, "ready"); + const release = join(dir, "release"); + const worker = startWorker(["hold", dir, ready, release]); + await waitForFiles([ready]); + await assert.rejects( + recoverFileTransactions(dir, { lockTimeoutMs: 50, pollIntervalMs: 5 }), + TransactionLockTimeoutError, + ); + const controller = new AbortController(); + const cancelled = recoverFileTransactions(dir, { + signal: controller.signal, + lockTimeoutMs: 5_000, + pollIntervalMs: 5, + }); + controller.abort(); + await assert.rejects(cancelled, { name: "AbortError" }); + await writeFile(release, "release", "utf8"); + const result = await worker.result; + assert.equal(result.code, 0, result.stderr); + }); +}); + +test("caught failures report the selected old state or finish recovered new state", async () => { + await withTempDir(async (dir) => { + await writeFile(join(dir, "value.txt"), "old", "utf8"); + await assert.rejects( + runFileTransaction( + dir, + (transaction) => transaction.commit(new Map([["value.txt", "new"]])), + { + fault(point) { + if (point === "journal-prepared") throw new Error("before decision"); + }, + }, + ), + TransactionCommitError, + ); + assert.equal(await readFile(join(dir, "value.txt"), "utf8"), "old"); + + await runFileTransaction( + dir, + (transaction) => transaction.commit(new Map([["value.txt", "new"]])), + { + fault(point) { + if (point === "target-backed-up") throw new Error("after decision"); + }, + }, + ); + assert.equal(await readFile(join(dir, "value.txt"), "utf8"), "new"); + }); +}); + +test("unknown transaction directories and invalid live-lock metadata fail closed", async () => { + await withTempDir(async (dir) => { + const transactions = join(dir, TRANSACTION_DIRECTORY); + const unexpected = join(transactions, "user-data"); + await mkdir(unexpected, { recursive: true }); + await writeFile(join(unexpected, "marker"), "keep", "utf8"); + await assert.rejects(recoverFileTransactions(dir), TransactionRecoveryError); + assert.equal(await readFile(join(unexpected, "marker"), "utf8"), "keep"); + await rm(unexpected, { recursive: true, force: true }); + + const lock = join(transactions, "lock"); + await mkdir(lock, { recursive: true }); + await writeFile(join(lock, "owner.json"), "{}", "utf8"); + await assert.rejects( + recoverFileTransactions(dir, { lockTimeoutMs: 20, pollIntervalMs: 5 }), + /owner metadata is invalid/, + ); + assert.equal(await readFile(join(lock, "owner.json"), "utf8"), "{}"); + }); +}); + +test("transaction targets reject symlink escape and excessive target count", async () => { + await withTempDir(async (dir) => { + const outside = await mkdtemp(join(tmpdir(), "file-transaction-outside-")); + try { + await symlink(outside, join(dir, "linked"), "dir"); + await assert.rejects( + runFileTransaction(dir, (transaction) => + transaction.commit(new Map([["linked/escape.txt", "no"]])), + ), + /not a real directory/, + ); + await assert.rejects(readFile(join(outside, "escape.txt"), "utf8")); + + const tooMany = new Map(); + for (let index = 0; index < 65; index++) tooMany.set(`target-${index}.txt`, "x"); + await assert.rejects( + runFileTransaction(dir, (transaction) => transaction.commit(tooMany)), + /65 targets; limit is 64/, + ); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/test/fixtures/file-transaction-worker.ts b/test/fixtures/file-transaction-worker.ts new file mode 100644 index 0000000..db5385e --- /dev/null +++ b/test/fixtures/file-transaction-worker.ts @@ -0,0 +1,90 @@ +import { access, readFile, writeFile } from "node:fs/promises"; +import { FilePublisher, StaleArtifactError } from "../../src/publisher.ts"; +import { + runFileTransaction, + type TransactionFaultPoint, +} from "../../src/file-transaction.ts"; + +async function waitFor(path: string): Promise { + for (;;) { + try { + await access(path); + return; + } catch { + await new Promise((resolveWait) => setTimeout(resolveWait, 5)); + } + } +} + +async function publishWorker(args: string[]): Promise { + const [dir, slug, html, expectedHash, readyPath, goPath] = args; + if (!dir || !slug || !html || !readyPath || !goPath) throw new Error("missing publish worker argument"); + await writeFile(readyPath, "ready", "utf8"); + await waitFor(goPath); + try { + const result = await new FilePublisher(dir).publish({ + slug, + html, + expectedHash: expectedHash === "-" ? undefined : expectedHash, + }); + process.stdout.write(`${JSON.stringify({ status: "committed", hash: result.hash })}\n`); + } catch (error) { + if (error instanceof StaleArtifactError) { + process.stdout.write(`${JSON.stringify({ status: "stale", hash: error.currentHash })}\n`); + return; + } + throw error; + } +} + +async function crashWorker(args: string[]): Promise { + const [dir, faultSpec] = args; + if (!dir || !faultSpec) throw new Error("missing crash worker argument"); + const [faultPoint, faultTarget] = faultSpec.split("@"); + await runFileTransaction( + dir, + async (transaction) => { + await transaction.commit( + new Map([ + ["a.txt", "new-a"], + ["b.txt", "new-b"], + ]), + ); + }, + { + fault(point, target) { + if ( + point === (faultPoint as TransactionFaultPoint) && + (faultTarget === undefined || target === faultTarget) + ) { + process.exit(86); + } + }, + }, + ); +} + +async function holdWorker(args: string[]): Promise { + const [dir, readyPath, releasePath] = args; + if (!dir || !readyPath || !releasePath) throw new Error("missing hold worker argument"); + await runFileTransaction(dir, async () => { + await writeFile(readyPath, String(process.pid), "utf8"); + await waitFor(releasePath); + }); +} + +async function main(): Promise { + const [mode, ...args] = process.argv.slice(2); + if (mode === "publish") return publishWorker(args); + if (mode === "crash") return crashWorker(args); + if (mode === "hold") return holdWorker(args); + if (mode === "read") { + const [path] = args; + if (!path) throw new Error("missing read path"); + process.stdout.write(await readFile(path, "utf8")); + return; + } + throw new Error(`unknown worker mode ${JSON.stringify(mode)}`); +} + +await main(); diff --git a/test/github-pages.test.ts b/test/github-pages.test.ts index 6b2f3e7..2dc8628 100644 --- a/test/github-pages.test.ts +++ b/test/github-pages.test.ts @@ -59,8 +59,10 @@ test("local state directories are never published", async () => { const localDir = join(dir, "local"); await mkdir(join(cloneDir, ".git"), { recursive: true }); await mkdir(join(localDir, ".state"), { recursive: true }); + await mkdir(join(localDir, ".transactions"), { recursive: true }); await mkdir(join(localDir, ".db"), { recursive: true }); await writeFile(join(localDir, ".state", "answers.json"), "{}"); + await writeFile(join(localDir, ".transactions", "journal.json"), "{}"); await writeFile(join(localDir, ".db", "x.json"), "{}"); const publisher = new GitHubPagesPublisher(localDir, { @@ -71,6 +73,7 @@ test("local state directories are never published", async () => { await publisher.publish({ slug: "demo", html: "x" }); await assert.rejects(readFile(join(cloneDir, ".state", "answers.json"), "utf8")); + await assert.rejects(readFile(join(cloneDir, ".transactions", "journal.json"), "utf8")); await assert.rejects(readFile(join(cloneDir, ".db", "x.json"), "utf8")); await rm(dir, { recursive: true, force: true }); }); diff --git a/test/model/file-transaction-model.ts b/test/model/file-transaction-model.ts new file mode 100644 index 0000000..58d2ad8 --- /dev/null +++ b/test/model/file-transaction-model.ts @@ -0,0 +1,24 @@ +import type { TransactionFaultPoint } from "../../src/file-transaction.ts"; + +export type TransactionOutcome = "old" | "new"; + +const PRE_COMMIT_POINTS = new Set([ + "stage-created", + "target-staged", + "journal-prepared", +]); + +export function modeledOutcome(point: TransactionFaultPoint): TransactionOutcome { + return PRE_COMMIT_POINTS.has(point) ? "old" : "new"; +} + +export const ALL_TRANSACTION_FAULT_POINTS: TransactionFaultPoint[] = [ + "stage-created", + "target-staged", + "journal-prepared", + "commit-decided", + "target-backed-up", + "target-replaced", + "commit-verified", + "journal-committed", +]; From b5b0463f8d542840c250940e9f1b7d411d0f7ea4 Mon Sep 17 00:00:00 2001 From: bitgorust Date: Sun, 16 Aug 2026 23:01:38 +0200 Subject: [PATCH 4/7] feat: add durable artifact identity migration --- .../change.json | 2 +- .../tasks.md | 6 +- .../artifact-publication-transaction/tasks.md | 2 +- src/artifact-migration.ts | 867 ++++++++++++++++++ src/artifact-schema.ts | 417 +++++++++ src/file-transaction.ts | 94 +- src/github-pages.ts | 10 +- src/publisher.ts | 273 +++++- test/artifact-migration.test.ts | 213 +++++ test/artifact-publisher-v2.test.ts | 144 +++ test/artifact-schema.test.ts | 108 +++ test/file-transaction.test.ts | 31 + test/github-pages.test.ts | 6 + 13 files changed, 2127 insertions(+), 46 deletions(-) create mode 100644 src/artifact-migration.ts create mode 100644 src/artifact-schema.ts create mode 100644 test/artifact-migration.test.ts create mode 100644 test/artifact-publisher-v2.test.ts create mode 100644 test/artifact-schema.test.ts diff --git a/specs/changes/artifact-identity-schema-migration/change.json b/specs/changes/artifact-identity-schema-migration/change.json index bc5fe5f..64571cc 100644 --- a/specs/changes/artifact-identity-schema-migration/change.json +++ b/specs/changes/artifact-identity-schema-migration/change.json @@ -3,7 +3,7 @@ "id": "artifact-identity-schema-migration", "title": "Introduce durable artifact identity and schema migration", "lane": "high-risk", - "status": "approved", + "status": "implementing", "affectedRequirements": [ "LIFE-01", "LIFE-02", diff --git a/specs/changes/artifact-identity-schema-migration/tasks.md b/specs/changes/artifact-identity-schema-migration/tasks.md index ee66e69..104f847 100644 --- a/specs/changes/artifact-identity-schema-migration/tasks.md +++ b/specs/changes/artifact-identity-schema-migration/tasks.md @@ -1,9 +1,9 @@ # Tasks: Introduce durable artifact identity and schema migration -- [ ] Confirm proposal validation and human approval. -- [ ] Add the artifact/revision schemas, exact validators, typed errors, and opaque-ID/slug index. +- [x] Confirm proposal validation and human approval. +- [x] Add the artifact/revision schemas, exact validators, typed errors, and opaque-ID/slug index. - [ ] Add every released local manifest/state fixture and historical shared-KV key fixture. -- [ ] Implement bounded inspect, dry-run repair report, backup, staged migration, resume, +- [x] Implement bounded inspect, dry-run repair report, backup, staged migration, resume, verification, and rollback without enabling unverified platforms by default. - [ ] Add deterministic identity, migration, idempotence, backup/restore, and fault/property tests. - [ ] Retain exact supported-write-platform results and explicit unavailable cells. diff --git a/specs/changes/artifact-publication-transaction/tasks.md b/specs/changes/artifact-publication-transaction/tasks.md index b5151ed..02ed70b 100644 --- a/specs/changes/artifact-publication-transaction/tasks.md +++ b/specs/changes/artifact-publication-transaction/tasks.md @@ -6,7 +6,7 @@ cleanup, and public-staging exclusions. - [x] Route create, update, restore, and managed reads through recovery and the transaction. - [x] Add deterministic fault injection and the bounded transaction state model. -- [ ] Add independent-process same-head/different-artifact races, stale takeover, retry, path, +- [x] Add independent-process same-head/different-artifact races, stale takeover, retry, path, corruption, cancellation, and resource-limit tests. - [ ] Retain exact supported-filesystem results and explicit unavailable cells. - [ ] Record validation/verification evidence and update `specs/current/artifact-lifecycle.spec.md`. diff --git a/src/artifact-migration.ts b/src/artifact-migration.ts new file mode 100644 index 0000000..844c8bb --- /dev/null +++ b/src/artifact-migration.ts @@ -0,0 +1,867 @@ +import { createHash, randomUUID } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { + ARTIFACT_ID_RE, + ARTIFACT_MANIFEST_FILE, + ARTIFACT_MANIFEST_SCHEMA_VERSION, + ARTIFACT_SLUG_RE, + emptyArtifactManifestV2, + readArtifactManifestV2, + validateArtifactManifestV2, + type ArtifactManifestV2, + type ArtifactRecordV2, + type RevisionRecordV2, +} from "./artifact-schema.ts"; +import { recoverFileTransactions, runFileTransaction } from "./file-transaction.ts"; +import { renderGallery } from "./gallery.ts"; +import { DEFAULT_MAX_BYTES } from "./render.ts"; +import type { ArtifactMeta, Manifest } from "./publisher.ts"; + +const MIGRATION_PLAN_SCHEMA_VERSION = 1; +const MAX_LEGACY_ARTIFACTS = 10_000; +const MAX_LEGACY_FILES = 50_000; +const COPY_BATCH_TARGETS = 32; +const UUID_RE = ARTIFACT_ID_RE; + +export type MigrationIssueSeverity = "warning" | "error"; + +export interface ArtifactMigrationIssue { + severity: MigrationIssueSeverity; + code: string; + artifact?: string; + path?: string; + detail: string; +} + +export interface ArtifactMigrationCopy { + sourcePath: string; + targetPath: string; + contentHash: string; + bytes: number; + purpose: "backup" | "revision"; +} + +export interface LegacyStateAssociation { + path: string; + artifactId: string; + kind: "decisions" | "comments" | "collection" | "datasource"; +} + +export interface ArtifactMigrationPlan { + schemaVersion: 1; + migrationId: string; + createdAt: string; + sourceManifestExisted: boolean; + sourceManifestHash: string | null; + alreadyCurrent: boolean; + canMigrate: boolean; + manifest: ArtifactManifestV2 | null; + copies: ArtifactMigrationCopy[]; + stateAssociations: LegacyStateAssociation[]; + issues: ArtifactMigrationIssue[]; +} + +export interface ArtifactMigrationOptions { + artifactIdFactory?: () => string; + migrationId?: string; + now?: string; +} + +export interface ArtifactMigrationResult { + migrationId: string; + status: "already-current" | "migrated" | "rolled-back"; + manifestHash: string | null; + issues: ArtifactMigrationIssue[]; +} + +interface MigrationInventory { + schemaVersion: 1; + migrationId: string; + sourceManifestExisted: boolean; + sourceManifestHash: string | null; + selectedManifestHash: string; + originalIndexExisted: boolean; + originalIndexHash: string | null; + copies: ArtifactMigrationCopy[]; +} + +interface LegacyPage { + sourcePath: string; + legacyRevision?: number; + bytes: number; + contentHash: string; + content: Uint8Array; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function errnoCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) return undefined; + const code = (error as { code?: unknown }).code; + return typeof code === "string" ? code : undefined; +} + +function sha256(value: Uint8Array | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function validTimestamp(value: unknown): string | null { + if ( + typeof value === "string" && + /(?:Z|[+-]\d{2}:\d{2})$/.test(value) && + !Number.isNaN(Date.parse(value)) + ) { + return value; + } + return null; +} + +function legacyString(value: unknown): string | undefined { + return typeof value === "string" && value.length <= 16_384 ? value : undefined; +} + +function issue( + issues: ArtifactMigrationIssue[], + severity: MigrationIssueSeverity, + code: string, + detail: string, + fields: { artifact?: string; path?: string } = {}, +): void { + issues.push({ severity, code, detail, ...fields }); +} + +async function readOptional(path: string): Promise { + try { + return await readFile(path); + } catch (error) { + if (errnoCode(error) === "ENOENT") return undefined; + throw error; + } +} + +async function readBoundedSource( + dir: string, + relativePath: string, + issues: ArtifactMigrationIssue[], + artifact?: string, +): Promise { + const path = join(dir, ...relativePath.split("/")); + let info; + try { + info = await lstat(path); + } catch (error) { + if (errnoCode(error) === "ENOENT") return undefined; + throw error; + } + if (info.isSymbolicLink() || !info.isFile()) { + issue(issues, "error", "unsafe-legacy-file", "legacy path is not a regular contained file", { + artifact, + path: relativePath, + }); + return undefined; + } + if (info.size > DEFAULT_MAX_BYTES) { + issue( + issues, + "error", + "oversized-legacy-file", + `legacy page has ${info.size} bytes; limit is ${DEFAULT_MAX_BYTES}`, + { artifact, path: relativePath }, + ); + return undefined; + } + const content = await readFile(path); + return { + sourcePath: relativePath, + bytes: content.byteLength, + contentHash: sha256(content), + content, + }; +} + +function legacyVersions(value: unknown): number[] { + if (!Array.isArray(value)) return []; + return [ + ...new Set( + value.filter( + (entry): entry is number => + typeof entry === "number" && Number.isSafeInteger(entry) && entry > 0, + ), + ), + ].sort((left, right) => left - right); +} + +function revisionTimestamp( + index: number, + total: number, + createdAt: string | null, + updatedAt: string | null, +): { value: string | null; source: "legacy-artifact" | "unknown" } { + if (index === total - 1 && updatedAt !== null) return { value: updatedAt, source: "legacy-artifact" }; + if (index === 0 && createdAt !== null) return { value: createdAt, source: "legacy-artifact" }; + return { value: null, source: "unknown" }; +} + +function asLegacyManifest(manifest: ArtifactManifestV2): Manifest { + const artifacts: Record = {}; + for (const artifact of Object.values(manifest.artifacts)) { + artifacts[artifact.slug] = { + slug: artifact.slug, + title: artifact.title, + icon: artifact.icon, + description: artifact.description, + source: artifact.source, + createdAt: artifact.createdAt ?? "unknown", + updatedAt: artifact.updatedAt ?? "unknown", + current: artifact.headRevision, + versions: artifact.revisions.map((revision) => revision.revision), + charts: artifact.charts, + bytes: artifact.bytes, + hash: artifact.contentHash.slice(0, 12), + }; + } + return { artifacts }; +} + +async function walkRegularFiles( + dir: string, + relativeRoot: string, + issues: ArtifactMigrationIssue[], +): Promise { + const root = join(dir, ...relativeRoot.split("/")); + let entries; + try { + entries = await readdir(root, { withFileTypes: true }); + } catch (error) { + if (errnoCode(error) === "ENOENT") return []; + throw error; + } + const found: string[] = []; + const pending = entries.map((entry) => `${relativeRoot}/${entry.name}`); + while (pending.length > 0) { + const relativePath = pending.shift(); + if (relativePath === undefined) break; + if (found.length + pending.length > MAX_LEGACY_FILES) { + issue(issues, "error", "legacy-file-limit", `legacy file inventory exceeds ${MAX_LEGACY_FILES}`); + break; + } + const info = await lstat(join(dir, ...relativePath.split("/"))); + if (info.isSymbolicLink()) { + issue(issues, "error", "unsafe-legacy-file", "legacy state contains a symbolic link", { + path: relativePath, + }); + } else if (info.isDirectory()) { + const children = await readdir(join(dir, ...relativePath.split("/")), { withFileTypes: true }); + for (const child of children) pending.push(`${relativePath}/${child.name}`); + } else if (info.isFile()) { + found.push(relativePath); + } else { + issue(issues, "error", "unsafe-legacy-file", "legacy state contains an unsupported file type", { + path: relativePath, + }); + } + } + return found.sort(); +} + +function associationFor( + path: string, + slugIndex: Record, +): LegacyStateAssociation | undefined { + const stateMatch = /^\.state\/([a-z0-9]+(?:-[a-z0-9]+)*)\.(comments\.)?json$/.exec(path); + if (stateMatch) { + const artifactId = slugIndex[stateMatch[1]]; + if (!artifactId) return undefined; + return { + path, + artifactId, + kind: stateMatch[2] === undefined ? "decisions" : "comments", + }; + } + const dbMatch = /^\.db\/([a-z0-9]+(?:-[a-z0-9]+)*)\/[^/]+\.json$/.exec(path); + if (dbMatch) { + const artifactId = slugIndex[dbMatch[1]]; + return artifactId ? { path, artifactId, kind: "collection" } : undefined; + } + const datasourceMatch = /^\.datasources\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/.exec(path); + if (datasourceMatch) { + const artifactId = slugIndex[datasourceMatch[1]]; + return artifactId ? { path, artifactId, kind: "datasource" } : undefined; + } + return undefined; +} + +export function mapLegacyCloudflareKey( + key: string, + siteId: string, + slugIndex: Record, +): + | { siteId: string; artifactId: string; kind: "decisions" | "comments" | "collection"; collection?: string } + | { issue: ArtifactMigrationIssue } { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(siteId)) { + return { + issue: { + severity: "error", + code: "invalid-site-id", + detail: "site identity is not a safe canonical identifier", + }, + }; + } + const state = /^(state|comments):([a-z0-9]+(?:-[a-z0-9]+)*)$/.exec(key); + if (state) { + const artifactId = slugIndex[state[2]]; + if (artifactId) { + return { + siteId, + artifactId, + kind: state[1] === "state" ? "decisions" : "comments", + }; + } + } + const collection = /^db:([a-z0-9]+(?:-[a-z0-9]+)*):([a-z0-9]+(?:-[a-z0-9]+)*)$/.exec(key); + if (collection) { + const artifactId = slugIndex[collection[1]]; + if (artifactId) { + return { siteId, artifactId, kind: "collection", collection: collection[2] }; + } + } + return { + issue: { + severity: "error", + code: "ambiguous-cloudflare-key", + path: key, + detail: "historical shared-KV key cannot be mapped to one known artifact", + }, + }; +} + +export async function planArtifactMigration( + dir: string, + options: ArtifactMigrationOptions = {}, +): Promise { + await recoverFileTransactions(dir); + const migrationId = options.migrationId ?? randomUUID(); + if (!UUID_RE.test(migrationId)) throw new Error("migrationId must be a UUID"); + const createdAt = options.now ?? new Date().toISOString(); + if (Number.isNaN(Date.parse(createdAt))) throw new Error("migration time must be an ISO timestamp"); + const idFactory = options.artifactIdFactory ?? randomUUID; + const issues: ArtifactMigrationIssue[] = []; + const manifestPath = join(dir, ARTIFACT_MANIFEST_FILE); + let manifestBytes: Uint8Array | undefined; + try { + const info = await lstat(manifestPath); + if (info.isSymbolicLink() || !info.isFile()) { + issue(issues, "error", "unsafe-legacy-file", "managed legacy path is not a regular file", { + path: ARTIFACT_MANIFEST_FILE, + }); + } else { + manifestBytes = await readFile(manifestPath); + } + } catch (error) { + if (errnoCode(error) !== "ENOENT") throw error; + } + const sourceManifestHash = manifestBytes === undefined ? null : sha256(manifestBytes); + const base: Omit = { + schemaVersion: MIGRATION_PLAN_SCHEMA_VERSION, + migrationId, + createdAt, + sourceManifestExisted: manifestBytes !== undefined, + sourceManifestHash, + issues, + }; + + let legacy: unknown = { artifacts: {} }; + if (manifestBytes !== undefined) { + try { + legacy = JSON.parse(Buffer.from(manifestBytes).toString("utf8")) as unknown; + } catch (error) { + issue(issues, "error", "malformed-manifest", error instanceof Error ? error.message : String(error), { + path: ARTIFACT_MANIFEST_FILE, + }); + return { ...base, alreadyCurrent: false, canMigrate: false, manifest: null, copies: [], stateAssociations: [] }; + } + } + if (isRecord(legacy) && legacy["schemaVersion"] === ARTIFACT_MANIFEST_SCHEMA_VERSION) { + try { + const manifest = validateArtifactManifestV2(legacy); + return { ...base, alreadyCurrent: true, canMigrate: true, manifest, copies: [], stateAssociations: [] }; + } catch (error) { + issue(issues, "error", "invalid-current-manifest", error instanceof Error ? error.message : String(error)); + return { ...base, alreadyCurrent: false, canMigrate: false, manifest: null, copies: [], stateAssociations: [] }; + } + } + if (isRecord(legacy) && typeof legacy["schemaVersion"] === "number") { + issue( + issues, + "error", + "unknown-future-schema", + `schemaVersion ${String(legacy["schemaVersion"])} is not supported`, + { path: ARTIFACT_MANIFEST_FILE }, + ); + return { ...base, alreadyCurrent: false, canMigrate: false, manifest: null, copies: [], stateAssociations: [] }; + } + if (!isRecord(legacy) || !isRecord(legacy["artifacts"])) { + issue(issues, "error", "invalid-legacy-manifest", "legacy manifest must contain an artifacts object"); + return { ...base, alreadyCurrent: false, canMigrate: false, manifest: null, copies: [], stateAssociations: [] }; + } + + const legacyArtifacts = Object.entries(legacy["artifacts"]); + if (legacyArtifacts.length > MAX_LEGACY_ARTIFACTS) { + issue(issues, "error", "legacy-artifact-limit", `legacy manifest exceeds ${MAX_LEGACY_ARTIFACTS} artifacts`); + } + const rootEntries = await readdir(dir, { withFileTypes: true }).catch((error: unknown) => { + if (errnoCode(error) === "ENOENT") return []; + throw error; + }); + for (const entry of rootEntries) { + if ( + entry.isSymbolicLink() && + (entry.name === ARTIFACT_MANIFEST_FILE || + entry.name === "index.html" || + /^([a-z0-9]+(?:-[a-z0-9]+)*)\.html$/.test(entry.name) || + /^([a-z0-9]+(?:-[a-z0-9]+)*)\.v\d+\.html$/.test(entry.name)) + ) { + issue(issues, "error", "unsafe-legacy-file", "managed legacy path is a symbolic link", { + path: entry.name, + }); + } + } + const rootFiles = new Set(rootEntries.filter((entry) => entry.isFile()).map((entry) => entry.name)); + const manifest = emptyArtifactManifestV2(); + const copies: ArtifactMigrationCopy[] = []; + const sourceFiles = new Set(); + if (manifestBytes !== undefined) sourceFiles.add(ARTIFACT_MANIFEST_FILE); + if (rootFiles.has("index.html")) sourceFiles.add("index.html"); + const usedIds = new Set(); + const knownSlugs = new Set(); + + for (const [key, rawMeta] of legacyArtifacts) { + if (!ARTIFACT_SLUG_RE.test(key)) { + issue(issues, "error", "invalid-legacy-slug", "legacy manifest key is not a safe slug", { artifact: key }); + continue; + } + knownSlugs.add(key); + const meta = isRecord(rawMeta) ? rawMeta : {}; + if (typeof meta["slug"] === "string" && meta["slug"] !== key) { + issue(issues, "warning", "legacy-slug-mismatch", "manifest key is used as the stable slug", { + artifact: key, + }); + } + const id = idFactory(); + if (!UUID_RE.test(id) || usedIds.has(id)) { + issue(issues, "error", "invalid-generated-id", "artifact ID factory returned an invalid or duplicate UUID", { + artifact: key, + }); + continue; + } + usedIds.add(id); + + const advertised = legacyVersions(meta["versions"]); + const diskVersions = [...rootFiles] + .map((name) => new RegExp(`^${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\.v(\\d+)\\.html$`).exec(name)) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => Number(match[1])) + .filter((value) => Number.isSafeInteger(value) && value > 0) + .sort((left, right) => left - right); + for (const version of advertised) { + if (!diskVersions.includes(version)) { + issue(issues, "warning", "missing-advertised-revision", `advertised revision ${version} has no file`, { + artifact: key, + path: `${key}.v${version}.html`, + }); + } + } + for (const version of diskVersions) { + if (!advertised.includes(version)) { + issue(issues, "warning", "orphan-revision-recovered", `unadvertised revision ${version} is recoverable`, { + artifact: key, + path: `${key}.v${version}.html`, + }); + } + } + + const pages: LegacyPage[] = []; + for (const version of diskVersions) { + const relativePath = `${key}.v${version}.html`; + const page = await readBoundedSource(dir, relativePath, issues, key); + if (page) { + pages.push({ ...page, legacyRevision: version }); + sourceFiles.add(relativePath); + } + } + const stablePath = `${key}.html`; + const stable = await readBoundedSource(dir, stablePath, issues, key); + if (stable) sourceFiles.add(stablePath); + const legacyCurrent = + typeof meta["current"] === "number" && Number.isSafeInteger(meta["current"]) && meta["current"] > 0 + ? meta["current"] + : undefined; + let selected = stable; + if (!selected && legacyCurrent !== undefined) { + selected = pages.find((page) => page.legacyRevision === legacyCurrent); + } + selected ??= pages.at(-1); + if (!selected) { + issue(issues, "error", "irrecoverable-legacy-artifact", "no stable or revision page bytes exist", { + artifact: key, + }); + continue; + } + if (!stable) { + issue(issues, "warning", "missing-stable-head", "stable page is missing; selected recoverable revision is used", { + artifact: key, + path: stablePath, + }); + } + if (legacyCurrent !== undefined && !pages.some((page) => page.legacyRevision === legacyCurrent)) { + issue(issues, "warning", "missing-current-revision", `current revision ${legacyCurrent} is not retained`, { + artifact: key, + }); + } + if (pages.at(-1)?.contentHash !== selected.contentHash) { + pages.push({ ...selected, legacyRevision: legacyCurrent ?? selected.legacyRevision }); + issue( + issues, + "warning", + "selected-head-materialized", + "selected legacy head is materialized as a new final revision to preserve history ordering", + { artifact: key }, + ); + } + + const titleValue = legacyString(meta["title"]); + const iconValue = legacyString(meta["icon"]); + if (meta["title"] !== undefined && titleValue === undefined) { + issue(issues, "error", "invalid-legacy-title", "legacy title is not representable", { artifact: key }); + continue; + } + if (meta["icon"] !== undefined && iconValue === undefined) { + issue(issues, "error", "invalid-legacy-icon", "legacy icon is not representable", { artifact: key }); + continue; + } + const title = titleValue && titleValue.length > 0 ? titleValue : key; + const icon = iconValue && iconValue.length > 0 ? iconValue : "šŸ“„"; + if (!titleValue) issue(issues, "warning", "defaulted-legacy-title", "missing title defaults to exact slug", { artifact: key }); + if (!iconValue) issue(issues, "warning", "defaulted-legacy-icon", "missing icon defaults to the legacy document icon", { artifact: key }); + const description = legacyString(meta["description"]); + const source = legacyString(meta["source"]); + const author = legacyString(meta["author"]); + const createdAt = validTimestamp(meta["createdAt"]); + const updatedAt = validTimestamp(meta["updatedAt"]); + if (meta["createdAt"] !== undefined && createdAt === null) { + issue(issues, "warning", "unknown-created-time", "invalid legacy createdAt is retained as unknown", { artifact: key }); + } + if (meta["updatedAt"] !== undefined && updatedAt === null) { + issue(issues, "warning", "unknown-updated-time", "invalid legacy updatedAt is retained as unknown", { artifact: key }); + } + const charts = + typeof meta["charts"] === "number" && Number.isSafeInteger(meta["charts"]) && meta["charts"] >= 0 + ? meta["charts"] + : 0; + const revisions: RevisionRecordV2[] = []; + for (let index = 0; index < pages.length; index++) { + const page = pages[index]; + const revision = index + 1; + const timestamp = revisionTimestamp(index, pages.length, createdAt, updatedAt); + const pagePath = `revisions/${id}/${revision}.html`; + revisions.push({ + revision, + createdAt: timestamp.value, + bytes: page.bytes, + contentHash: page.contentHash, + pagePath, + title, + icon, + description, + source, + author, + charts, + provenance: { + kind: "migration", + legacyRevision: page.legacyRevision, + timestampSource: timestamp.source, + }, + }); + copies.push({ + sourcePath: page.sourcePath, + targetPath: pagePath, + contentHash: page.contentHash, + bytes: page.bytes, + purpose: "revision", + }); + } + const head = revisions.at(-1); + if (!head) continue; + const record: ArtifactRecordV2 = { + id, + slug: key, + title, + icon, + description, + source, + author, + createdAt, + updatedAt: head.createdAt, + headRevision: head.revision, + revisions, + charts, + bytes: head.bytes, + contentHash: head.contentHash, + deploymentReferences: [], + }; + manifest.artifacts[id] = record; + manifest.slugIndex[key] = id; + } + + for (const name of rootFiles) { + const stableMatch = /^([a-z0-9]+(?:-[a-z0-9]+)*)\.html$/.exec(name); + const versionMatch = /^([a-z0-9]+(?:-[a-z0-9]+)*)\.v\d+\.html$/.exec(name); + const slug = versionMatch?.[1] ?? (name === "index.html" ? undefined : stableMatch?.[1]); + if (slug && !knownSlugs.has(slug)) { + issue(issues, "warning", "untracked-page", "HTML file is preserved but not attached to an artifact", { + artifact: slug, + path: name, + }); + sourceFiles.add(name); + } + } + + const internalFiles = ( + await Promise.all( + [".state", ".db", ".datasources"].map((root) => walkRegularFiles(dir, root, issues)), + ) + ).flat(); + const stateAssociations: LegacyStateAssociation[] = []; + for (const path of internalFiles) { + sourceFiles.add(path); + const association = associationFor(path, manifest.slugIndex); + if (association) stateAssociations.push(association); + else { + issue(issues, "warning", "unmapped-local-state", "local state is backed up but cannot be mapped to one artifact", { + path, + }); + } + } + + for (const sourcePath of [...sourceFiles].sort()) { + const content = await readBoundedSource(dir, sourcePath, issues); + if (!content) continue; + copies.push({ + sourcePath, + targetPath: `.backups/migrations/${migrationId}/files/${sourcePath}`, + contentHash: content.contentHash, + bytes: content.bytes, + purpose: "backup", + }); + } + + try { + validateArtifactManifestV2(manifest); + } catch (error) { + issue(issues, "error", "generated-manifest-invalid", error instanceof Error ? error.message : String(error)); + } + const canMigrate = !issues.some((entry) => entry.severity === "error"); + return { + ...base, + alreadyCurrent: false, + canMigrate, + manifest, + copies, + stateAssociations, + }; +} + +function migrationReport(plan: ArtifactMigrationPlan): string { + return `${JSON.stringify( + { + schemaVersion: 1, + migrationId: plan.migrationId, + createdAt: plan.createdAt, + canMigrate: plan.canMigrate, + sourceManifestHash: plan.sourceManifestHash, + artifacts: plan.manifest ? Object.keys(plan.manifest.artifacts).length : 0, + copies: plan.copies.length, + stateAssociations: plan.stateAssociations, + issues: plan.issues, + }, + null, + 2, + )}\n`; +} + +function parseInventory(value: unknown): MigrationInventory { + if ( + !isRecord(value) || + value["schemaVersion"] !== 1 || + typeof value["migrationId"] !== "string" || + !UUID_RE.test(value["migrationId"]) || + typeof value["sourceManifestExisted"] !== "boolean" || + (value["sourceManifestHash"] !== null && typeof value["sourceManifestHash"] !== "string") || + typeof value["selectedManifestHash"] !== "string" || + typeof value["originalIndexExisted"] !== "boolean" || + (value["originalIndexHash"] !== null && typeof value["originalIndexHash"] !== "string") || + !Array.isArray(value["copies"]) + ) { + throw new Error("migration inventory is invalid"); + } + return value as unknown as MigrationInventory; +} + +export async function executeArtifactMigration( + dir: string, + plan: ArtifactMigrationPlan, +): Promise { + if (plan.alreadyCurrent) { + return { + migrationId: plan.migrationId, + status: "already-current", + manifestHash: plan.manifest ? sha256(`${JSON.stringify(plan.manifest, null, 2)}\n`) : null, + issues: plan.issues, + }; + } + if (!plan.canMigrate || !plan.manifest) { + throw new Error("artifact migration preflight failed; inspect the repair report before retrying"); + } + const targetManifest = plan.manifest; + validateArtifactManifestV2(targetManifest); + const manifestText = `${JSON.stringify(targetManifest, null, 2)}\n`; + const selectedManifestHash = sha256(manifestText); + const currentManifest = await readOptional(join(dir, ARTIFACT_MANIFEST_FILE)); + const currentHash = currentManifest === undefined ? null : sha256(currentManifest); + if (currentHash !== plan.sourceManifestHash) { + if (currentHash === selectedManifestHash) { + return { + migrationId: plan.migrationId, + status: "migrated", + manifestHash: selectedManifestHash, + issues: plan.issues, + }; + } + throw new Error("artifact migration source changed after preflight; regenerate the plan"); + } + + for (let offset = 0; offset < plan.copies.length; offset += COPY_BATCH_TARGETS) { + const batch = plan.copies.slice(offset, offset + COPY_BATCH_TARGETS); + await runFileTransaction(dir, async (transaction) => { + const files = new Map(); + for (const copy of batch) { + const existing = await readOptional(join(dir, ...copy.targetPath.split("/"))); + if (existing !== undefined) { + if (sha256(existing) !== copy.contentHash) { + throw new Error(`migration target already exists with different bytes: ${copy.targetPath}`); + } + continue; + } + const source = await readFile(join(dir, ...copy.sourcePath.split("/"))); + if (source.byteLength !== copy.bytes || sha256(source) !== copy.contentHash) { + throw new Error(`migration source changed after preflight: ${copy.sourcePath}`); + } + files.set(copy.targetPath, source); + } + if (files.size > 0) await transaction.commit(files); + }); + } + + const originalIndexCopy = plan.copies.find( + (copy) => copy.purpose === "backup" && copy.sourcePath === "index.html", + ); + const inventory: MigrationInventory = { + schemaVersion: 1, + migrationId: plan.migrationId, + sourceManifestExisted: plan.sourceManifestExisted, + sourceManifestHash: plan.sourceManifestHash, + selectedManifestHash, + originalIndexExisted: originalIndexCopy !== undefined, + originalIndexHash: originalIndexCopy?.contentHash ?? null, + copies: plan.copies, + }; + const inventoryText = `${JSON.stringify(inventory, null, 2)}\n`; + await runFileTransaction(dir, async (transaction) => { + const liveManifest = await readOptional(join(dir, ARTIFACT_MANIFEST_FILE)); + const liveHash = liveManifest === undefined ? null : sha256(liveManifest); + if (liveHash !== plan.sourceManifestHash) { + throw new Error("artifact migration source changed before selection"); + } + for (const source of new Map(plan.copies.map((copy) => [copy.sourcePath, copy])).values()) { + const live = await readFile(join(dir, ...source.sourcePath.split("/"))); + if (live.byteLength !== source.bytes || sha256(live) !== source.contentHash) { + throw new Error(`migration source changed before selection: ${source.sourcePath}`); + } + } + for (const copy of plan.copies.filter((entry) => entry.purpose === "revision")) { + const prepared = await readFile(join(dir, ...copy.targetPath.split("/"))); + if (sha256(prepared) !== copy.contentHash) { + throw new Error(`prepared revision failed verification: ${copy.targetPath}`); + } + } + await transaction.commit( + new Map([ + [ARTIFACT_MANIFEST_FILE, manifestText], + ["index.html", renderGallery(asLegacyManifest(targetManifest))], + [`.backups/migrations/${plan.migrationId}/inventory.json`, inventoryText], + [`.migrations/${plan.migrationId}/report.json`, migrationReport(plan)], + ]), + ); + }); + const selected = await readArtifactManifestV2(dir); + if (sha256(`${JSON.stringify(selected, null, 2)}\n`) !== selectedManifestHash) { + throw new Error("selected artifact manifest failed post-migration verification"); + } + return { + migrationId: plan.migrationId, + status: "migrated", + manifestHash: selectedManifestHash, + issues: plan.issues, + }; +} + +export async function rollbackArtifactMigration( + dir: string, + migrationId: string, +): Promise { + if (!UUID_RE.test(migrationId)) throw new Error("migrationId must be a UUID"); + const inventoryPath = `.backups/migrations/${migrationId}/inventory.json`; + const inventory = parseInventory( + JSON.parse(await readFile(join(dir, ...inventoryPath.split("/")), "utf8")) as unknown, + ); + const liveManifest = await readFile(join(dir, ARTIFACT_MANIFEST_FILE)); + if (sha256(liveManifest) !== inventory.selectedManifestHash) { + throw new Error("selected manifest changed after migration; rollback requires a new preflight"); + } + const manifestBackupPath = `.backups/migrations/${migrationId}/files/${ARTIFACT_MANIFEST_FILE}`; + const indexBackupPath = `.backups/migrations/${migrationId}/files/index.html`; + const oldManifest = inventory.sourceManifestExisted + ? await readFile(join(dir, ...manifestBackupPath.split("/"))) + : null; + const oldIndex = inventory.originalIndexExisted + ? await readFile(join(dir, ...indexBackupPath.split("/"))) + : null; + if ( + (oldManifest !== null && sha256(oldManifest) !== inventory.sourceManifestHash) || + (oldIndex !== null && sha256(oldIndex) !== inventory.originalIndexHash) + ) { + throw new Error("migration backup verification failed; rollback made no changes"); + } + await runFileTransaction(dir, (transaction) => + transaction.commit( + new Map([ + [ARTIFACT_MANIFEST_FILE, oldManifest], + ["index.html", oldIndex], + ]), + ), + ); + const restoredManifest = await readOptional(join(dir, ARTIFACT_MANIFEST_FILE)); + const restoredHash = restoredManifest === undefined ? null : sha256(restoredManifest); + if (restoredHash !== inventory.sourceManifestHash) { + throw new Error("migration rollback post-verification failed"); + } + return { + migrationId, + status: "rolled-back", + manifestHash: restoredHash, + issues: [], + }; +} diff --git a/src/artifact-schema.ts b/src/artifact-schema.ts new file mode 100644 index 0000000..b557d4e --- /dev/null +++ b/src/artifact-schema.ts @@ -0,0 +1,417 @@ +import { lstat, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { recoverFileTransactions } from "./file-transaction.ts"; + +export const ARTIFACT_MANIFEST_SCHEMA_VERSION = 2; +export const ARTIFACT_MANIFEST_FILE = "manifest.json"; +export const ARTIFACT_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +export const ARTIFACT_SLUG_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +export const CONTENT_HASH_RE = /^[0-9a-f]{64}$/; + +const MAX_ARTIFACTS = 10_000; +const MAX_REVISIONS_PER_ARTIFACT = 10_000; +const MAX_TEXT_LENGTH = 16_384; +const MAX_DEPLOYMENT_REFERENCES = 1_000; + +export type RevisionProvenanceKind = "create" | "update" | "restore" | "migration"; + +export interface RevisionProvenanceV2 { + kind: RevisionProvenanceKind; + restoredFrom?: number; + legacyRevision?: number; + timestampSource?: "recorded" | "legacy-artifact" | "unknown"; +} + +export interface RevisionRecordV2 { + revision: number; + createdAt: string | null; + bytes: number; + contentHash: string; + pagePath: string; + title: string; + icon: string; + description?: string; + source?: string; + author?: string; + charts: number; + provenance: RevisionProvenanceV2; +} + +export interface DeploymentReferenceV2 { + capability: "public-static" | "authenticated" | "connector-capable"; + target: string; + url: string; + revision: number; + createdAt: string; +} + +export interface ArtifactRecordV2 { + id: string; + slug: string; + title: string; + icon: string; + description?: string; + source?: string; + author?: string; + createdAt: string | null; + updatedAt: string | null; + headRevision: number; + revisions: RevisionRecordV2[]; + charts: number; + bytes: number; + contentHash: string; + deploymentReferences: DeploymentReferenceV2[]; +} + +export interface ArtifactManifestV2 { + schemaVersion: 2; + artifacts: Record; + slugIndex: Record; +} + +export class ArtifactSchemaError extends Error { + readonly issues: string[]; + + constructor(issues: string[]) { + super(`artifact manifest schema validation failed: ${issues.join("; ")}`); + this.name = "ArtifactSchemaError"; + this.issues = [...issues]; + } +} + +export class ArtifactMigrationRequiredError extends Error { + constructor() { + super("artifact manifest uses a legacy schema; run lifecycle migration preflight before enabling schema 2"); + this.name = "ArtifactMigrationRequiredError"; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasOnlyKeys(value: Record, allowed: ReadonlySet): boolean { + return Object.keys(value).every((key) => allowed.has(key)); +} + +function boundedString(value: unknown, allowEmpty = false): value is string { + return ( + typeof value === "string" && + value.length <= MAX_TEXT_LENGTH && + (allowEmpty || value.length > 0) + ); +} + +function validTimestamp(value: unknown, nullable = false): value is string | null { + if (nullable && value === null) return true; + return ( + typeof value === "string" && + value.length <= 64 && + /(?:Z|[+-]\d{2}:\d{2})$/.test(value) && + !Number.isNaN(Date.parse(value)) + ); +} + +function positiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0; +} + +function nonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +const PROVENANCE_KEYS = new Set([ + "kind", + "restoredFrom", + "legacyRevision", + "timestampSource", +]); + +function validateProvenance(value: unknown, path: string, issues: string[]): void { + if (!isRecord(value) || !hasOnlyKeys(value, PROVENANCE_KEYS)) { + issues.push(`${path} must be an exact provenance object`); + return; + } + if ( + value["kind"] !== "create" && + value["kind"] !== "update" && + value["kind"] !== "restore" && + value["kind"] !== "migration" + ) { + issues.push(`${path}.kind is invalid`); + } + if (value["restoredFrom"] !== undefined && !positiveInteger(value["restoredFrom"])) { + issues.push(`${path}.restoredFrom must be a positive integer`); + } + if (value["legacyRevision"] !== undefined && !positiveInteger(value["legacyRevision"])) { + issues.push(`${path}.legacyRevision must be a positive integer`); + } + if ( + value["timestampSource"] !== undefined && + value["timestampSource"] !== "recorded" && + value["timestampSource"] !== "legacy-artifact" && + value["timestampSource"] !== "unknown" + ) { + issues.push(`${path}.timestampSource is invalid`); + } + if (value["kind"] === "restore" && !positiveInteger(value["restoredFrom"])) { + issues.push(`${path}.restoredFrom is required for restore provenance`); + } +} + +const REVISION_KEYS = new Set([ + "revision", + "createdAt", + "bytes", + "contentHash", + "pagePath", + "title", + "icon", + "description", + "source", + "author", + "charts", + "provenance", +]); + +function validateRevision( + value: unknown, + artifactId: string, + expectedRevision: number, + path: string, + issues: string[], +): void { + if (!isRecord(value) || !hasOnlyKeys(value, REVISION_KEYS)) { + issues.push(`${path} must be an exact revision object`); + return; + } + if (value["revision"] !== expectedRevision) { + issues.push(`${path}.revision must be contiguous and equal ${expectedRevision}`); + } + if (!validTimestamp(value["createdAt"], true)) issues.push(`${path}.createdAt is invalid`); + if (!nonNegativeInteger(value["bytes"])) issues.push(`${path}.bytes is invalid`); + if (typeof value["contentHash"] !== "string" || !CONTENT_HASH_RE.test(value["contentHash"])) { + issues.push(`${path}.contentHash must be a SHA-256 digest`); + } + if (value["pagePath"] !== `revisions/${artifactId}/${expectedRevision}.html`) { + issues.push(`${path}.pagePath is not canonical`); + } + for (const key of ["title", "icon"] as const) { + if (!boundedString(value[key])) issues.push(`${path}.${key} is invalid`); + } + for (const key of ["description", "source", "author"] as const) { + if (value[key] !== undefined && !boundedString(value[key], true)) { + issues.push(`${path}.${key} is invalid`); + } + } + if (!nonNegativeInteger(value["charts"])) issues.push(`${path}.charts is invalid`); + validateProvenance(value["provenance"], `${path}.provenance`, issues); +} + +const DEPLOYMENT_KEYS = new Set([ + "capability", + "target", + "url", + "revision", + "createdAt", +]); + +function validateDeployment(value: unknown, path: string, issues: string[]): void { + if (!isRecord(value) || !hasOnlyKeys(value, DEPLOYMENT_KEYS)) { + issues.push(`${path} must be an exact deployment reference`); + return; + } + if ( + value["capability"] !== "public-static" && + value["capability"] !== "authenticated" && + value["capability"] !== "connector-capable" + ) { + issues.push(`${path}.capability is invalid`); + } + if (!boundedString(value["target"])) issues.push(`${path}.target is invalid`); + if (!boundedString(value["url"])) { + issues.push(`${path}.url is invalid`); + } else { + try { + const url = new URL(value["url"]); + if (url.protocol !== "https:" && url.protocol !== "http:") { + issues.push(`${path}.url must use http or https`); + } + } catch { + issues.push(`${path}.url is invalid`); + } + } + if (!positiveInteger(value["revision"])) issues.push(`${path}.revision is invalid`); + if (!validTimestamp(value["createdAt"])) issues.push(`${path}.createdAt is invalid`); +} + +const ARTIFACT_KEYS = new Set([ + "id", + "slug", + "title", + "icon", + "description", + "source", + "author", + "createdAt", + "updatedAt", + "headRevision", + "revisions", + "charts", + "bytes", + "contentHash", + "deploymentReferences", +]); + +function validateArtifact(value: unknown, id: string, path: string, issues: string[]): void { + if (!isRecord(value) || !hasOnlyKeys(value, ARTIFACT_KEYS)) { + issues.push(`${path} must be an exact artifact object`); + return; + } + if (value["id"] !== id || !ARTIFACT_ID_RE.test(id)) issues.push(`${path}.id is invalid`); + if (typeof value["slug"] !== "string" || !ARTIFACT_SLUG_RE.test(value["slug"])) { + issues.push(`${path}.slug is invalid`); + } + for (const key of ["title", "icon"] as const) { + if (!boundedString(value[key])) issues.push(`${path}.${key} is invalid`); + } + for (const key of ["description", "source", "author"] as const) { + if (value[key] !== undefined && !boundedString(value[key], true)) { + issues.push(`${path}.${key} is invalid`); + } + } + if (!validTimestamp(value["createdAt"], true)) issues.push(`${path}.createdAt is invalid`); + if (!validTimestamp(value["updatedAt"], true)) issues.push(`${path}.updatedAt is invalid`); + if (!positiveInteger(value["headRevision"])) issues.push(`${path}.headRevision is invalid`); + if ( + !Array.isArray(value["revisions"]) || + value["revisions"].length === 0 || + value["revisions"].length > MAX_REVISIONS_PER_ARTIFACT + ) { + issues.push(`${path}.revisions must be a bounded non-empty array`); + } else { + for (let index = 0; index < value["revisions"].length; index++) { + validateRevision(value["revisions"][index], id, index + 1, `${path}.revisions[${index}]`, issues); + } + const head = value["revisions"].at(-1); + if (value["headRevision"] !== value["revisions"].length) { + issues.push(`${path}.headRevision must select the latest contiguous revision`); + } + if (isRecord(head)) { + for (const key of ["title", "icon", "description", "source", "author", "charts", "bytes", "contentHash"] as const) { + if (value[key] !== head[key]) issues.push(`${path}.${key} must match the head revision`); + } + if (value["updatedAt"] !== head["createdAt"]) { + issues.push(`${path}.updatedAt must match the head revision timestamp`); + } + } + } + if (!nonNegativeInteger(value["charts"])) issues.push(`${path}.charts is invalid`); + if (!nonNegativeInteger(value["bytes"])) issues.push(`${path}.bytes is invalid`); + if (typeof value["contentHash"] !== "string" || !CONTENT_HASH_RE.test(value["contentHash"])) { + issues.push(`${path}.contentHash is invalid`); + } + if ( + !Array.isArray(value["deploymentReferences"]) || + value["deploymentReferences"].length > MAX_DEPLOYMENT_REFERENCES + ) { + issues.push(`${path}.deploymentReferences is invalid`); + } else { + for (let index = 0; index < value["deploymentReferences"].length; index++) { + validateDeployment( + value["deploymentReferences"][index], + `${path}.deploymentReferences[${index}]`, + issues, + ); + } + } +} + +const MANIFEST_KEYS = new Set(["schemaVersion", "artifacts", "slugIndex"]); + +export function validateArtifactManifestV2(value: unknown): ArtifactManifestV2 { + const issues: string[] = []; + if (!isRecord(value) || !hasOnlyKeys(value, MANIFEST_KEYS)) { + throw new ArtifactSchemaError(["manifest must be an exact object"]); + } + if (value["schemaVersion"] !== ARTIFACT_MANIFEST_SCHEMA_VERSION) { + issues.push(`schemaVersion must equal ${ARTIFACT_MANIFEST_SCHEMA_VERSION}`); + } + if (!isRecord(value["artifacts"])) { + issues.push("artifacts must be an object"); + } else { + const entries = Object.entries(value["artifacts"]); + if (entries.length > MAX_ARTIFACTS) issues.push(`artifacts exceeds ${MAX_ARTIFACTS}`); + for (const [id, artifact] of entries) { + validateArtifact(artifact, id, `artifacts.${id}`, issues); + } + } + if (!isRecord(value["slugIndex"])) { + issues.push("slugIndex must be an object"); + } else if (isRecord(value["artifacts"])) { + const expected = new Map(); + for (const [id, artifact] of Object.entries(value["artifacts"])) { + if (isRecord(artifact) && typeof artifact["slug"] === "string") { + if (expected.has(artifact["slug"])) issues.push(`duplicate active slug ${artifact["slug"]}`); + expected.set(artifact["slug"], id); + } + } + for (const [slug, id] of Object.entries(value["slugIndex"])) { + if (!ARTIFACT_SLUG_RE.test(slug) || typeof id !== "string") { + issues.push(`slugIndex.${slug} is invalid`); + } else if (expected.get(slug) !== id) { + issues.push(`slugIndex.${slug} does not match artifact records`); + } + } + if (Object.keys(value["slugIndex"]).length !== expected.size) { + issues.push("slugIndex does not contain exactly one entry per artifact"); + } + } + if (issues.length > 0) throw new ArtifactSchemaError(issues); + return value as unknown as ArtifactManifestV2; +} + +export function emptyArtifactManifestV2(): ArtifactManifestV2 { + return { schemaVersion: ARTIFACT_MANIFEST_SCHEMA_VERSION, artifacts: {}, slugIndex: {} }; +} + +export function parseArtifactManifestV2(raw: string): ArtifactManifestV2 { + let value: unknown; + try { + value = JSON.parse(raw) as unknown; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new ArtifactSchemaError([`manifest is not valid JSON: ${detail}`]); + } + if (isRecord(value) && value["schemaVersion"] === ARTIFACT_MANIFEST_SCHEMA_VERSION) { + return validateArtifactManifestV2(value); + } + if (isRecord(value) && typeof value["schemaVersion"] === "number") { + throw new ArtifactSchemaError([`unsupported future or unknown schemaVersion ${String(value["schemaVersion"])}`]); + } + throw new ArtifactMigrationRequiredError(); +} + +export async function readArtifactManifestV2(dir: string): Promise { + await recoverFileTransactions(dir); + const manifestPath = join(dir, ARTIFACT_MANIFEST_FILE); + let raw: string; + try { + const info = await lstat(manifestPath); + if (info.isSymbolicLink() || !info.isFile()) { + throw new ArtifactSchemaError(["manifest path must be a regular file"]); + } + raw = await readFile(manifestPath, "utf8"); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + return emptyArtifactManifestV2(); + } + throw error; + } + return parseArtifactManifestV2(raw); +} diff --git a/src/file-transaction.ts b/src/file-transaction.ts index f6928d4..cf2f2b3 100644 --- a/src/file-transaction.ts +++ b/src/file-transaction.ts @@ -44,7 +44,7 @@ export interface FileTransactionOptions { } export interface FileTransactionContext { - commit(files: ReadonlyMap): Promise; + commit(files: ReadonlyMap): Promise; } interface LockOwner { @@ -61,7 +61,7 @@ interface JournalTarget { path: string; existed: boolean; oldHash: string | null; - newHash: string; + newHash: string | null; bytes: number; } @@ -187,8 +187,8 @@ function parseJournal(value: unknown): TransactionJournal | undefined { typeof target["path"] !== "string" || typeof target["existed"] !== "boolean" || (target["oldHash"] !== null && typeof target["oldHash"] !== "string") || - typeof target["newHash"] !== "string" || - !SHA256_RE.test(target["newHash"]) || + (target["newHash"] !== null && + (typeof target["newHash"] !== "string" || !SHA256_RE.test(target["newHash"]))) || typeof target["bytes"] !== "number" || !Number.isSafeInteger(target["bytes"]) || target["bytes"] < 0 || @@ -448,31 +448,32 @@ async function targetHash(path: string): Promise { } } +function matchesSelectedHash(actual: string | undefined, expected: string | null): boolean { + return expected === null ? actual === undefined : actual === expected; +} + async function restorePrepared(root: string, transactionPath: string, journal: TransactionJournal): Promise { for (const target of journal.targets) { const destination = await containedPath(root, target.path); const backup = join(transactionPath, "old", ...safeSegments(target.path)); const current = await targetHash(destination); - if (current === target.newHash) { - if (!target.existed) { - await rm(destination, { force: true }); - } else if (await exists(backup)) { + if (target.existed) { + if (current === target.oldHash) continue; + if (await exists(backup)) { await mkdir(dirname(destination), { recursive: true }); + if (current !== undefined) await rm(destination, { force: true }); await rename(backup, destination); await syncDirectory(dirname(backup)); await syncDirectory(dirname(destination)); } else { throw new TransactionRecoveryError(journal.id, `old bytes missing for ${target.path}`); } - } else if (target.existed && current !== target.oldHash) { - if (await exists(backup)) { - await mkdir(dirname(destination), { recursive: true }); - await rename(backup, destination); - await syncDirectory(dirname(backup)); - await syncDirectory(dirname(destination)); - } else { - throw new TransactionRecoveryError(journal.id, `unexpected old bytes for ${target.path}`); + } else if (current !== undefined) { + if (!matchesSelectedHash(current, target.newHash)) { + throw new TransactionRecoveryError(journal.id, `unexpected new bytes for ${target.path}`); } + await rm(destination, { force: true }); + await syncDirectory(dirname(destination)); } const restored = await targetHash(destination); if ((target.existed && restored !== target.oldHash) || (!target.existed && restored !== undefined)) { @@ -489,8 +490,11 @@ async function rollForward(root: string, transactionPath: string, journal: Trans const staged = join(transactionPath, "new", ...safeSegments(target.path)); const backup = join(transactionPath, "old", ...safeSegments(target.path)); const current = await targetHash(destination); - if (current === target.newHash) continue; - if (!(await exists(staged)) || (await targetHash(staged)) !== target.newHash) { + if (matchesSelectedHash(current, target.newHash)) continue; + if ( + target.newHash !== null && + (!(await exists(staged)) || (await targetHash(staged)) !== target.newHash) + ) { throw new TransactionRecoveryError(journal.id, `staged bytes missing for ${target.path}`); } if (current !== undefined) { @@ -503,15 +507,17 @@ async function rollForward(root: string, transactionPath: string, journal: Trans await syncDirectory(dirname(backup)); await syncDirectory(dirname(destination)); } - await mkdir(dirname(destination), { recursive: true }); - await rename(staged, destination); - await syncFile(destination); - await syncDirectory(dirname(staged)); - await syncDirectory(dirname(destination)); + if (target.newHash !== null) { + await mkdir(dirname(destination), { recursive: true }); + await rename(staged, destination); + await syncFile(destination); + await syncDirectory(dirname(staged)); + await syncDirectory(dirname(destination)); + } } for (const target of journal.targets) { const destination = await containedPath(root, target.path); - if ((await targetHash(destination)) !== target.newHash) { + if (!matchesSelectedHash(await targetHash(destination), target.newHash)) { throw new TransactionRecoveryError(journal.id, `commit verification failed for ${target.path}`); } } @@ -558,7 +564,7 @@ async function recoverHeld(root: string): Promise { async function commitHeld( root: string, lock: HeldLock, - files: ReadonlyMap, + files: ReadonlyMap, options: FileTransactionOptions, ): Promise { if (files.size === 0) throw new Error("artifact transaction has no target files"); @@ -566,9 +572,19 @@ async function commitHeld( throw new Error(`artifact transaction has ${files.size} targets; limit is ${MAX_TRANSACTION_TARGETS}`); } const normalized = [...files.entries()] - .map(([path, value]) => [path, typeof value === "string" ? Buffer.from(value, "utf8") : Buffer.from(value)] as const) + .map( + ([path, value]) => + [ + path, + value === null + ? null + : typeof value === "string" + ? Buffer.from(value, "utf8") + : Buffer.from(value), + ] as const, + ) .sort(([left], [right]) => left.localeCompare(right)); - const totalBytes = normalized.reduce((sum, [, value]) => sum + value.byteLength, 0); + const totalBytes = normalized.reduce((sum, [, value]) => sum + (value?.byteLength ?? 0), 0); if (totalBytes > MAX_TRANSACTION_BYTES) { throw new Error(`artifact transaction has ${totalBytes} bytes; limit is ${MAX_TRANSACTION_BYTES}`); } @@ -593,13 +609,15 @@ async function commitHeld( throw error; }, ); - await writeDurable(join(transactionPath, "new", ...safeSegments(path)), value); + if (value !== null) { + await writeDurable(join(transactionPath, "new", ...safeSegments(path)), value); + } targets.push({ path, existed: current !== undefined, oldHash: current === undefined ? null : hashBytes(current), - newHash: hashBytes(value), - bytes: value.byteLength, + newHash: value === null ? null : hashBytes(value), + bytes: value?.byteLength ?? 0, }); options.fault?.("target-staged", path); } @@ -631,17 +649,19 @@ async function commitHeld( await syncDirectory(dirname(destination)); options.fault?.("target-backed-up", target.path); } - await mkdir(dirname(destination), { recursive: true }); - await rename(staged, destination); - await syncFile(destination); - await syncDirectory(dirname(staged)); - await syncDirectory(dirname(destination)); + if (target.newHash !== null) { + await mkdir(dirname(destination), { recursive: true }); + await rename(staged, destination); + await syncFile(destination); + await syncDirectory(dirname(staged)); + await syncDirectory(dirname(destination)); + } options.fault?.("target-replaced", target.path); } for (const target of targets) { const destination = await containedPath(root, target.path); - if ((await targetHash(destination)) !== target.newHash) { + if (!matchesSelectedHash(await targetHash(destination), target.newHash)) { throw new TransactionRecoveryError(id, `verification failed for ${target.path}`); } } @@ -658,7 +678,7 @@ async function commitHeld( (await Promise.all( targets.map(async (target) => { const destination = await containedPath(root, target.path); - return (await targetHash(destination)) === target.newHash; + return matchesSelectedHash(await targetHash(destination), target.newHash); }), )).every(Boolean); if (selectedNew) return; diff --git a/src/github-pages.ts b/src/github-pages.ts index b9b7857..98f2cc8 100644 --- a/src/github-pages.ts +++ b/src/github-pages.ts @@ -30,7 +30,15 @@ export function pagesBaseUrl(repo: string): string { return `https://${owner}.github.io/${name}/`; } -const SKIP_ENTRIES = new Set([".git", ".state", ".db", ".datasources", ".transactions"]); +const SKIP_ENTRIES = new Set([ + ".git", + ".state", + ".db", + ".datasources", + ".transactions", + ".backups", + ".migrations", +]); export async function copyArtifacts(fromDir: string, toDir: string): Promise { await mkdir(toDir, { recursive: true }); diff --git a/src/publisher.ts b/src/publisher.ts index 69694e8..7934ad7 100644 --- a/src/publisher.ts +++ b/src/publisher.ts @@ -1,5 +1,5 @@ -import { readFile, readdir } from "node:fs/promises"; -import { createHash } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; import { join } from "node:path"; import { renderGallery } from "./gallery.ts"; import { @@ -7,6 +7,16 @@ import { runFileTransaction, type FileTransactionContext, } from "./file-transaction.ts"; +import { + ARTIFACT_ID_RE, + ARTIFACT_MANIFEST_FILE, + emptyArtifactManifestV2, + parseArtifactManifestV2, + validateArtifactManifestV2, + type ArtifactManifestV2, + type ArtifactRecordV2, + type RevisionRecordV2, +} from "./artifact-schema.ts"; import { ArtifactTooLargeError, DEFAULT_MAX_BYTES, @@ -20,6 +30,10 @@ export function contentHash(html: string): string { return createHash("sha256").update(html, "utf8").digest("hex").slice(0, 12); } +export function fullContentHash(html: string): string { + return createHash("sha256").update(html, "utf8").digest("hex"); +} + export class StaleArtifactError extends Error { readonly currentHash: string; constructor(slug: string, currentHash: string) { @@ -72,6 +86,11 @@ export interface Publisher { publish(input: PublishInput): Promise; } +export interface FilePublisherOptions { + schemaVersion?: 1 | 2; + artifactIdFactory?: () => string; +} + const MANIFEST_FILE = "manifest.json"; const GALLERY_FILE = "index.html"; @@ -87,6 +106,48 @@ async function readManifest(dir: string): Promise { } } +async function readManifestV2Locked(dir: string): Promise { + const path = join(dir, ARTIFACT_MANIFEST_FILE); + try { + const info = await lstat(path); + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error("artifact manifest path must be a regular file"); + } + return parseArtifactManifestV2(await readFile(path, "utf8")); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + return emptyArtifactManifestV2(); + } + throw error; + } +} + +function manifestV2GalleryView(manifest: ArtifactManifestV2): Manifest { + const artifacts: Record = {}; + for (const artifact of Object.values(manifest.artifacts)) { + artifacts[artifact.slug] = { + slug: artifact.slug, + title: artifact.title, + icon: artifact.icon, + description: artifact.description, + source: artifact.source, + createdAt: artifact.createdAt ?? "unknown", + updatedAt: artifact.updatedAt ?? "unknown", + current: artifact.headRevision, + versions: artifact.revisions.map((revision) => revision.revision), + charts: artifact.charts, + bytes: artifact.bytes, + hash: artifact.contentHash.slice(0, 12), + }; + } + return { artifacts }; +} + function footerHtml(meta: ArtifactMeta): string { return [ '