diff --git a/.github/workflows/consolidation-gates.yml b/.github/workflows/consolidation-gates.yml index 3f42414e..351b898e 100644 --- a/.github/workflows/consolidation-gates.yml +++ b/.github/workflows/consolidation-gates.yml @@ -54,6 +54,14 @@ jobs: rust-toolchain.toml \ crates \ fixtures \ + packages/grida-reftest/svg-assertions \ + packages/grida-reftest/package.json \ + packages/grida-reftest/tsconfig.json \ + packages/grida-reftest/vitest.config.ts \ + package.json \ + pnpm-lock.yaml \ + pnpm-workspace.yaml \ + .nvmrc \ third_party; then engine=true fi @@ -87,7 +95,11 @@ jobs: needs: scope if: needs.scope.outputs.engine == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + # Existing architecture budget plus the pilot's bounded cold CLI build. + timeout-minutes: 65 + env: + # Reports are immutable observations, not reusable Cargo build outputs. + SVG_ASSERTION_DIR: svg-assertion-pilot-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Checkout head revision uses: actions/checkout@v4 @@ -133,6 +145,9 @@ jobs: uses: Swatinem/rust-cache@v2 with: shared-key: consolidation-seams + # Seed a cache containing the new dev-profile CLI build, not only + # the prior test-profile artifacts restored by the old immutable key. + key: svg-assertion-pilot-v1 cache-all-crates: true cache-workspace-crates: true cache-on-failure: true @@ -159,6 +174,40 @@ jobs: cargo test --locked -p rframe --test architecture cargo test --locked -p n0 --test architecture + - name: Setup pnpm for the SVG assertion pilot + uses: pnpm/action-setup@v4 + + - name: Setup Node for the SVG assertion pilot + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: pnpm + + - name: Install the pinned assertion-tool dependencies and Chromium + run: | + pnpm install --frozen-lockfile + pnpm -C packages/grida-reftest exec playwright install --with-deps chromium + + - name: Verify assertion contracts and the actual n0 CLI pilot + # The registry crate uses its packaged VCS identity for binary downloads. + # FORCE_SKIA_BINARIES_DOWNLOAD selects an upstream-checkout test path + # which instead builds Skia from source when no Git checkout exists. + run: | + pnpm -C packages/grida-reftest exec tsc --noEmit -p svg-assertions/tsconfig.json + pnpm -C packages/grida-reftest exec vitest run svg-assertions --passWithNoTests=false + pnpm -C packages/grida-reftest exec tsx svg-assertions/cli.ts \ + --manifest "$GITHUB_WORKSPACE/packages/grida-reftest/svg-assertions/pilot.json" \ + --out "$RUNNER_TEMP/$SVG_ASSERTION_DIR" + + - name: Upload SVG assertion observations + if: always() + uses: actions/upload-artifact@v4 + with: + name: svg-assertion-pilot-${{ github.run_id }} + path: ${{ runner.temp }}/${{ env.SVG_ASSERTION_DIR }} + if-no-files-found: warn + retention-days: 7 + legacy_pixel_sweep: name: legacy pixel sweep needs: scope diff --git a/crates/n0_cli/README.md b/crates/n0_cli/README.md index f430f9aa..783e05d5 100644 --- a/crates/n0_cli/README.md +++ b/crates/n0_cli/README.md @@ -607,7 +607,7 @@ cargo run -p n0_cli --bin n0 -- \ The filter estate contains 26 chassis/blur cells, 60 shadow-graph, 28 native drop-shadow, 27 color-matrix, 34 component-transfer, 38 blend, 37 morphology, 91 turbulence/displacement, 41 convolution-rung, and 71 diffuse-lighting - cells. The complete primitive corpus contains 1,537 Chromium-baked cells plus + cells. The complete primitive corpus contains 1,539 Chromium-baked cells plus 16 sampled frames; the text estate contains sixteen exact text pixel cells and eight exact-number artifact-geometry witnesses (six Allerta and two Bungee), and the named refusal register has 351 rows. `feFlood`, `feComposite`, diff --git a/docs/wg/consolidation/svg-engine-of-record.md b/docs/wg/consolidation/svg-engine-of-record.md index 9ab9a2c2..afbfcaf0 100644 --- a/docs/wg/consolidation/svg-engine-of-record.md +++ b/docs/wg/consolidation/svg-engine-of-record.md @@ -119,7 +119,7 @@ from the dated addenda below: carrying admitted repeating-pattern paint and admitted source/target filter composition. `crates/n0_cli/README.md` is the statement of record. -- **The corpus** is 1,537 Chromium-baked primitive cells plus 16 sampled frames, +- **The corpus** is 1,539 Chromium-baked primitive cells plus 16 sampled frames, with a separate sixteen-cell exact text suite whose current cells select hash-pinned Ahem and Ahem-derived bytes from explicit family/face environments, and eight exact-number artifact-geometry @@ -6523,3 +6523,67 @@ The reproducer renewed the captures and 284 CLI executions under one fixed binary, verified old records and oracle bytes, and reran the full gate and status without further diff. Full relevant tests, trace execution/pixels, formatting, strict Clippy and link/OSS checks pass. + +## 2026-09-09 — assertion-driven SVG harness pilot + +The owner-approved [assertion proposal](https://github.com/gridaco/nothing/issues/140) +adds an observation and regression instrument, not a rendering capability or a +new support inventory. Its [contract and setup](../../../packages/grida-reftest/svg-assertions/README.md) +separate input identity, a described claim, renderer observations, the chosen +reference, and a discrete verdict. A named-refusal assertion cannot be read as +rendering support; a required unresolved/missing case cannot become a passing +gate. Image differences remain diagnostic evidence, never a similarity score. + +The pilot imports the unchanged resvg-test-suite sharp-rectangle source and a +separate color-control derivative through the existing primitive workflow, +with upstream identity and MIT notice retained. Each is exact to its own +Chromium oracle at a 500×500 initial viewport, in both actual CLI admissions. +Changing only the subject fill changes 160,000 Chromium pixels at maximum +channel delta 255, while the decorative frame and underlay remain unchanged. +The existing geometry-unit refusal witness supplies the separate refusal +assertion; it is not promoted or duplicated in the refusal register. + +The complete primitive corpus now contains 1,539 cells. The 351 refusal rows, +16 sampled frames, 16 exact text cells and eight geometry witnesses are +unchanged. No checklist row ticks, existing oracle changes, or new tolerance +accompanies this instrument. The pinned capture remains the single Chromium +module, shared by probes, bakes and this runner. Stored upstream PNGs and fresh +resvg outputs are independent observations, never alternate ways to satisfy a +declared Chromium assertion. FLIP and legacy-tool retirement remain separate. + +For sensitivity, temporarily replacing CSS green with blue at the n0 solid +paint boundary fails the existing `just gate` on the new upstream rectangle +with 160,000 differing pixels at maximum delta 255. The assertion pilot also +fails that rectangle while its blue control and expected-refusal case remain +passing. The renderer source is restored exactly; no painter change is part +of the tool. The earlier scratch survey remains **measured, not celled** +outside these two promoted cases and does not establish a corpus-wide verdict. + +Independent review caught an image-decoder contract mismatch before landing: +the pinned decoder erases hidden RGB for non-palette `tRNS` and accepts a +replacement PNG header after the initial dimension check. The harness now +validates bounded chunk framing, CRCs and one leading header before allocation, +then strict bounded decompression and the exact scanline extent before pixels; +encodings that cannot preserve this profile's exact RGBA8 meaning explicitly +refuse. Palette transparency remains lossless and tested. Decoded buffers are +case-local, repeats retain hashes, and controls reload one verified pair at a +time. Nonregular file inputs refuse without blocking on a FIFO writer. The +required contract-test invocation fails when no tests are discovered. +These are instrument protections, not changes to engine semantics or oracles. + +The independent reproducer renewed the scratch control, all existing fixture +gates and STATUS, then ran the pilot against both admissions with fresh +Chromium and separately pinned resvg/stored observations. Both rendering +assertions and the named-refusal assertion pass, with tool/input identities +unchanged. Full relevant Rust tests, formatting, strict Clippy, 40 tool-contract +tests, typechecks and link/OSS checks pass. No Workflow runner was exposed: +TICK/LAW and REPRO ran as manual independent roles. Two TICK/LAW audits produced +the fixes above; a final audit attempt was blocked by a tool-policy error. +The fixes have local regression checks and independent REPRO evidence, +but no final independent TICK/LAW approval is claimed. + +PR review additionally guards partial Chromium output without losing the +report: missing repeat zero keeps its empty slot instead of promoting repeat +one, comparisons check for decoded pixels, and capture failures retain signal +or exit-code causes even with empty stderr. Regressions exercise those branches; +the renewed actual-CLI pilot remains exact after the changes. diff --git a/docs/wg/consolidation/web-checklist.md b/docs/wg/consolidation/web-checklist.md index c6a28f3f..685e9ac7 100644 --- a/docs/wg/consolidation/web-checklist.md +++ b/docs/wg/consolidation/web-checklist.md @@ -30,6 +30,11 @@ in [the statement of record](../../../crates/n0_cli/README.md), rung history in [the D-N register](./svg-engine-of-record.md), and no score is computed from this list (FLIP is unratified). +The [SVG assertion harness](../../../packages/grida-reftest/svg-assertions/README.md) +distinguishes described rendering assertions, named refusals, and unresolved +observations. None of those individual results substitutes for this checklist's +full-grammar tick rule. + ## HTML diff --git a/fixtures/web-first/LICENSE.resvg b/fixtures/web-first/LICENSE.resvg new file mode 100644 index 00000000..715586c3 --- /dev/null +++ b/fixtures/web-first/LICENSE.resvg @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Reizner Evgeniy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/fixtures/web-first/README.md b/fixtures/web-first/README.md index 5eda748b..45903ddd 100644 --- a/fixtures/web-first/README.md +++ b/fixtures/web-first/README.md @@ -19,7 +19,7 @@ Every root primitive here is a closed enumeration in `primitives.json` with a committed Chromium oracle beside it. Text follows the ratified corpus-growth law in its own closed [text estate](./text/README.md): sixteen exact text cells and eight exact-number real-font artifact-geometry witnesses. The current evidence -estate is 1,537 primitive cells plus 16 sampled frames, those twenty-four text +estate is 1,539 primitive cells plus 16 sampled frames, those twenty-four text witnesses, and 351 named refusal rows. Pixel cells require byte equality, except only the primitive rows carrying an explicit measured tolerance block. The real-font witness grades geometry before rasterization and makes no Chromium @@ -35,6 +35,7 @@ separates source membership from visible paint and records the bounded profile. | File | Role | | --- | --- | +| `svg-upstream-resvg-rect-simple-case.svg` · `svg-upstream-resvg-rect-blue-control.svg` | Assertion-harness pilot: the unchanged upstream sharp rectangle and a separate subject-fill color control, captured at a 500×500 initial viewport. Both are exact in strict and best-effort CLI renders; changing only green to blue changes 160,000 Chromium pixels at maximum channel delta 255. Imported from resvg-test-suite revision `d8e064337faf01bc5a9579187a56dbdbe3eacc72`, `tests/shapes/rect/simple-case.svg`, under [its MIT notice](./LICENSE.resvg). These are scene assertions, not a new feature grant. | | `svg-group-blend-sibling-{multiply,screen}-{transparent,fill-zero,fill-alpha,empty-server,invalid-fallback}.svg` | Ten selected-paint sibling cells: transparent color, zero fill opacity, zero color alpha, a valid stopless linear server and a missing server with transparent fallback retain source contribution without visible fill. | | `svg-group-blend-sibling-{multiply,screen}-{stroke-only,stroke-zero,stroke-thin,stroke-px,stroke-percent,stroke-dash,solid-stroke}.svg` | Fourteen decorated-rectangle cells: omitted transparent/zero-alpha solid strokes, width spellings and inert dash/join state retain local source enclosure, including beside a solid-filled sibling. | | `svg-group-blend-sibling-{multiply,screen}-{boundary,edge,reversed,multiple,two-ramps}.svg` | Ten enclosure and composition controls: fractional boundaries, offscreen edges, sibling order, multiple paintless contributors and two live ramps. | @@ -482,3 +483,12 @@ stale one. Probe *matrices* stay scratch and are never committed; a probe is a question asked once, and what it proves lands as cells and README rows, not as a shadow corpus. The pre-landing verification ritual is the saved `verify-rung` workflow (`.agents/workflows/verify-rung.js`). + +`assertions-test` checks the separate assertion-tool contracts; +`assertions-gate ` runs its small actual-CLI pilot +and fresh Chromium captures. The [SVG assertion tool](../../packages/grida-reftest/svg-assertions/README.md) +records each described claim, reference decision, both admissions and their +diagnostics, independent exact image comparisons, and a discrete verdict. A +verified refusal is not rendering support; unresolved or missing required +cases cannot silently pass. This supplements the existing corpus gates and +uses the same capture module. It does not create a score or a second checklist. diff --git a/fixtures/web-first/STATUS.md b/fixtures/web-first/STATUS.md index eb6670be..c13988d5 100644 --- a/fixtures/web-first/STATUS.md +++ b/fixtures/web-first/STATUS.md @@ -19,7 +19,7 @@ Not a conformance claim: no score is computed or implied (FLIP is unratified), and the corpus enumerates constructs, not the SVG surface. -## Chromium-baked cells (1537) +## Chromium-baked cells (1539) Cells are checked against their committed Chromium oracles using exact bytes unless a manifest entry declares a measured, bounded @@ -1490,6 +1490,8 @@ to its fixture source. No new image is committed for this view. svg-transform-no-separator svg-transform-runtogether svg-translucent-fill-rgba +svg-upstream-resvg-rect-blue-control +svg-upstream-resvg-rect-simple-case svg-use svg-use-ancestor-circle svg-use-chain diff --git a/fixtures/web-first/chromium/svg-upstream-resvg-rect-blue-control.png b/fixtures/web-first/chromium/svg-upstream-resvg-rect-blue-control.png new file mode 100644 index 00000000..0d6fe841 Binary files /dev/null and b/fixtures/web-first/chromium/svg-upstream-resvg-rect-blue-control.png differ diff --git a/fixtures/web-first/chromium/svg-upstream-resvg-rect-simple-case.png b/fixtures/web-first/chromium/svg-upstream-resvg-rect-simple-case.png new file mode 100644 index 00000000..ff64c48d Binary files /dev/null and b/fixtures/web-first/chromium/svg-upstream-resvg-rect-simple-case.png differ diff --git a/fixtures/web-first/justfile b/fixtures/web-first/justfile index d4230964..ea2c5d0b 100644 --- a/fixtures/web-first/justfile +++ b/fixtures/web-first/justfile @@ -59,3 +59,13 @@ status: # entry is `standalone-svg` or `html-inline-svg`. add id source entry="standalone-svg": python3 {{justfile_directory()}}/add_cell.py {{id}} --source {{source}} --entry {{entry}} + +# Assertion-tool contracts and deliberately failing synthetic controls; no scores. +assertions-test: + {{node_env}} pnpm -C {{justfile_directory()}}/../../packages/grida-reftest exec tsc --noEmit -p svg-assertions/tsconfig.json + {{node_env}} pnpm -C {{justfile_directory()}}/../../packages/grida-reftest exec vitest run svg-assertions --passWithNoTests=false + +# Actual CLI pilot + fresh pinned Chromium, with a new absolute output directory. +# This supplements, and does not replace, the existing full fixture gate. +assertions-gate out: + {{node_env}} {{cargo_env}} pnpm -C {{justfile_directory()}}/../../packages/grida-reftest exec tsx {{justfile_directory()}}/../../packages/grida-reftest/svg-assertions/cli.ts --manifest {{justfile_directory()}}/../../packages/grida-reftest/svg-assertions/pilot.json --out "{{out}}" diff --git a/fixtures/web-first/oracle-bake.json b/fixtures/web-first/oracle-bake.json index edb4690b..f2ce8c04 100644 --- a/fixtures/web-first/oracle-bake.json +++ b/fixtures/web-first/oracle-bake.json @@ -5,7 +5,7 @@ "bake_script_sha256": "2bdb5f933d072a1e87c9a675c3342fcf506c0955c0f5c26e2988f4e8fa37c4f2", "capture_module_sha256": "069296201718c43d29efe356fbea893781b73250dca51de3b6d47468d74027b0", "suite": "primitives.json", - "suite_sha256": "afc67c1f35b4289366d72acd150b813b91e79dc4de9b4aba39dd7fad8286b652", + "suite_sha256": "939969dd01cd520c8f9b41377d11980d5454f6161158a5326bb84c1874ef65dd", "capture": { "device_scale_factor": 1, "omit_background": true, @@ -13186,6 +13186,24 @@ "width": 64, "height": 64 }, + { + "id": "svg-upstream-resvg-rect-blue-control", + "source": "svg-upstream-resvg-rect-blue-control.svg", + "source_sha256": "18722a00cf6f1b0bfb417dbca3c4a59237b628150f5bebaff467cde54c386b14", + "oracle": "chromium/svg-upstream-resvg-rect-blue-control.png", + "oracle_sha256": "6bdaceeba4a8327ece87bea27a7819434563957e8cbcdf0d4c93648aee04460a", + "width": 500, + "height": 500 + }, + { + "id": "svg-upstream-resvg-rect-simple-case", + "source": "svg-upstream-resvg-rect-simple-case.svg", + "source_sha256": "0b487592d89c6e43573eb91eb92cb18b85bd5139c31143f534ba3e43a9dfd0cc", + "oracle": "chromium/svg-upstream-resvg-rect-simple-case.png", + "oracle_sha256": "4fd110181e80bc6c24e6d9cc2d273ad477d3c82c5fb57bef0fb871f4d08a063c", + "width": 500, + "height": 500 + }, { "id": "svg-use", "source": "svg-use.svg", diff --git a/fixtures/web-first/primitives.json b/fixtures/web-first/primitives.json index 1e416a51..bce961ce 100644 --- a/fixtures/web-first/primitives.json +++ b/fixtures/web-first/primitives.json @@ -11856,6 +11856,22 @@ "width": 64, "height": 64 }, + { + "id": "svg-upstream-resvg-rect-blue-control", + "source": "svg-upstream-resvg-rect-blue-control.svg", + "entry": "standalone-svg", + "oracle": "chromium/svg-upstream-resvg-rect-blue-control.png", + "width": 500, + "height": 500 + }, + { + "id": "svg-upstream-resvg-rect-simple-case", + "source": "svg-upstream-resvg-rect-simple-case.svg", + "entry": "standalone-svg", + "oracle": "chromium/svg-upstream-resvg-rect-simple-case.png", + "width": 500, + "height": 500 + }, { "id": "svg-use", "source": "svg-use.svg", diff --git a/fixtures/web-first/svg-upstream-resvg-rect-blue-control.svg b/fixtures/web-first/svg-upstream-resvg-rect-blue-control.svg new file mode 100644 index 00000000..527f09ce --- /dev/null +++ b/fixtures/web-first/svg-upstream-resvg-rect-blue-control.svg @@ -0,0 +1,11 @@ + + Simple case + + + + + + + + + diff --git a/fixtures/web-first/svg-upstream-resvg-rect-simple-case.svg b/fixtures/web-first/svg-upstream-resvg-rect-simple-case.svg new file mode 100644 index 00000000..c34a6b2e --- /dev/null +++ b/fixtures/web-first/svg-upstream-resvg-rect-simple-case.svg @@ -0,0 +1,11 @@ + + Simple case + + + + + + + + + diff --git a/packages/grida-reftest/svg-assertions/README.md b/packages/grida-reftest/svg-assertions/README.md new file mode 100644 index 00000000..c5050c3f --- /dev/null +++ b/packages/grida-reftest/svg-assertions/README.md @@ -0,0 +1,205 @@ +# SVG assertions + +A developer harness for **one described claim, one discrete verdict**. It runs +the actual n0 CLI in strict and best-effort modes, captures pinned Chromium, +and preserves separately named image comparisons and diagnostics. No image +similarity score, pass-rate percentage, or SVG completeness estimate exists in +this tool. The reviewed proposal is +[gridaco/nothing#140](https://github.com/gridaco/nothing/issues/140). + +This directory uses the package's installed dependencies only. It is not an +exported SDK or part of the legacy `reftest` command. It imports no legacy +scoring, renderer selection, or automatic reference-selection code. + +## Run the pilot + +From the repository root, use the normal [engine setup](../../../docs/contributing/setup.md) +and the Node/pnpm versions pinned by the repository, then: + +```sh +pnpm install --frozen-lockfile +pnpm -C packages/grida-reftest exec playwright install chromium +just -f fixtures/web-first/justfile assertions-test +mkdir -p target +just -f fixtures/web-first/justfile assertions-gate "$PWD/target/svg-assertion-pilot" +``` + +The output directory must **not already exist**; its parent must exist. Each +new run needs a new name. Nothing is blessed, overwritten, or deleted. Open +`index.html` in that output directory for the images and explanations; +`report.json` includes commands, diagnostics, both repeats, input/tool hashes, +and every available named comparison. Output artifacts are local or CI review +artifacts, not a new committed corpus. + +The pilot requires no downloaded resvg corpus, system font, external resource, +or resvg executable. It points to two ordinary registered Web-first cells and +one existing refusal witness. Their original fixture gates still apply. + +The two positive cells use the unchanged +`tests/shapes/rect/simple-case.svg` from +[resvg-test-suite](https://github.com/linebender/resvg-test-suite), revision +`d8e064337faf01bc5a9579187a56dbdbe3eacc72`, and a separately identified derivative +changing only the subject's `green` fill to `blue`. The +[MIT notice](../../../fixtures/web-first/LICENSE.resvg) applies to both. +Both render at a 500×500 initial viewport; their `viewBox` remains untouched. +The fresh Chromium color pair must differ. Each complete output must match its +own committed Chromium reference, not the other color's image. The refusal +witness checks the exact current CLI declarations for geometry units; a pass +there is labelled **expected refusal**, never rendering support. + +## A case is not its filename + +```text +Input identity + declared environment + described claim + | + Renderer observations + | + Independent named comparisons + | + Reviewed executable assertion + | + PASS / FAIL / UNRESOLVED +``` + +[pilot.json](./pilot.json) is a complete manifest example. All fields are +mandatory, including explicit `null` where an optional reference or assertion +does not exist. Unknown fields, duplicate IDs, missing controls, invalid hashes, +and empty suites are errors. Paths are relative to the manifest, not the shell. +Source bytes and any baked/stored PNG are SHA-256 pinned. The manifest and tool +identities are copied into the observation record and verified against changes +during the run. Sources are copied into a fresh run directory before rendering; +the authored input is never rewritten to match another renderer's export. + +- `description` states the specific claim; it may be `null` only without an + assertion. Tags or descriptions are not painted into the SVG. +- `review` states whether someone reviewed the source and its environment, why, + and any explicit blockers. This is a recorded review decision, **not an + automatic SVG feature detector**. Presence of an element is insufficient. +- `required` controls CI obligation. It is not a support label. +- `assertion: null` produces an observation only. It cannot satisfy a required + gate even if all pictures happen to match. +- `render-exact` selects a pinned Chromium reference and a different same-size + Chromium control case with a stated reason. A control must be reviewed, + repeatable and active. A filename-only placeholder cannot satisfy this check. +- `refusal` declares strict/best-effort exit codes and the exact expected + diagnostic text. The success footer alone is removed from comparison; raw + stdout/stderr remain recorded. A missing or additional diagnostic fails. +- `stored` is an optional historical second-opinion PNG. It is never an + alternate route to passing the declared Chromium assertion. + +Manifests and SVG inputs are bounded to 1 MiB each; SVG must be UTF-8. +Canvases are bounded to 1..2048 pixels per axis, and suites to 1..128 cases +per manifest. The initial profile is **static self-contained SVG without text, +external resources, or unresolved export sizing**. The review must establish +that profile; the harness does not add a parser, matcher, font resolver, or I/O +policy to make an ineligible case fit. Declare the limitation in `blockers`. +Chromium's failed resource requests are also retained as capture failures. + +## Read results correctly + +| Result | Meaning | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `PASS` / `render-exact` | Both CLI admissions render without degradation and match the declared reference; repeats and fresh Chromium agree; the control changes pixels. | +| `PASS` / `refusal` | The exact named refusal/degradation contract was verified. No rendering-support claim. | +| `FAIL` | The declared assertion was violated, including unexpected output, diagnostics, refusal, or unstable CLI rendering. | +| `UNRESOLVED` | The input/reference/environment cannot support a judgment, or the claimed control is not discriminating. Not a hidden zero or pass. | +| `OBSERVATION` | No executable assertion was declared. Raw comparisons are evidence only. | + +The normal command exits nonzero unless all required assertions pass and every +requested case is accounted for, with no run-integrity failure. A required case +that cannot run cannot silently disappear. No-required-case manifests do not +certify a gate. `--observe` explicitly runs a non-gating exploration; it still +fails on run-integrity errors, and labels the invocation as observe-only. + +Exact means **dimensions and all decoded RGBA bytes**, including RGB under +zero alpha. Compressed PNG bytes are recorded separately. Differences show +locations, maximum channel deltas, alpha differences and invisible-RGB +attribution, not normalized similarity. Repeat captures from one renderer also +require stable PNG bytes. No masking, blanket AA exclusion, resizing, background +flattening, or tolerance is available in this profile. Existing narrower +fixture exceptions are not extended to these cases. + +The decoder admits noninterlaced PNGs with 8-bit channels or indexed palettes +(1/2/4/8-bit indices, 8-bit palette channels). Palette transparency retains its +hidden RGB. Non-palette `tRNS`, 16-bit channels, other channel depths, interlace +and animated PNG chunks explicitly refuse: this decoder cannot preserve their +meaning within the pilot's lossless, bounded RGBA8 contract. A required image +that cannot be decoded cannot pass. Complete chunk framing, CRCs, one leading +header, strict zlib checksums/input consumption and exact filtered-scanline +length are checked before accepting pixels. Decompression has a hard output +cap, PNG files are bounded to 32 MiB before reading, and nonregular file inputs +(including FIFOs) refuse without waiting for a writer. Only first samples for the current case retain decoded +buffers; repeats retain hashes, and cross-case controls reload one checked pair +at a time. Corpus size does not multiply the live decoded-image working set. + +The viewer retains omissions next to best-effort pictures. Two missing images +can look identical; that never satisfies an undegraded rendering assertion. +A mismatch against a chosen reference is not rescued by matching resvg or a +historical PNG. A standards disagreement remains a separately reviewed question; +this tool does not turn an implementation majority into an oracle. + +## Optional second opinions and exploration + +For local exploration, explicitly download a pinned upstream corpus (local-only; +see the provisioning work in +[gridaco/nothing#7](https://github.com/gridaco/nothing/issues/7)). Write a scratch +manifest using the same contract; identify the original sources and optional +stored PNGs by hash. Do not import upstream result tables or score reports. + +Run the CLI directly to add a pinned current resvg executable: + +```sh +pnpm -C packages/grida-reftest exec tsx svg-assertions/cli.ts \ + --manifest /absolute/path/to/scratch-manifest.json \ + --out /absolute/path/to/new-output-directory \ + --observe \ + --resvg /absolute/path/to/resvg \ + --resvg-sha256 EXACT_EXECUTABLE_SHA256 \ + --resvg-version '0.47.0' +``` + +The paths and hash above are placeholders, not files included in this repo. +Record the installed binary's actual hash and version; installing resvg is an +explicit local step. It runs with system fonts disabled, on the same copied +source, and twice. Ineligible/unreviewed sources are not sent to resvg. Its +absence, identity mismatch, warning or render failure remains visibly distinct +from the assertion's chosen reference. Initial viewport and export scaling are +not interchangeable: the manifest review must establish alignment before +interpreting this additional comparison. + +The runner builds `n0_cli` once and then uses `cargo run -p n0_cli --bin n0` +for every actual render. Execution is sequential and bounded: twenty minutes for +the build, one minute per render/capture process, and bounded captured output. +Timeouts terminate the process group. Chromium runs through +[the sole capture module](../../../fixtures/web-first/chromium_capture.ts) +in a bounded worker; no capture posture is duplicated here. + +The cold-build allowance is for compilation, not slow rendering: the pinned +Linux Skia GL/SVG/WebP feature combination has no matching prebuilt archive, +and hosted source compilation exceeded the initial ten-minute build bound. +The enclosing CI job allows its existing 45 minutes plus 20 for this added +build. It does not change build features to match a different binary archive. + +Run on a stable checkout without concurrent Cargo builds/tests or edits to the +inputs and tools. Cargo lock contention can consume a render's timeout; changing +a recorded executable or input invalidates the run. Keep that failed report and +rerun in a new directory once the environment is stable, without weakening the +assertion or extending a timeout to conceal the cause. + +## Promotion and CI + +Exploration identifies work. To promote a claim, review its meaning and source +dependencies, keep reductions separate, prove an active control, and register +new committed cells through the existing `just add` / `bake` / `gate` / `status` +workflow. Prove the real code-path gate fails under a deliberate perturbation, +then restore and re-gate. Do not loosen an assertion after observing failure. +The source corpus and +[Web checklist](../../../docs/wg/consolidation/web-checklist.md) remain the +authoritative evidence and work queue; this pilot is not another support list. + +Consolidation's seam job runs the synthetic contract tests (zero discovered +tests is a failure) and real CLI pilot +on a clean checkout, then retains its report for review. Changes to this tool +or its dependency pins activate that gate. Existing Rust pixel/refusal tests +continue to run independently. Legacy runner retirement, WPT adapters, wider +resource/font profiles, a general consensus rule, and FLIP are not implemented. diff --git a/packages/grida-reftest/svg-assertions/capture-worker.ts b/packages/grida-reftest/svg-assertions/capture-worker.ts new file mode 100644 index 00000000..3664c7b2 --- /dev/null +++ b/packages/grida-reftest/svg-assertions/capture-worker.ts @@ -0,0 +1,51 @@ +/** Bounded by the parent process. No alternate Chromium capture posture. */ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { + captureFirstSvg, + deterministicContext, + launchDeterministicChromium, +} from "../../../fixtures/web-first/chromium_capture"; + +async function main(): Promise { + const [source, out, width, height, version] = process.argv.slice(2); + const bytes = await readFile(source); + const browser = await launchDeterministicChromium(); + try { + if (browser.version() !== version) + throw new Error( + `Chromium version drift: ${browser.version()} != ${version}` + ); + const context = await deterministicContext(browser); + try { + const page = await context.newPage(); + page.setDefaultTimeout(15000); + page.setDefaultNavigationTimeout(15000); + const failures: string[] = []; + page.on("requestfailed", (request) => + failures.push(`${request.url()}: ${request.failure()?.errorText}`) + ); + for (let i = 0; i < 2; i++) { + const png = await captureFirstSvg(page, { + source: bytes, + width: Number(width), + height: Number(height), + media: "image/svg+xml", + label: "assertion input", + }); + await writeFile(join(out, `chromium-${i}.png`), png, { flag: "wx" }); + } + if (failures.length) + throw new Error(`resource requests failed: ${failures.join("; ")}`); + await page.close(); + } finally { + await context.close(); + } + } finally { + await browser.close(); + } +} +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/packages/grida-reftest/svg-assertions/cli.ts b/packages/grida-reftest/svg-assertions/cli.ts new file mode 100644 index 00000000..4026c24e --- /dev/null +++ b/packages/grida-reftest/svg-assertions/cli.ts @@ -0,0 +1,66 @@ +import { resolve } from "node:path"; +import { run, type Options } from "./runner"; + +async function main(): Promise { + const argv = process.argv.slice(2), + values = new Map(); + let observe = false; + while (argv.length) { + const flag = argv.shift()!; + if (flag === "--observe" && !observe) { + observe = true; + continue; + } + if ( + ![ + "--manifest", + "--out", + "--resvg", + "--resvg-sha256", + "--resvg-version", + ].includes(flag) || + values.has(flag) || + !argv.length + ) + throw new Error(`unknown, duplicate, or incomplete option: ${flag}`); + values.set(flag, argv.shift()!); + } + if (!values.get("--manifest") || !values.get("--out")) + throw new Error( + "usage: cli.ts --manifest FILE --out NEW_DIRECTORY [--observe] [--resvg FILE --resvg-sha256 HASH --resvg-version TEXT]" + ); + const options: Options = { + manifest: resolve(values.get("--manifest")!), + out: resolve(values.get("--out")!), + }; + if ( + ["--resvg", "--resvg-sha256", "--resvg-version"].some((key) => + values.has(key) + ) + ) { + const executable = values.get("--resvg"), + sha256 = values.get("--resvg-sha256"), + version = values.get("--resvg-version"); + if (!executable || !sha256 || !/^[a-f0-9]{64}$/.test(sha256) || !version) + throw new Error( + "fresh resvg requires an executable, sha256 and exact version text" + ); + options.resvg = { executable, sha256, version }; + } + const report = await run(options); + for (const r of report.cases) + console.log( + `${r.verdict.status} ${r.case.id} [${r.verdict.kind}]: ${r.verdict.reasons.join("; ")}` + ); + console.log(`Observations: ${options.out}/index.html`); + if (observe) { + console.log( + "OBSERVE ONLY: this invocation does not certify a required gate." + ); + if (report.integrity.length) process.exitCode = 1; + } else if (!report.gate_ready) process.exitCode = 1; +} +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/packages/grida-reftest/svg-assertions/model.test.ts b/packages/grida-reftest/svg-assertions/model.test.ts new file mode 100644 index 00000000..830b362f --- /dev/null +++ b/packages/grida-reftest/svg-assertions/model.test.ts @@ -0,0 +1,442 @@ +import { describe, expect, it } from "vitest"; +import { PNG } from "pngjs"; +import { crc32, deflateSync } from "node:zlib"; +import { + compare, + decode, + evaluate, + gateReady, + parseSuite, + repeatProblem, + sha256, + type Case, + type Evidence, + type Image, + type Observation, + type Suite, +} from "./model"; + +const hash = "a".repeat(64); +const file = { path: "subject.svg", sha256: hash }; +function rendering(id = "subject", control = "control"): Case { + return { + id, + source: file, + description: "The solid rectangle covers its underlay.", + required: true, + width: 2, + height: 1, + review: { + status: "reviewed", + reason: "Hermetic source and active color control reviewed.", + blockers: [], + }, + stored: null, + assertion: { + kind: "render-exact", + decision: "Pinned Chromium compatibility, not general conformance.", + reference: { path: "expected.png", sha256: hash }, + control: { + case: control, + why: "Changing the subject fill changes the result.", + }, + }, + }; +} +function suite(): Suite { + return { + schema_version: 1, + profile: "static-self-contained-svg-v1", + capture: { sha256: hash, browser_version: "test-version" }, + cases: [rendering(), rendering("control", "subject")], + }; +} +function observation(): Observation { + const sample = { + execution: { + command: ["test-observation"], + exit: 0, + signal: null, + error: null, + stdout: "", + stderr: "", + }, + diagnostics: "", + image: { + path: "image.png", + png_sha256: hash, + rgba_sha256: hash, + width: 2, + height: 1, + }, + }; + return { + problem: null, + samples: [structuredClone(sample), structuredClone(sample)], + }; +} +const image = (values: number[], width = 2, height = 1): Image => ({ + width, + height, + rgba: Buffer.from(values), +}); +const original = image([0, 0, 0, 0, 20, 30, 40, 255]); +const changed = image([1, 0, 0, 0, 20, 30, 40, 255]); + +function chunk(type: string, data = Buffer.alloc(0)): Buffer { + const result = Buffer.alloc(data.length + 12); + result.writeUInt32BE(data.length); + result.write(type, 4, "ascii"); + data.copy(result, 8); + result.writeUInt32BE(crc32(result.subarray(4, -4)), result.length - 4); + return result; +} +function ihdr( + width = 1, + height = 1, + depth = 8, + color = 6, + interlace = 0 +): Buffer { + const data = Buffer.alloc(13); + data.writeUInt32BE(width); + data.writeUInt32BE(height, 4); + data[8] = depth; + data[9] = color; + data[12] = interlace; + return chunk("IHDR", data); +} +function png(...chunks: Buffer[]): Buffer { + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + ...chunks, + ]); +} +function evidence(): Evidence { + return { + problems: [], + strict: observation(), + best: observation(), + chromium: observation(), + pairs: { + "strict-baked": compare(original, original), + "best-baked": compare(original, original), + "chromium-baked": compare(original, original), + "strict-best": compare(original, original), + "chromium-control": compare(original, changed), + }, + }; +} +describe("strict case contracts", () => { + it("accepts only a described assertion with an identified independent expectation", () => + expect(parseSuite(suite())).toEqual(suite())); + it.each(["similarity", "tolerance", "score", "optionalTypo"])( + "rejects an unknown field %s", + (key) => { + const s = { ...suite(), [key]: true }; + expect(() => parseSuite(s)).toThrow("expected exactly"); + } + ); + it("rejects empty, duplicate and missing-control cases", () => { + expect(() => parseSuite({ ...suite(), cases: [] })).toThrow( + "declare 1..128 cases" + ); + expect(() => + parseSuite({ ...suite(), cases: [rendering(), rendering()] }) + ).toThrow("duplicate case id"); + expect(() => parseSuite({ ...suite(), cases: [rendering()] })).toThrow( + "missing or self-referential control" + ); + }); + it("rejects undescribed assertions, path IDs, dimensions and malformed hashes", () => { + const invalid: Array<[Partial, string]> = [ + [{ description: null }, "assertions require a description"], + [{ id: "../escape" }, "invalid case id"], + [{ width: 0 }, "dimensions must be 1..2048"], + [{ height: 2049 }, "dimensions must be 1..2048"], + [{ source: { ...file, sha256: "unknown" } }, "expected sha256"], + ]; + for (const [patch, reason] of invalid) + expect(() => + parseSuite({ + ...suite(), + cases: [ + { ...rendering(), ...patch }, + rendering("control", "subject"), + ], + }) + ).toThrow(reason); + }); +}); +describe("exact pixels, not image grades", () => { + it("one hidden RGB byte fails exact comparison and stays attributed", () => + expect(compare(original, changed)).toEqual({ + relation: "different-rgba", + pixels: 1, + max_delta: 1, + hidden_rgb_pixels: 1, + alpha_pixels: 0, + bounds: [0, 0, 0, 0], + })); + it("rejects equal pixel counts with different dimensions", () => + expect(compare(original, image([...original.rgba], 1, 2)).relation).toBe( + "dimension-mismatch" + )); + it("compares decoded pixels rather than PNG compression", () => { + const png = new PNG({ width: 2, height: 1 }); + png.data.set(original.rgba); + const a = PNG.sync.write(png, { deflateLevel: 0 }), + b = PNG.sync.write(png, { deflateLevel: 9 }); + expect(a.equals(b)).toBe(false); + expect(compare(decode(a), decode(b)).relation).toBe("exact-rgba"); + }); + it("rejects invalid and oversized PNGs before decoding", () => { + expect(() => decode(Buffer.alloc(24))).toThrow("invalid PNG"); + const png = PNG.sync.write(new PNG({ width: 1, height: 1 })); + png.writeUInt32BE(99999, 16); + png.writeUInt32BE(crc32(png.subarray(12, 29)), 29); + expect(() => decode(png)).toThrow("dimensions must be 1..2048"); + }); + it("refuses non-palette tRNS before the decoder can erase hidden RGB", () => { + for (const rgb of [ + [1, 2, 3], + [4, 5, 6], + ]) { + const transparent = Buffer.alloc(6); + rgb.forEach((value, i) => transparent.writeUInt16BE(value, i * 2)); + const bytes = png( + ihdr(1, 1, 8, 2), + chunk("tRNS", transparent), + chunk("IDAT", deflateSync(Buffer.from([0, ...rgb]))), + chunk("IEND") + ); + expect(() => decode(bytes)).toThrow("non-palette tRNS loses hidden RGB"); + } + }); + it("refuses empty, truncated, unchecked or surplus compressed image data", () => { + const complete = deflateSync(Buffer.from([0, 1, 2, 3, 0])); + const wrongChecksum = Buffer.from(complete); + wrongChecksum[wrongChecksum.length - 1] ^= 1; + for (const compressed of [ + Buffer.alloc(0), + deflateSync(Buffer.from([0, 1, 2, 3])), + complete.subarray(0, -1), + complete.subarray(0, -4), + wrongChecksum, + Buffer.concat([complete, Buffer.from([0])]), + deflateSync(Buffer.alloc(6)), + deflateSync(Buffer.alloc(100000)), + ]) + expect(() => + decode(png(ihdr(), chunk("IDAT", compressed), chunk("IEND"))) + ).toThrow("invalid PNG compressed data"); + }); + it("accepts one complete zlib stream split across contiguous empty or nonempty IDATs", () => { + const compressed = deflateSync(Buffer.from([0, 1, 2, 3, 0])); + const bytes = png( + ihdr(), + chunk("IDAT"), + chunk("IDAT", compressed.subarray(0, 3)), + chunk("IDAT"), + chunk("IDAT", compressed.subarray(3)), + chunk("IDAT"), + chunk("IEND") + ); + expect([...decode(bytes).rgba]).toEqual([1, 2, 3, 0]); + }); + it("preserves every hidden RGB channel in palette and RGBA8 PNGs", () => { + const decoded = [ + [1, 2, 3], + [4, 5, 6], + ].map((rgb) => { + const bytes = png( + ihdr(1, 1, 1, 3), + chunk("PLTE", Buffer.from(rgb)), + chunk("tRNS", Buffer.from([0])), + chunk("IDAT", deflateSync(Buffer.from([0, 0]))), + chunk("IEND") + ); + const result = decode(bytes); + expect([...result.rgba]).toEqual([...rgb, 0]); + const rgba = new PNG({ width: 1, height: 1 }); + rgba.data.set([...rgb, 0]); + expect(compare(result, decode(PNG.sync.write(rgba))).relation).toBe( + "exact-rgba" + ); + return result; + }); + expect(compare(decoded[0], decoded[1])).toMatchObject({ + relation: "different-rgba", + hidden_rgb_pixels: 1, + pixels: 1, + }); + }); + it("rejects replacement headers before their dimensions reach allocation", () => { + const bytes = png( + ihdr(), + ihdr(2049, 1), + chunk("IDAT", deflateSync(Buffer.alloc(1 + 2049 * 4))), + chunk("IEND") + ); + expect(() => decode(bytes)).toThrow("duplicate PNG IHDR"); + }); + it("refuses rescaling, unbounded interlaced inflate and animation encodings", () => { + for (const header of [ihdr(1, 1, 16), ihdr(1, 1, 8, 6, 1)]) + expect(() => + decode( + png( + header, + chunk("IDAT", deflateSync(Buffer.alloc(9))), + chunk("IEND") + ) + ) + ).toThrow("unsupported PNG encoding"); + expect(() => + decode( + png( + ihdr(), + chunk("acTL", Buffer.alloc(8)), + chunk("IDAT", deflateSync(Buffer.alloc(5))), + chunk("IEND") + ) + ) + ).toThrow("unsupported PNG chunk: acTL"); + }); + it("rejects malformed chunk boundaries, CRC, ordering and trailing bytes", () => { + const good = png( + ihdr(), + chunk("IDAT", deflateSync(Buffer.alloc(5))), + chunk("IEND") + ); + const badCrc = Buffer.from(good); + badCrc[29] ^= 1; + expect(() => decode(badCrc)).toThrow("CRC"); + expect(() => decode(good.subarray(0, -1))).toThrow("truncated PNG chunk"); + expect(() => decode(Buffer.concat([good, Buffer.from([0])]))).toThrow( + "invalid PNG end" + ); + expect(() => decode(png(chunk("tEXt"), ihdr(), chunk("IEND")))).toThrow( + "begin with IHDR" + ); + expect(() => + decode( + png(ihdr(), chunk("IDAT"), chunk("tEXt"), chunk("IDAT"), chunk("IEND")) + ) + ).toThrow("data order"); + }); +}); +describe("assertion verdicts", () => { + it("accepts a fully evidenced exact assertion", () => + expect(evaluate(rendering(), evidence()).status).toBe("PASS")); + it("does not let a matching second opinion rescue a declared-reference mismatch", () => { + const e = evidence(); + e.pairs["strict-baked"] = compare(original, changed); + e.pairs["strict-resvg"] = compare(original, original); + expect(evaluate(rendering(), e).status).toBe("FAIL"); + }); + it("fails a missing comparison even if every available image matches", () => { + const e = evidence(); + delete e.pairs["best-baked"]; + expect(evaluate(rendering(), e).status).toBe("FAIL"); + }); + it("does not admit matching omissions or unexpected refusals", () => { + const e = evidence(); + e.best.samples.forEach( + (s) => + (s.diagnostics = "degraded: skipped svg/image[1]: unsupported image") + ); + expect(evaluate(rendering(), e).status).toBe("FAIL"); + e.strict.samples.forEach((s) => { + s.execution.exit = 1; + s.image = null; + }); + expect(evaluate(rendering(), e).status).toBe("FAIL"); + }); + it("does not judge an environment-blocked or unreviewed source", () => { + const c = rendering(); + c.review.blockers.push("relative image not provided"); + expect(evaluate(c, evidence()).status).toBe("UNRESOLVED"); + c.review.blockers = []; + c.review.status = "unreviewed"; + expect(evaluate(c, evidence()).status).toBe("UNRESOLVED"); + }); + it("does not judge a placeholder with an ineffective control", () => { + const e = evidence(); + e.pairs["chromium-control"] = compare(original, original); + expect(evaluate(rendering(), e).status).toBe("UNRESOLVED"); + }); + it("preserves undescribed inputs as observations", () => { + const c = rendering(); + c.assertion = null; + c.description = null; + expect(evaluate(c, evidence()).status).toBe("OBSERVATION"); + }); + it("refuses to judge with changed inputs or an unavailable browser", () => { + const e = evidence(); + e.problems.push("hash drift"); + expect(evaluate(rendering(), e).status).toBe("UNRESOLVED"); + e.problems = []; + e.chromium.problem = "wrong version"; + expect(evaluate(rendering(), e).status).toBe("UNRESOLVED"); + }); + it("detects missing outputs, crashes and nondeterministic repeats", () => { + for (const fault of ["missing", "crash", "different", "diagnostic"]) { + const e = evidence(); + if (fault === "missing") e.best.samples[1].image = null; + if (fault === "crash") e.best.samples[1].execution.signal = "SIGSEGV"; + if (fault === "different") + e.best.samples[1].image!.rgba_sha256 = sha256("different"); + if (fault === "diagnostic") + e.best.samples[1].diagnostics = "warning: changed"; + expect(repeatProblem(e.best)).not.toBeNull(); + expect(evaluate(rendering(), e).status).toBe("FAIL"); + } + }); + it("verifies named refusals without turning them into rendering support", () => { + const c = rendering(); + c.assertion = { + kind: "refusal", + decision: "Guard the declared unsupported unit.", + strict: { + exit: 1, + diagnostics: "error: render failed: attribute cx is not a number", + }, + best: { + exit: 0, + diagnostics: + "degraded: skipped svg/circle[1]: attribute cx is not a number", + }, + }; + const expected = c.assertion; + const e = evidence(); + e.strict.samples.forEach((s) => { + s.execution.exit = 1; + s.image = null; + s.diagnostics = expected.strict.diagnostics; + }); + e.best.samples.forEach((s) => { + s.diagnostics = expected.best.diagnostics; + }); + expect(evaluate(c, e)).toEqual({ + status: "PASS", + kind: "refusal", + reasons: [], + }); + e.best.samples.forEach((s) => (s.diagnostics = "")); + expect(evaluate(c, e).status).toBe("FAIL"); + e.strict.samples = []; + expect(evaluate(c, e).status).toBe("FAIL"); + }); + it("fails closed on omitted, substituted, unresolved, or empty required gates", () => { + const c = rendering(), + v = evaluate(c, evidence()); + expect(gateReady([c], { subject: v }, [])).toBe(true); + expect(gateReady([c], {}, [])).toBe(false); + expect(gateReady([c], { replacement: v }, [])).toBe(false); + expect( + gateReady([c], { subject: { ...v, status: "UNRESOLVED" } }, []) + ).toBe(false); + expect(gateReady([c], { subject: v }, ["tool drift"])).toBe(false); + expect(gateReady([], {}, [])).toBe(false); + }); +}); diff --git a/packages/grida-reftest/svg-assertions/model.ts b/packages/grida-reftest/svg-assertions/model.ts new file mode 100644 index 00000000..bb84c407 --- /dev/null +++ b/packages/grida-reftest/svg-assertions/model.ts @@ -0,0 +1,537 @@ +/** Assertion results are not image scores. This module owns no renderer, + * process, filesystem, tolerance, or reference-selection heuristic. */ +import { createHash } from "node:crypto"; +import { PNG } from "pngjs"; +import { crc32, inflateSync } from "node:zlib"; + +export type FileIdentity = { path: string; sha256: string }; +export type Assertion = + | { + kind: "render-exact"; + reference: FileIdentity; + decision: string; + control: { case: string; why: string }; + } + | { + kind: "refusal"; + decision: string; + strict: { exit: number; diagnostics: string }; + best: { exit: number; diagnostics: string }; + }; +export interface Case { + id: string; + source: FileIdentity; + description: string | null; + required: boolean; + width: number; + height: number; + review: { + status: "reviewed" | "unreviewed"; + reason: string; + blockers: string[]; + }; + assertion: Assertion | null; + stored: FileIdentity | null; +} +export interface Suite { + schema_version: 1; + profile: "static-self-contained-svg-v1"; + capture: { sha256: string; browser_version: string }; + cases: Case[]; +} +export const sha256 = (bytes: Uint8Array | string): string => + createHash("sha256").update(bytes).digest("hex"); + +// Reject unknown fields: a misspelled requirement must not quietly disappear. +function object(value: unknown, fields: string[]): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("expected object"); + const record = value as Record; + if ( + Object.keys(record).some((key) => !fields.includes(key)) || + fields.some((key) => !(key in record)) + ) + throw new Error(`expected exactly: ${fields.join(", ")}`); + return record; +} +function text(value: unknown): string { + if (typeof value !== "string" || !value.trim() || value.length > 8192) + throw new Error("expected nonempty bounded text"); + return value; +} +function hash(value: unknown): string { + const result = text(value); + if (!/^[a-f0-9]{64}$/.test(result)) throw new Error("expected sha256"); + return result; +} +function identity(value: unknown): FileIdentity { + const f = object(value, ["path", "sha256"]); + const path = text(f.path); + if (path.includes("\0") || path.includes("\n")) + throw new Error("invalid input path"); + return { path, sha256: hash(f.sha256) }; +} +function dimension(value: unknown): number { + if (!Number.isInteger(value) || Number(value) < 1 || Number(value) > 2048) + throw new Error("dimensions must be 1..2048"); + return Number(value); +} +function departure(value: unknown): { exit: number; diagnostics: string } { + const r = object(value, ["exit", "diagnostics"]); + if (r.exit !== 0 && r.exit !== 1) + throw new Error("refusal exit must be 0 or 1"); + return { exit: r.exit, diagnostics: text(r.diagnostics) }; +} +function assertion(value: unknown): Assertion | null { + if (value === null) return null; + if (!value || typeof value !== "object" || !("kind" in value)) + throw new Error("missing assertion kind"); + if (value.kind === "render-exact") { + const a = object(value, ["kind", "reference", "decision", "control"]); + const c = object(a.control, ["case", "why"]); + return { + kind: "render-exact", + reference: identity(a.reference), + decision: text(a.decision), + control: { case: text(c.case), why: text(c.why) }, + }; + } + if (value.kind === "refusal") { + const a = object(value, ["kind", "decision", "strict", "best"]); + const strict = departure(a.strict), + best = departure(a.best); + if ( + strict.exit !== 1 || + !strict.diagnostics.startsWith("error: render failed: ") + ) + throw new Error("strict must name a render refusal"); + if (best.exit === 0 && !best.diagnostics.startsWith("degraded: ")) + throw new Error("best effort must declare degradation"); + return { kind: "refusal", decision: text(a.decision), strict, best }; + } + throw new Error("unknown assertion kind"); +} +export function parseSuite(value: unknown): Suite { + const s = object(value, ["schema_version", "profile", "capture", "cases"]); + if (s.schema_version !== 1 || s.profile !== "static-self-contained-svg-v1") + throw new Error("unsupported manifest profile/version"); + const cap = object(s.capture, ["sha256", "browser_version"]); + if (!Array.isArray(s.cases) || !s.cases.length || s.cases.length > 128) + throw new Error("declare 1..128 cases"); + const cases = s.cases.map((value) => { + const c = object(value, [ + "id", + "source", + "description", + "required", + "width", + "height", + "review", + "assertion", + "stored", + ]); + const id = text(c.id), + r = object(c.review, ["status", "reason", "blockers"]); + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(id) || id.length > 100) + throw new Error("invalid case id"); + if (typeof c.required !== "boolean") + throw new Error("required must be boolean"); + if (r.status !== "reviewed" && r.status !== "unreviewed") + throw new Error("unknown review status"); + if (!Array.isArray(r.blockers)) + throw new Error("blockers must be explicit"); + const result: Case = { + id, + source: identity(c.source), + description: c.description === null ? null : text(c.description), + required: c.required, + width: dimension(c.width), + height: dimension(c.height), + review: { + status: r.status, + reason: text(r.reason), + blockers: r.blockers.map(text), + }, + assertion: assertion(c.assertion), + stored: c.stored === null ? null : identity(c.stored), + }; + if (result.assertion && !result.description) + throw new Error("assertions require a description"); + return result; + }); + if (new Set(cases.map((c) => c.id)).size !== cases.length) + throw new Error("duplicate case id"); + for (const c of cases) { + const a = c.assertion; + if ( + a?.kind === "render-exact" && + (!cases.some((other) => other.id === a.control.case) || + a.control.case === c.id) + ) + throw new Error(`${c.id}: missing or self-referential control`); + } + return { + schema_version: 1, + profile: "static-self-contained-svg-v1", + capture: { + sha256: hash(cap.sha256), + browser_version: text(cap.browser_version), + }, + cases, + }; +} + +export interface Image { + width: number; + height: number; + rgba: Buffer; +} +export interface ImageRecord { + path: string; + png_sha256: string; + rgba_sha256: string; + width: number; + height: number; +} +export const MAX_PNG_BYTES = 32 * 1024 * 1024; + +export function decode(bytes: Buffer): Image { + // pngjs accepts replacement IHDRs and erases RGB for non-palette tRNS. + // Validate the complete bounded stream BEFORE trusting its allocation or + // normalization. Interlaced inflate and 16-bit rescaling are outside v1. + if ( + bytes.length < 33 || + bytes.length > MAX_PNG_BYTES || + !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) + ) + throw new Error("invalid PNG"); + let width = 0, + height = 0, + color = -1, + depth = 0, + palette = 0; + let header = false, + transparency = false, + data = false, + dataEnded = false, + end = false; + const idat: Buffer[] = []; + for (let offset = 8; offset < bytes.length;) { + if (offset + 12 > bytes.length) throw new Error("truncated PNG chunk"); + const length = bytes.readUInt32BE(offset), + next = offset + length + 12; + if (next > bytes.length) throw new Error("truncated PNG chunk"); + const type = bytes.toString("latin1", offset + 4, offset + 8); + if (!/^[A-Za-z]{2}[A-Z][A-Za-z]$/.test(type)) + throw new Error("invalid PNG chunk type"); + if ( + crc32(bytes.subarray(offset + 4, next - 4)) !== + bytes.readUInt32BE(next - 4) + ) + throw new Error("invalid PNG chunk CRC"); + if (!header && type !== "IHDR") throw new Error("PNG must begin with IHDR"); + if (type === "IHDR") { + if (header || offset !== 8 || length !== 13) + throw new Error("invalid or duplicate PNG IHDR"); + header = true; + width = dimension(bytes.readUInt32BE(offset + 8)); + height = dimension(bytes.readUInt32BE(offset + 12)); + depth = bytes[offset + 16]; + color = bytes[offset + 17]; + if ( + (color === 3 + ? ![1, 2, 4, 8].includes(depth) + : depth !== 8 || ![0, 2, 4, 6].includes(color)) || + bytes[offset + 18] !== 0 || + bytes[offset + 19] !== 0 || + bytes[offset + 20] !== 0 + ) + throw new Error( + "unsupported PNG encoding: require noninterlaced 8-bit channels or indexed palette" + ); + } else if (type === "PLTE") { + if ( + palette || + data || + length === 0 || + length % 3 || + length > 768 || + ![2, 3, 6].includes(color) || + (color === 3 && length / 3 > 2 ** depth) + ) + throw new Error("invalid PNG palette"); + palette = length / 3; + } else if (type === "tRNS") { + if (color !== 3) + throw new Error( + "unsupported PNG encoding: non-palette tRNS loses hidden RGB" + ); + if (transparency || data || !palette || length === 0 || length > palette) + throw new Error("invalid PNG transparency"); + transparency = true; + } else if (type === "IDAT") { + if (dataEnded || (color === 3 && !palette)) + throw new Error("invalid PNG data order"); + data = true; + idat.push(bytes.subarray(offset + 8, next - 4)); + } else if (type === "IEND") { + if (!data || length !== 0 || next !== bytes.length) + throw new Error("invalid PNG end"); + end = true; + } else { + if (["acTL", "fcTL", "fdAT"].includes(type) || /^[A-Z]/.test(type)) + throw new Error(`unsupported PNG chunk: ${type}`); + } + if (data && type !== "IDAT") dataEnded = true; + offset = next; + } + if (!end) throw new Error("missing PNG end"); + // pngjs's bounded inflater tolerates incomplete streams/scanlines, inventing + // transparent pixels. Native strict inflate must first verify the checksum, + // complete input consumption and exact row extent under a hard output cap. + const channels = color === 2 ? 3 : color === 4 ? 2 : color === 6 ? 4 : 1; + const scanlineBytes = + (Math.ceil((width * channels * depth) / 8) + 1) * height; + const compressed = Buffer.concat(idat); + try { + // Node's info:true runtime result is richer than @types/node's overload. + const inflated = inflateSync(compressed, { + maxOutputLength: scanlineBytes + 1, + info: true, + }) as unknown as { + buffer: Buffer; + engine: { bytesWritten: number }; + }; + if ( + inflated.buffer.length !== scanlineBytes || + inflated.engine.bytesWritten !== compressed.length + ) + throw new Error("incomplete or surplus compressed data / scanlines"); + } catch (error) { + throw new Error(`invalid PNG compressed data: ${String(error)}`); + } + const p = PNG.sync.read(bytes); + if ( + p.width !== width || + p.height !== height || + p.data.length !== width * height * 4 + ) + throw new Error("invalid RGBA length"); + return { width: p.width, height: p.height, rgba: p.data }; +} +export type Comparison = + | { relation: "dimension-mismatch"; left: number[]; right: number[] } + | { + relation: "exact-rgba" | "different-rgba"; + pixels: number; + max_delta: number; + hidden_rgb_pixels: number; + alpha_pixels: number; + bounds: number[] | null; + }; +export function compare(a: Image, b: Image): Comparison { + if (a.width !== b.width || a.height !== b.height) + return { + relation: "dimension-mismatch", + left: [a.width, a.height], + right: [b.width, b.height], + }; + let pixels = 0, + max_delta = 0, + hidden_rgb_pixels = 0, + alpha_pixels = 0; + let minX = a.width, + minY = a.height, + maxX = -1, + maxY = -1; + for (let i = 0; i < a.rgba.length; i += 4) { + let different = false; + for (let ch = 0; ch < 4; ch++) { + const d = Math.abs(a.rgba[i + ch] - b.rgba[i + ch]); + different ||= d !== 0; + max_delta = Math.max(max_delta, d); + } + if (!different) continue; + pixels++; + if (a.rgba[i + 3] === 0 && b.rgba[i + 3] === 0) hidden_rgb_pixels++; + if (a.rgba[i + 3] !== b.rgba[i + 3]) alpha_pixels++; + const x = (i / 4) % a.width, + y = Math.floor(i / 4 / a.width); + minX = Math.min(minX, x); + maxX = Math.max(maxX, x); + minY = Math.min(minY, y); + maxY = Math.max(maxY, y); + } + return { + relation: pixels ? "different-rgba" : "exact-rgba", + pixels, + max_delta, + hidden_rgb_pixels, + alpha_pixels, + bounds: pixels ? [minX, minY, maxX, maxY] : null, + }; +} +export interface Execution { + command: string[]; + exit: number | null; + signal: string | null; + error: string | null; + stdout: string; + stderr: string; +} +export interface Sample { + execution: Execution; + diagnostics: string; + image: ImageRecord | null; +} +export interface Observation { + samples: Sample[]; + problem: string | null; + invocation?: Execution; +} +export type Verdict = { + status: "PASS" | "FAIL" | "UNRESOLVED" | "OBSERVATION"; + kind: "render-exact" | "refusal" | "observation"; + reasons: string[]; +}; +export interface Evidence { + problems: string[]; + strict: Observation; + best: Observation; + chromium: Observation; + pairs: Record; +} +function sameImage(a: ImageRecord | null, b: ImageRecord | null): boolean { + return a === null + ? b === null + : b !== null && + a.width === b.width && + a.height === b.height && + a.rgba_sha256 === b.rgba_sha256; +} +export function repeatProblem(o: Observation): string | null { + if (o.problem) return o.problem; + if (o.samples.length !== 2) return "missing repeated observation"; + const [a, b] = o.samples; + if ( + a.execution.error || + b.execution.error || + a.execution.signal || + b.execution.signal + ) + return "renderer execution failed"; + if ( + a.execution.exit !== b.execution.exit || + a.diagnostics !== b.diagnostics || + a.execution.stdout !== b.execution.stdout || + !sameImage(a.image, b.image) || + a.image?.png_sha256 !== b.image?.png_sha256 + ) + return "nondeterministic observation"; + return null; +} +function rendered(o: Observation, c: Case): boolean { + return ( + !repeatProblem(o) && + o.samples.every( + (s) => + s.execution.exit === 0 && + !s.execution.stdout && + !s.diagnostics && + s.image?.width === c.width && + s.image.height === c.height + ) + ); +} +export function evaluate(c: Case, e: Evidence): Verdict { + const kind = c.assertion?.kind ?? "observation"; + if (!c.assertion) + return { + status: "OBSERVATION", + kind, + reasons: [...e.problems, "no reviewed assertion"], + }; + if ( + e.problems.length || + c.review.status !== "reviewed" || + c.review.blockers.length + ) + return { + status: "UNRESOLVED", + kind, + reasons: [ + ...e.problems, + ...c.review.blockers, + ...(c.review.status !== "reviewed" ? ["claim not reviewed"] : []), + ], + }; + const reasons: string[] = []; + if (c.assertion.kind === "refusal") { + for (const admission of ["strict", "best"] as const) { + const o = e[admission], + expected = c.assertion[admission]; + const p = repeatProblem(o); + if (p) reasons.push(`${admission}: ${p}`); + if ( + o.samples.some( + (s) => + s.execution.exit !== expected.exit || + s.execution.stdout || + s.diagnostics !== expected.diagnostics || + (expected.exit === 0 + ? s.image?.width !== c.width || s.image?.height !== c.height + : s.image !== null) + ) + ) + reasons.push( + `${admission}: expected named refusal/degradation not observed` + ); + } + } else { + if (!rendered(e.strict, c) || !rendered(e.best, c)) + reasons.push("n0 did not render repeatably without degradation"); + if (!rendered(e.chromium, c)) + return { + status: "UNRESOLVED", + kind, + reasons: [ + ...reasons, + "Chromium reference observation unavailable or unstable", + ], + }; + for (const pair of [ + "strict-baked", + "best-baked", + "chromium-baked", + "strict-best", + ]) + if (e.pairs[pair]?.relation !== "exact-rgba") + reasons.push(`${pair}: exact assertion violated or comparison missing`); + if (e.pairs["chromium-control"]?.relation !== "different-rgba") + return { + status: "UNRESOLVED", + kind, + reasons: [ + ...reasons, + "control does not demonstrate the claimed branch", + ], + }; + } + return { status: reasons.length ? "FAIL" : "PASS", kind, reasons }; +} +/** A required case cannot disappear, be replaced, or become a silent skip. */ +export function gateReady( + cases: Case[], + verdicts: Record, + integrity: string[] +): boolean { + return ( + !integrity.length && + Object.keys(verdicts).length === cases.length && + cases.some((c) => c.required) && + cases.every( + (c) => + c.id in verdicts && (!c.required || verdicts[c.id].status === "PASS") + ) + ); +} diff --git a/packages/grida-reftest/svg-assertions/pilot.json b/packages/grida-reftest/svg-assertions/pilot.json new file mode 100644 index 00000000..75ae64a7 --- /dev/null +++ b/packages/grida-reftest/svg-assertions/pilot.json @@ -0,0 +1,97 @@ +{ + "schema_version": 1, + "profile": "static-self-contained-svg-v1", + "capture": { + "sha256": "069296201718c43d29efe356fbea893781b73250dca51de3b6d47468d74027b0", + "browser_version": "149.0.7827.55" + }, + "cases": [ + { + "id": "resvg-rect-green", + "source": { + "path": "../../../fixtures/web-first/svg-upstream-resvg-rect-simple-case.svg", + "sha256": "0b487592d89c6e43573eb91eb92cb18b85bd5139c31143f534ba3e43a9dfd0cc" + }, + "description": "The unchanged upstream green rectangle covers the red underlay and matches the declared Chromium raster at a 500x500 initial viewport. This asserts this scene, not the whole rect grammar.", + "required": true, + "width": 500, + "height": 500, + "review": { + "status": "reviewed", + "reason": "Static viewBox-only, self-contained path/rect scene with no text or external resources. Upstream resvg-test-suite d8e064337faf01bc5a9579187a56dbdbe3eacc72, tests/shapes/rect/simple-case.svg; MIT notice in fixtures/web-first/LICENSE.resvg. Only the subject fill differs from its blue control.", + "blockers": [] + }, + "assertion": { + "kind": "render-exact", + "reference": { + "path": "../../../fixtures/web-first/chromium/svg-upstream-resvg-rect-simple-case.png", + "sha256": "4fd110181e80bc6c24e6d9cc2d273ad477d3c82c5fb57bef0fb871f4d08a063c" + }, + "decision": "Pinned Chromium compatibility under the existing Web-first capture; no resvg alternative or tolerance. Source and oracle are registered in primitives.json/oracle-bake.json.", + "control": { + "case": "resvg-rect-blue", + "why": "Changing only the subject rectangle's green fill to blue must change the captured pixels; the decorative frame is unchanged. The probe changes 160000 pixels at maximum channel delta 255." + } + }, + "stored": null + }, + { + "id": "resvg-rect-blue", + "source": { + "path": "../../../fixtures/web-first/svg-upstream-resvg-rect-blue-control.svg", + "sha256": "18722a00cf6f1b0bfb417dbca3c4a59237b628150f5bebaff467cde54c386b14" + }, + "description": "The separately identified blue-fill derivative renders the same geometry and hides the same red underlay. It must match its own declared Chromium raster, not the green original.", + "required": true, + "width": 500, + "height": 500, + "review": { + "status": "reviewed", + "reason": "Same static self-contained 500x500 profile as resvg-rect-green; derived only by replacing the subject fill with blue. MIT notice in fixtures/web-first/LICENSE.resvg.", + "blockers": [] + }, + "assertion": { + "kind": "render-exact", + "reference": { + "path": "../../../fixtures/web-first/chromium/svg-upstream-resvg-rect-blue-control.png", + "sha256": "6bdaceeba4a8327ece87bea27a7819434563957e8cbcdf0d4c93648aee04460a" + }, + "decision": "Pinned Chromium compatibility for the independent color control, registered in the existing primitive corpus. No resvg alternative or tolerance.", + "control": { + "case": "resvg-rect-green", + "why": "Restoring only the subject fill to green must change the picture while leaving the geometry and decorative frame unchanged." + } + }, + "stored": null + }, + { + "id": "geometry-unit-refusal", + "source": { + "path": "../../../fixtures/web-first/unsupported/svg-geometry-unit-values.svg", + "sha256": "d86e72339168f894a4cdfe4f6e0b0b8d693db852608be75ab4bbd73e7ad9f6d1" + }, + "description": "Verify the existing cx/cy/r unit-value refusals through the actual CLI: strict rejects the first circle; best effort names each skipped circle. This does not assert rendered support for these units or the correctness of fallback pixels.", + "required": true, + "width": 64, + "height": 64, + "review": { + "status": "reviewed", + "reason": "Existing registered unsupported-corpus witness; static 64x64 source, no external resources or text. The original Rust refusal gate remains authoritative for corpus membership.", + "blockers": [] + }, + "assertion": { + "kind": "refusal", + "decision": "Guard the exact existing CLI named-degradation contract, independently from any positive image comparison.", + "strict": { + "exit": 1, + "diagnostics": "error: render failed: attribute cx=\"12pt\" is not a number" + }, + "best": { + "exit": 0, + "diagnostics": "degraded: skipped svg/circle[1]: attribute cx=\"12pt\" is not a number\ndegraded: skipped svg/circle[2]: attribute cy=\"16px\" is not a number\ndegraded: skipped svg/circle[3]: attribute r=\"8px\" is not a number" + } + }, + "stored": null + } + ] +} diff --git a/packages/grida-reftest/svg-assertions/report.ts b/packages/grida-reftest/svg-assertions/report.ts new file mode 100644 index 00000000..c9606ba3 --- /dev/null +++ b/packages/grida-reftest/svg-assertions/report.ts @@ -0,0 +1,48 @@ +import type { Report } from "./runner"; + +export const escapeHtml = (value: unknown): string => + String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + +/** Static local viewer. No scripts, remote assets, winner selection or scores. */ +export function renderReport(report: Report): string { + const e = escapeHtml; + const labels: Record = { + baked: "Baked Chromium reference", + chromium: "Fresh Chromium", + strict: "n0 strict", + best: "n0 best effort", + stored: "Upstream stored PNG", + resvg: "Fresh resvg", + }; + return `SVG assertion observations + +

SVG assertion observations

One described claim, one discrete verdict. Image differences are evidence, not grades.

+

${report.gate_ready ? "Required assertions satisfied." : "Required assertion gate is NOT ready."} This is not an SVG support or conformance result.

+

Profile: ${e(report.manifest.profile)} · Chromium ${e(report.manifest.capture.browser_version)} · Raw observations

+
Run provenance: versions, hashes and build identity
${e(JSON.stringify(report.tools, null, 2))}
+${report.integrity.length ? `
${e(report.integrity.join("\n"))}
` : ""} +${report.cases + .map( + ( + r + ) => `

${e(r.case.id)}

${e(r.case.description ?? "Undescribed upstream observation")}

+

${e(r.verdict.status)} · ${r.verdict.kind === "refusal" ? "expected refusal assertion — NOT rendering support" : e(r.verdict.kind)} · ${r.case.required ? "required" : "exploratory"}

+

${e(r.case.assertion?.decision ?? "No reference decision")}

${e(r.verdict.reasons.join("\n"))}
+
${["baked", "chromium", "strict", "best", "stored", "resvg"] + .map((name) => { + const o = r.observations[name], + sample = o.samples[0], + image = sample?.image; + return `
${e(labels[name])}
${image ? `${e(labels[name])} output for ${e(r.case.id)}` : "

No image

"}
${e(o.problem ?? sample?.diagnostics ?? "")}
`; + }) + .join( + "" + )}
Exact comparisons, diagnostics and both repeats
${e(JSON.stringify({ pairs: r.pairs, observations: r.observations }, null, 2))}
` + ) + .join("\n")}`; +} diff --git a/packages/grida-reftest/svg-assertions/runner.test.ts b/packages/grida-reftest/svg-assertions/runner.test.ts new file mode 100644 index 00000000..fa15dff9 --- /dev/null +++ b/packages/grida-reftest/svg-assertions/runner.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from "vitest"; +import { + chromiumObservation, + cliDiagnostics, + command, + executionFailure, + firstSampleComparisons, + readBounded, +} from "./runner"; +import { repeatProblem, type Execution, type ImageRecord } from "./model"; +import { escapeHtml } from "./report"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { execFileSync } from "node:child_process"; + +describe("partial Chromium observations", () => { + const execution: Execution = { + command: ["capture"], + exit: 0, + signal: null, + error: null, + stdout: "", + stderr: "", + }; + const record: ImageRecord = { + path: "chromium-1.png", + png_sha256: "a".repeat(64), + rgba_sha256: "b".repeat(64), + width: 1, + height: 1, + }; + it("retains missing repeat zero without promoting repeat one or crashing comparisons", async () => { + const partial = await chromiumObservation(execution, async (i) => { + if (i === 0) throw new Error("missing first PNG"); + return record; + }); + expect(partial.samples).toHaveLength(2); + expect(partial.samples[0].image).toBeNull(); + expect(partial.samples[1].image).toBe(record); + expect(partial.problem).toContain("missing first PNG"); + expect(repeatProblem(partial)).not.toBeNull(); + expect(firstSampleComparisons({ chromium: partial }, new Map())).toEqual( + {} + ); + }); + it("guards absent decoded pixels even when image metadata exists", async () => { + const complete = await chromiumObservation(execution, async () => record); + expect( + firstSampleComparisons( + { strict: complete, chromium: complete }, + new Map() + ) + ).toEqual({}); + const images = new Map([ + [record.path, { width: 1, height: 1, rgba: Buffer.alloc(4) }], + ]); + expect( + firstSampleComparisons({ strict: complete, chromium: complete }, images)[ + "strict-chromium" + ].relation + ).toBe("exact-rgba"); + }); + it("names signal and exit-only failures even with empty stderr", async () => { + expect( + executionFailure({ ...execution, exit: null, signal: "SIGKILL" }) + ).toBe("signal SIGKILL"); + expect(executionFailure({ ...execution, exit: 7, stderr: " " })).toBe( + "exit code 7" + ); + expect(executionFailure({ ...execution, exit: null })).toBe( + "no exit status" + ); + const failed = await chromiumObservation( + { ...execution, exit: 7 }, + async () => record + ); + expect(failed.problem).toBe("Chromium capture failed: exit code 7"); + }); +}); + +describe("bounded file observations", () => { + it.skipIf(process.platform === "win32")( + "refuses a FIFO without waiting for a writer", + async () => { + const dir = await mkdtemp(join(tmpdir(), "svg-assertion-fifo-test-")); + try { + const fifo = join(dir, "input.pipe"); + execFileSync("mkfifo", [fifo]); + await expect(readBounded(fifo, 100)).rejects.toThrow("file bound"); + } finally { + await rm(dir, { recursive: true }); + } + }, + 2000 + ); + it("reads exact bytes but refuses oversized files and non-files before allocation", async () => { + const dir = await mkdtemp(join(tmpdir(), "svg-assertion-file-test-")); + try { + const file = join(dir, "input.bin"); + await writeFile(file, Buffer.from([1, 2, 3, 4])); + expect([...(await readBounded(file, 4))]).toEqual([1, 2, 3, 4]); + await expect(readBounded(file, 3)).rejects.toThrow("file bound"); + await expect(readBounded(dir, 100)).rejects.toThrow("file bound"); + } finally { + await rm(dir, { recursive: true }); + } + }); +}); + +describe("bounded process observations", () => { + it("records a missing executable instead of fabricating a refusal", async () => { + const r = await command("/nonexistent/svg-assertions-executable", []); + expect(r.error).not.toBeNull(); + expect(r.exit).not.toBe(0); + }); + it("kills a timed-out renderer", async () => { + const r = await command( + process.execPath, + ["-e", "setInterval(()=>{},1000)"], + 50 + ); + expect(r.error).toBe("execution-timeout"); + expect(r.exit).not.toBe(0); + }); + it("retains stdout, stderr, exit status and crash signal separately", async () => { + const r = await command(process.execPath, [ + "-e", + "process.stdout.write('out');process.stderr.write('err');process.exit(7)", + ]); + expect(r.stdout).toBe("out"); + expect(r.stderr).toBe("err"); + expect(r.exit).toBe(7); + expect(r.error).toBeNull(); + }); +}); +describe("diagnostic and display safety", () => { + it("does not hide a degraded footer when the declaration lines disappeared", () => { + const r = { + command: [], + exit: 0, + signal: null, + error: null, + stdout: "", + stderr: + "rendered input.svg -> output.png (64x64, base-shared-frame, 1 degraded, 137 bytes)\n", + }; + expect( + cliDiagnostics( + r, + "input.svg", + "output.png", + { width: 64, height: 64 }, + 137 + ) + ).toContain("1 degraded"); + }); + it("removes only the exact successful CLI footer", () => { + const r = { + command: [], + exit: 0, + signal: null, + error: null, + stdout: "", + stderr: + "degraded: skipped svg/circle[1]: unsupported unit\nrendered input.svg -> output.png (64x64, base-shared-frame, 1 degraded, 137 bytes)\n", + }; + expect( + cliDiagnostics( + r, + "input.svg", + "output.png", + { width: 64, height: 64 }, + 137 + ) + ).toBe("degraded: skipped svg/circle[1]: unsupported unit"); + expect( + cliDiagnostics( + r, + "different.svg", + "output.png", + { width: 64, height: 64 }, + 137 + ) + ).toContain("rendered input.svg"); + expect( + cliDiagnostics( + { ...r, exit: 1 }, + "input.svg", + "output.png", + { width: 64, height: 64 }, + 137 + ) + ).toContain("rendered input.svg"); + }); + it("escapes untrusted descriptions and renderer messages", () => + expect(escapeHtml('&')).toBe( + "<img src=x onerror="alert(1)">&" + )); +}); diff --git a/packages/grida-reftest/svg-assertions/runner.ts b/packages/grida-reftest/svg-assertions/runner.ts new file mode 100644 index 00000000..d60b4906 --- /dev/null +++ b/packages/grida-reftest/svg-assertions/runner.ts @@ -0,0 +1,640 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { readFile, writeFile, mkdir, open } from "node:fs/promises"; +import { dirname, join, resolve, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + compare, + decode, + evaluate, + gateReady, + parseSuite, + repeatProblem, + sha256, + MAX_PNG_BYTES, + type Case, + type Comparison, + type Execution, + type FileIdentity, + type Image, + type ImageRecord, + type Observation, + type Sample, + type Suite, + type Verdict, +} from "./model"; +import { renderReport } from "./report"; + +export const toolDir = dirname(fileURLToPath(import.meta.url)); +export const repoDir = resolve(toolDir, "../../.."); +const capturePath = join(repoDir, "fixtures/web-first/chromium_capture.ts"); +const env = { + ...process.env, + PATH: `${process.env.HOME}/.cargo/bin:${process.env.PATH}`, +}; +const MAX_OUTPUT = 4 * 1024 * 1024; + +/** Bound file input before allocation, including a file growing during read. */ +export async function readBounded( + path: string, + limit: number +): Promise { + // A FIFO must not block in open before descriptor validation can reject it. + const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > limit) + throw new Error(`input exceeds file bound: ${path}`); + const bytes = Buffer.alloc(stat.size + 1); + let total = 0; + while (total < bytes.length) { + const { bytesRead } = await file.read( + bytes, + total, + bytes.length - total, + null + ); + if (!bytesRead) break; + total += bytesRead; + } + if (total > stat.size || total > limit) + throw new Error(`input grew during read: ${path}`); + return bytes.subarray(0, total); + } finally { + await file.close(); + } +} + +/** No shell interpolation. Kill the process group on timeout/output overflow, + * including Chromium grandchildren and cargo's child renderer. */ +export function command( + executable: string, + args: string[], + timeout = 60000 +): Promise { + return new Promise((resolveResult) => { + let stdout = "", + stderr = "", + error: string | null = null, + bytes = 0; + const child = spawn(executable, args, { + cwd: repoDir, + env, + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + }); + const stop = (reason: string): void => { + error ??= reason; + try { + if (child.pid && process.platform !== "win32") + process.kill(-child.pid, "SIGKILL"); + else child.kill("SIGKILL"); + } catch { + /* Already exited. */ + } + }; + const timer = setTimeout(() => stop("execution-timeout"), timeout); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + bytes += Buffer.byteLength(chunk); + if (bytes > MAX_OUTPUT) stop("output-limit"); + else stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + bytes += Buffer.byteLength(chunk); + if (bytes > MAX_OUTPUT) stop("output-limit"); + else stderr += chunk; + }); + child.on("error", (e) => { + error = e.message; + }); + child.on("close", (exit, signal) => { + clearTimeout(timer); + resolveResult({ + command: [executable, ...args], + exit, + signal, + error, + stdout, + stderr, + }); + }); + }); +} +const unavailable = (problem: string): Observation => ({ + problem, + samples: [], +}); +function successful(e: Execution): boolean { + return e.exit === 0 && !e.signal && !e.error; +} + +export function executionFailure(e: Execution): string { + return ( + e.error || + e.stderr.trim() || + (e.signal + ? `signal ${e.signal}` + : e.exit === null + ? "no exit status" + : `exit code ${e.exit}`) + ); +} + +/** Keep failed capture slots: a second image never masquerades as repeat zero. */ +export async function chromiumObservation( + execution: Execution, + read: (index: number) => Promise +): Promise { + const observation: Observation = { + samples: [], + invocation: execution, + problem: successful(execution) + ? null + : `Chromium capture failed: ${executionFailure(execution)}`, + }; + for (let i = 0; i < 2; i++) { + let image: ImageRecord | null = null; + try { + image = await read(i); + } catch (error) { + observation.problem ??= `Chromium output unavailable: ${String(error)}`; + } + observation.samples.push({ + execution, + diagnostics: execution.stderr.trim(), + image, + }); + } + return observation; +} + +export function firstSampleComparisons( + observations: Record, + images: ReadonlyMap +): Record { + const result: Record = {}; + const labels = ["strict", "best", "chromium", "baked", "stored", "resvg"]; + for (let i = 0; i < labels.length; i++) { + for (const right of labels.slice(i + 1)) { + const left = labels[i]; + const a = observations[left]?.samples[0]?.image, + b = observations[right]?.samples[0]?.image; + const leftImage = a ? images.get(a.path) : undefined, + rightImage = b ? images.get(b.path) : undefined; + if (leftImage && rightImage) + result[`${left}-${right}`] = compare(leftImage, rightImage); + } + } + return result; +} + +/** Remove only the CLI's exact known success footer for this invocation. + * Everything else, including unknown stderr/stdout, remains evidence. */ +export function cliDiagnostics( + e: Execution, + source: string, + out: string, + c: Pick, + bytes: number | null +): string { + const lines = e.stderr.trimEnd().split("\n"); + const last = lines.at(-1) ?? ""; + const prefix = `rendered ${source} -> ${out} (${c.width}x${c.height}, base-shared-frame, `; + if (e.exit === 0 && bytes !== null && last.startsWith(prefix)) { + const suffix = last.slice(prefix.length); + const declaredDegradations = suffix.match(/^(\d+) degraded, /); + const actualDegradations = lines.filter((line) => + line.startsWith("degraded: ") + ).length; + if ( + suffix === `${bytes} bytes)` || + (declaredDegradations !== null && + Number(declaredDegradations[1]) === actualDegradations && + suffix.replace(/^\d+ degraded, /, "") === `${bytes} bytes)`) + ) + lines.pop(); + } + return lines.join("\n").trimEnd(); +} +export interface CaseResult { + case: Case; + problems: string[]; + observations: Record; + pairs: Record; + verdict: Verdict; +} +export interface Report { + schema_version: 1; + kind: "svg-assertion-observations"; + manifest: Suite; + manifest_sha256: string; + source_manifest: string; + tools: Record; + integrity: string[]; + cases: CaseResult[]; + gate_ready: boolean; +} +export interface Options { + manifest: string; + out: string; + resvg?: { executable: string; sha256: string; version: string }; +} + +export async function run(options: Options): Promise { + const manifestPath = resolve(options.manifest), + base = dirname(manifestPath), + out = resolve(options.out); + const manifestBytes = await readBounded(manifestPath, 1024 * 1024); + const suite = parseSuite(JSON.parse(manifestBytes.toString("utf8"))); + // Caller supplies a NEW directory. An existing directory is never reused. + await mkdir(out); + const write = async (name: string, bytes: string | Buffer): Promise => { + await writeFile(join(out, name), bytes, { flag: "wx" }); + }; + await write("input-manifest.json", manifestBytes); + const watched = new Map([ + [manifestPath, sha256(manifestBytes)], + ]); + const inputLimits = new Map([[manifestPath, 1024 * 1024]]); + const integrity: string[] = []; + const tools: Record = { + node: process.version, + platform: process.platform, + arch: process.arch, + profile: suite.profile, + }; + for (const path of [ + capturePath, + ...[ + "model.ts", + "runner.ts", + "report.ts", + "capture-worker.ts", + "cli.ts", + ].map((name) => join(toolDir, name)), + ]) { + const hash = sha256(await readFile(path)); + watched.set(path, hash); + tools[relative(repoDir, path)] = hash; + } + if (watched.get(capturePath) !== suite.capture.sha256) + integrity.push("capture-module-hash-drift"); + tools.chromium_version = suite.capture.browser_version; + tools.git = await command("git", ["rev-parse", "HEAD"]); + const build = await command( + "cargo", + [ + "build", + "--locked", + "--message-format=json", + "-p", + "n0_cli", + "--bin", + "n0", + ], + // The pinned Linux GL/SVG/WebP combination has no prebuilt Skia archive; + // cold source compilation exceeds ten minutes on hosted CI. This build + // budget is separate from the one-minute render/capture process bound. + 20 * 60000 + ); + tools.n0_build = build; + let n0Path: string | null = null; + if (successful(build)) { + for (const line of build.stdout.split("\n")) { + try { + const artifact = JSON.parse(line); + if ( + artifact.reason === "compiler-artifact" && + artifact.target?.name === "n0" && + typeof artifact.executable === "string" + ) + n0Path = artifact.executable; + } catch { + /* Cargo may emit non-artifact lines. */ + } + } + } + if (!n0Path) integrity.push("n0-build-unavailable"); + else { + const hash = sha256(await readFile(n0Path)); + watched.set(n0Path, hash); + tools.n0_binary_sha256 = hash; + } + let resvgProblem = "fresh resvg not requested"; + if (options.resvg) { + try { + const path = resolve(options.resvg.executable), + hash = sha256(await readFile(path)); + watched.set(path, hash); + const version = await command(path, ["--version"]); + tools.resvg = { executable: path, sha256: hash, version }; + if ( + hash !== options.resvg.sha256 || + !successful(version) || + version.stdout.trim() !== options.resvg.version || + version.stderr + ) + resvgProblem = "resvg identity mismatch"; + else resvgProblem = ""; + } catch (error) { + resvgProblem = String(error); + } + } + async function image( + path: string, + retain?: Map + ): Promise<{ record: ImageRecord; bytes: Buffer }> { + const bytes = await readBounded(path, MAX_PNG_BYTES), + decoded = decode(bytes), + name = relative(out, path); + retain?.set(name, decoded); + return { + record: { + path: name, + png_sha256: sha256(bytes), + rgba_sha256: sha256(decoded.rgba), + width: decoded.width, + height: decoded.height, + }, + bytes, + }; + } + async function input(id: FileIdentity, limit: number): Promise { + const path = resolve(base, id.path), + bytes = await readBounded(path, limit); + const hash = sha256(bytes); + inputLimits.set(path, limit); + if (watched.has(path) && watched.get(path) !== hash) + integrity.push(`input changed between reads: ${id.path}`); + else watched.set(path, hash); + if (hash !== id.sha256) { + integrity.push(`declared input hash mismatch: ${id.path}`); + throw new Error(`input hash mismatch: ${id.path}`); + } + return bytes; + } + async function reference( + c: Case, + id: FileIdentity | null, + label: string, + images: Map + ): Promise { + if (!id) return unavailable(`${label} not supplied`); + try { + const bytes = await input(id, MAX_PNG_BYTES), + name = `${c.id}/${label}.png`; + await write(name, bytes); + const { record } = await image(join(out, name), images); + const sample: Sample = { + execution: { + command: ["stored-file", id.path], + exit: 0, + signal: null, + error: null, + stdout: "", + stderr: "", + }, + diagnostics: "", + image: record, + }; + return { samples: [sample], problem: null }; + } catch (error) { + return unavailable(String(error)); + } + } + const results: CaseResult[] = []; + for (const c of suite.cases) { + // Only first samples of this case are needed for its named comparisons. + // Repeats retain hashes, never decoded buffers; completed cases release + // their pixels. Cross-case controls are reloaded one pair at a time below. + const images = new Map(); + await mkdir(join(out, c.id)); + const problems: string[] = []; + const observations: Record = {}; + const result: CaseResult = { + case: c, + problems, + observations, + pairs: {}, + verdict: { status: "OBSERVATION", kind: "observation", reasons: [] }, + }; + results.push(result); + const sourcePath = join(out, c.id, "source.svg"); + let sourceReady = false; + try { + const bytes = await input(c.source, 1024 * 1024); + new TextDecoder("utf-8", { fatal: true }).decode(bytes); + await write(`${c.id}/source.svg`, bytes); + watched.set(sourcePath, sha256(bytes)); + inputLimits.set(sourcePath, 1024 * 1024); + sourceReady = true; + } catch (error) { + problems.push(String(error)); + } + observations.baked = await reference( + c, + c.assertion?.kind === "render-exact" ? c.assertion.reference : null, + "baked", + images + ); + observations.stored = await reference(c, c.stored, "stored", images); + if (c.assertion?.kind === "render-exact" && observations.baked.problem) + problems.push(observations.baked.problem); + if (c.stored && observations.stored.problem) + problems.push(observations.stored.problem); + for (const admission of ["strict", "best"] as const) { + const observation: Observation = { samples: [], problem: null }; + observations[admission] = observation; + if (!sourceReady || !n0Path) { + observation.problem = "source or n0 unavailable"; + continue; + } + for (let i = 0; i < 2; i++) { + const output = join(out, c.id, `${admission}-${i}.png`); + const execution = await command("cargo", [ + "run", + "--locked", + "--quiet", + "-p", + "n0_cli", + "--bin", + "n0", + "--", + sourcePath, + output, + `${c.width}x${c.height}`, + admission === "strict" ? "--strict" : "--best-effort", + ]); + let record: ImageRecord | null = null, + bytes: Buffer | null = null; + try { + const rendered = await image(output, i === 0 ? images : undefined); + record = rendered.record; + bytes = rendered.bytes; + } catch (error) { + if ( + execution.exit === 0 || + (error as NodeJS.ErrnoException).code !== "ENOENT" + ) + execution.error ??= `missing/invalid output: ${String(error)}`; + } + observation.samples.push({ + execution, + image: record, + diagnostics: cliDiagnostics( + execution, + sourcePath, + output, + c, + bytes?.length ?? null + ), + }); + } + } + const chromium: Observation = { samples: [], problem: null }; + observations.chromium = chromium; + if (!sourceReady || integrity.includes("capture-module-hash-drift")) + chromium.problem = "source or pinned capture unavailable"; + else { + const execution = await command(process.execPath, [ + "--import", + "tsx", + join(toolDir, "capture-worker.ts"), + sourcePath, + join(out, c.id), + String(c.width), + String(c.height), + suite.capture.browser_version, + ]); + observations.chromium = await chromiumObservation( + execution, + async (i) => + ( + await image( + join(out, c.id, `chromium-${i}.png`), + i === 0 ? images : undefined + ) + ).record + ); + } + const resvg: Observation = { samples: [], problem: null }; + observations.resvg = resvg; + if ( + resvgProblem || + !sourceReady || + c.review.status !== "reviewed" || + c.review.blockers.length + ) + resvg.problem = + resvgProblem || "source/environment not reviewed for local resvg"; + else if (options.resvg) + for (let i = 0; i < 2; i++) { + const output = join(out, c.id, `resvg-${i}.png`); + const execution = await command(resolve(options.resvg.executable), [ + "--skip-system-fonts", + "--width", + String(c.width), + sourcePath, + output, + ]); + let record: ImageRecord | null = null; + try { + record = (await image(output, i === 0 ? images : undefined)).record; + } catch (error) { + execution.error ??= `missing/invalid output: ${String(error)}`; + } + resvg.samples.push({ + execution, + diagnostics: execution.stderr.trim(), + image: record, + }); + } + if (!resvg.problem) resvg.problem = repeatProblem(resvg); + result.pairs = firstSampleComparisons(observations, images); + } + for (const result of results) { + const c = result.case; + if (c.assertion?.kind === "render-exact") { + const id = c.assertion.control.case, + control = results.find((r) => r.case.id === id); + const a = result.observations.chromium.samples[0]?.image, + b = control?.observations.chromium.samples[0]?.image; + if ( + a && + b && + control && + !control.problems.length && + control.case.review.status === "reviewed" && + !control.case.review.blockers.length && + !repeatProblem(control.observations.chromium) && + control.observations.chromium.samples.every( + (s) => + successful(s.execution) && !s.diagnostics && !s.execution.stdout + ) + ) { + try { + const reload = async (record: ImageRecord): Promise => { + const bytes = await readBounded( + join(out, record.path), + MAX_PNG_BYTES + ); + if (sha256(bytes) !== record.png_sha256) + throw new Error(`control output changed: ${record.path}`); + return decode(bytes); + }; + result.pairs["chromium-control"] = compare( + await reload(a), + await reload(b) + ); + } catch (error) { + result.problems.push(String(error)); + } + } + } + } + for (const [path, hash] of watched) { + try { + const limit = inputLimits.get(path); + const bytes = + limit === undefined + ? await readFile(path) + : await readBounded(path, limit); + if (sha256(bytes) !== hash) + integrity.push( + `input/tool changed during run: ${relative(repoDir, path)}` + ); + } catch { + integrity.push( + `input/tool disappeared during run: ${relative(repoDir, path)}` + ); + } + } + for (const result of results) { + result.verdict = evaluate(result.case, { + problems: [...integrity, ...result.problems], + strict: result.observations.strict, + best: result.observations.best, + chromium: result.observations.chromium, + pairs: result.pairs, + }); + } + const verdicts = Object.fromEntries( + results.map((r) => [r.case.id, r.verdict]) + ); + const report: Report = { + schema_version: 1, + kind: "svg-assertion-observations", + manifest: suite, + manifest_sha256: sha256(manifestBytes), + source_manifest: manifestPath, + tools, + integrity, + cases: results, + gate_ready: gateReady(suite.cases, verdicts, integrity), + }; + await write("report.json", `${JSON.stringify(report, null, 2)}\n`); + await write("index.html", renderReport(report)); + return report; +} diff --git a/packages/grida-reftest/svg-assertions/tsconfig.json b/packages/grida-reftest/svg-assertions/tsconfig.json new file mode 100644 index 00000000..6c3e18ef --- /dev/null +++ b/packages/grida-reftest/svg-assertions/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "lib": ["es2022", "dom"], + "paths": { "@playwright/test": ["../node_modules/@playwright/test"] } + }, + "include": ["*.ts"], + "exclude": [] +} diff --git a/packages/grida-reftest/tsconfig.json b/packages/grida-reftest/tsconfig.json index 038db948..d68b8ffe 100644 --- a/packages/grida-reftest/tsconfig.json +++ b/packages/grida-reftest/tsconfig.json @@ -11,5 +11,6 @@ "skipLibCheck": true, "resolveJsonModule": true, "isolatedModules": true - } + }, + "exclude": ["node_modules", "svg-assertions"] }