Repository navigation
Expand file tree
/
Copy pathrender-and-diff.mjs
More file actions
1354 lines (1280 loc) · 56 KB
/
Copy pathrender-and-diff.mjs
File metadata and controls
1354 lines (1280 loc) · 56 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env node
/**
* scripts/render-and-diff.mjs — one loop pass, one command.
*
* node scripts/render-and-diff.mjs --project <id> --revision <id> [--root <ws>]
* [--against reference|parent] [--skip-render] [--json]
*
* Every pass of the iteration loop runs the same deterministic chain: render,
* scale the reference to the render's size, diff, check that the links in the
* data are live in the PDF and that the document itself is whole, write the
* evidence into the revision, ask whether the loop may continue. The serif
* acceptance run paid for that as three to four separate model turns per pass —
* and solved the scaling step itself, with ImageMagick shell arithmetic that
* left junk files in the user's project root.
*
* The link and document checks are here rather than at approval time because
* they are what the diff cannot see. An annotation has no pixels, so a document
* whose every link is dead diffs identically to one where they all work; and a
* page reading "Page 1 of 1" in a three-page document is forty grey pixels.
*
* This is that chain as one call. The agent's part of a pass — look at the
* images, judge, decide the one cause to fix — is untouched; what is removed
* is the taxi ride between the tools.
*
* The exit code is the loop's own verdict, so a skill can branch without
* parsing anything:
*
* 0 READY_FOR_APPROVAL stop and hand over to the user
* 2 REVISE keep going; the JSON names the focus
* 4 CONVERGENCE_LIMIT_REACHED the loop spent its budget with work open
* 3 BLOCKED stop; no usable document can be produced
* 1 a step failed (compile error, missing image, …)
*/
import fs from "node:fs";
import path from "node:path";
import { spawnSync } from "node:child_process";
import {
describeWorkspaceLine,
installRoot,
projectDir as workspaceProjectDir,
resolveWorkspace,
} from "./lib/workspace.mjs";
import { createRequire } from "node:module";
import { pagePairs } from "./lib/page-pairs.mjs";
import { describeAttempts, readAttempts, recordAttempt } from "./lib/attempts.mjs";
import { compareEdgeBands } from "./lib/edge-bands.mjs";
import { describeIgnoredCopies, resolveTemplateSource } from "./lib/template-source.mjs";
import { INFERRED_NEW_SCOPE, resolveScope } from "./lib/pipeline-config.mjs";
const repoRoot = installRoot();
function usage(code = 0) {
process.stdout.write(
"usage: node scripts/render-and-diff.mjs --project <id> --revision <id> [--root <workspace>]\n" +
" [--against reference|parent] [--skip-render] [--json]\n\n" +
" --project <id> the project\n" +
" --revision <id> the revision to render and judge\n" +
" --against <what> what to diff the render against (default: reference;\n" +
" parent diffs against the parent revision's output.png,\n" +
" which is what refactor-only and data-only gates need)\n" +
" --skip-render reuse the existing output.png (diff and verdict only)\n" +
" --root <workspace> workspace override (default: discovered)\n" +
" --json machine-readable result\n\n" +
"exit: 0 ready for approval | 2 revise | 4 convergence limit | 3 blocked | 1 a step failed\n",
);
process.exit(code);
}
function parseArgs(argv) {
const out = { project: null, revision: null, root: null, against: "reference", skipRender: false, json: false };
for (let i = 0; i < argv.length; i += 1) {
const a = argv[i];
if (a === "--help" || a === "-h") usage(0);
else if (a === "--json") out.json = true;
else if (a === "--skip-render") out.skipRender = true;
else if (a === "--project" || a === "-p") out.project = argv[++i];
else if (a === "--revision" || a === "-r") out.revision = argv[++i];
else if (a === "--against") out.against = argv[++i];
else if (a === "--root") out.root = argv[++i];
else {
process.stderr.write(`[render-and-diff] unknown argument: ${a}\n`);
usage(2);
}
}
if (!out.project || !out.revision) {
process.stderr.write("[render-and-diff] --project and --revision are required\n");
usage(2);
}
if (!["reference", "parent"].includes(out.against)) {
process.stderr.write("[render-and-diff] --against must be reference or parent\n");
usage(2);
}
return out;
}
/**
* How many regions the pass classifies. Three, because the ranking is by
* concentration and the tail is where even wear lives: a fourth package costs
* context to say "this region differs about as much as the page does".
*/
const EVIDENCE_REGIONS = 3;
const args = parseArgs(process.argv.slice(2));
const workspace = resolveWorkspace({ explicitRoot: args.root ?? null });
const banner = describeWorkspaceLine(workspace);
if (banner && !args.json) console.log(banner);
const projectDir = workspaceProjectDir(workspace, args.project);
const revisionDir = path.join(projectDir, "revisions", args.revision);
const result = {
project: args.project,
revision: args.revision,
steps: [],
diff: null,
regions: null,
loop: null,
};
/**
* Page 1's image pair, kept so the region step measures the same two images
* the page number came from rather than re-deriving them and risking a
* different pair.
*/
let pagePair1 = null;
/**
* The visual-analysis.json governing this revision.
*
* A `visual-change` revision writes its own. The narrow scopes — data-only,
* asset-only, refactor-only — deliberately skip the analyser, so theirs is the
* nearest ancestor's: the regions did not move, which is the entire premise of
* those scopes. Walking up rather than giving up is what lets the region gate
* apply to exactly the scopes config/pipeline.json assigns it to.
*/
function nearestVisualAnalysis(startDir) {
let dir = startDir;
const seen = new Set();
while (dir && !seen.has(dir)) {
seen.add(dir);
const candidate = path.join(dir, "visual-analysis.json");
if (fs.existsSync(candidate)) return candidate;
const revision = (() => {
try {
return JSON.parse(fs.readFileSync(path.join(dir, "revision.json"), "utf8"));
} catch {
return null;
}
})();
if (!revision?.parentRevisionId) return null;
dir = path.join(projectDir, "revisions", revision.parentRevisionId);
}
return null;
}
/**
* Which file the project itself calls page 1 of the reference.
*
* Older projects predate `import-reference` and name it something else; the
* manifest is the only place that knows, so it is asked rather than assumed.
*/
function projectReferenceImage() {
const file = path.join(projectDir, "template-project.json");
if (!fs.existsSync(file)) return null;
try {
const declared = JSON.parse(fs.readFileSync(file, "utf8")).referenceImage;
return declared ? path.join(projectDir, declared) : null;
} catch {
// A malformed manifest is reported by the tools that own it; here it just
// means falling back to the canonical name.
return null;
}
}
/**
* The public class the project declares, simple name only, or null.
*
* The same field `publish-template.mjs` reads, so the two resolve the revision's
* template to the same file. A project that declares nothing is not a fault: the
* flow's own name is then the only candidate.
*/
function declaredTemplateClass() {
const file = path.join(projectDir, "template-project.json");
if (!fs.existsSync(file)) return null;
try {
const meta = JSON.parse(fs.readFileSync(file, "utf8"));
const declared = meta.render?.templateClass ?? meta.templateClass;
return typeof declared === "string" && declared.trim() !== "" ? declared.trim().split(".").pop() : null;
} catch {
return null;
}
}
/** The revision's overflow dataset, by name, or null. */
/**
* Has the page size been measured and answered? `page-size.mjs` owns the
* question (exit 0 settled, 5 unanswered); a failure to ask is read as settled
* so a missing tool cannot turn every pass into a page-size question.
*/
function pageSizeSettled() {
const asked = run(path.join(repoRoot, "scripts", "page-size.mjs"), [
"--project", args.project,
"--root", workspace.root,
"--json",
]);
if (asked.status === 5) return false;
if (asked.status !== 0) return true;
try {
return JSON.parse(asked.stdout).settled !== false;
} catch {
return true;
}
}
function overflowFixture() {
if (!fs.existsSync(revisionDir)) return null;
const found = fs.readdirSync(revisionDir).filter((f) => /-data\.overflow\.json$/.test(f));
return found.length ? found[0] : null;
}
function step(name, fn) {
const entry = { name, ok: false };
result.steps.push(entry);
try {
fn(entry);
entry.ok = true;
} catch (cause) {
entry.error = cause.message;
finish(1);
}
}
function run(command, runArgs) {
const spawned = spawnSync(process.execPath, [command, ...runArgs], { encoding: "utf8" });
return {
status: spawned.status,
stdout: spawned.stdout ?? "",
output: `${spawned.stdout ?? ""}${spawned.stderr ?? ""}`,
};
}
/** Lines that say what went wrong, as opposed to what was happening at the time. */
const ERROR_SHAPED =
/(\berror\b|\bexception\b|\bcannot\b|\brequired\b|\bfailed\b|\bmissing\b|BUILD FAILURE|Caused by|^\s+at\s)/i;
/**
* The part of a failed command's output that explains the failure.
*
* ## Why this is not just the tail
*
* A create run hit a compile error and was shown this and nothing else:
*
* FAIL render
* render failed:
* [asset-resolver] cache HIT mdi:heart (svg) -> cf1179b29151
* [asset-resolver] icon "heart": mdi:heart (explicit) -> heart.svg
*
* Both lines are progress chatter. The compiler's complaint was in the output
* and never reached the reader, so the run re-invoked `render.mjs` on its own to
* find out what had happened — two extra turns, on every failure.
*
* Taking the last N lines assumes the error is last. It often is not: resolvers
* and caches keep talking after the thing that failed. So error-shaped lines are
* preferred, and the tail is what happens when none are found — which is the
* right fallback, because an unrecognised failure is exactly when raw output is
* worth more than a filter's opinion of it.
*
* @param {string} output combined stdout and stderr
* @param {number} [cap] most lines to return
*/
function excerptFailure(output, cap = 15) {
const lines = (output ?? "").trim().split("\n");
const signal = lines.filter((line) => ERROR_SHAPED.test(line));
// `[workspace] …` is printed by every script on every run, and it matches
// "required" often enough through paths to crowd out the real line.
const meaningful = signal.filter((line) => !/^\s*\[workspace\]/.test(line));
const chosen = meaningful.length > 0 ? meaningful : signal.length > 0 ? signal : lines;
return chosen.slice(-cap).join("\n");
}
function finish(code) {
if (args.json) {
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
} else {
for (const entry of result.steps) {
console.log(` ${entry.ok ? "ok " : "FAIL"} ${entry.name}${entry.detail ? ` ${entry.detail}` : ""}`);
// The whole explanation, not the first three lines of it. A failed step is
// the reason this command has anything to say, and trimming its account to
// keep the summary tidy is what sent one run off to re-render by hand just
// to read an error it had already been handed.
if (entry.error) console.log(` ${entry.error.split("\n").join("\n ")}`);
}
if (result.diff) {
console.log(
`\n diff: ${result.diff.mismatchPx} px (${result.diff.percent.toFixed(3)}%) — ${result.diff.classification}` +
(result.diff.perceptual
? ` · perceptual ${result.diff.perceptual.ssim} — ${result.diff.perceptual.classification} (provisional)`
: ""),
);
}
for (const d of result.document?.defects ?? []) {
console.log(` DOCUMENT DEFECT ${d.id}: ${d.detail}`);
}
for (const n of result.document?.notes ?? []) {
console.log(` note ${n}`);
}
for (const m of result.links?.missing ?? []) {
console.log(` MISSING LINK ${m.at} = ${m.target} (declared in the data, absent from the render)`);
}
for (const c of result.links?.undeclared ?? []) {
console.log(` warn ${c.at} = ${c.value} (link-shaped, no href recorded)`);
}
if (result.loop) {
console.log(` loop: ${result.loop.verdict}` + (result.loop.next ? ` — ${result.loop.next}` : ""));
}
console.log("");
}
process.exit(code);
}
// -------------------------------------------------------------------- steps ---
// Before anything: the barrier authoring was told to run, run by the pass that
// would otherwise compile past it. `create-3-author.md` says "exit 1 means do
// not start", and a sentence enforces nothing — this is where the refusal has a
// mechanism. config/pipeline.json declares it under `barriers.analysis`; the
// conditions here are that declaration's `appliesWhen`.
step("discovery", (entry) => {
let revision = null;
try {
revision = JSON.parse(fs.readFileSync(path.join(revisionDir, "revision.json"), "utf8"));
} catch {
/* no revision record: the render step reports that in its own words */
}
const scope = resolveScope({ revision, revisionId: args.revision });
if (scope !== INFERRED_NEW_SCOPE) {
entry.detail = `not a first revision (${scope}) — discovery was joined before its parent rendered`;
return;
}
// The first render only. Once one has succeeded the join was checked before
// it; re-rendering the same revision is not the moment discovery changes.
if (fs.existsSync(path.join(revisionDir, "output.png"))) {
entry.detail = "already rendered once — the join was checked before that render";
return;
}
// A project that never wrote the discovery artifacts — the examples that
// predate them — is not held to a barrier it did not enter. The moment
// visual-analysis.json exists the fan-out has begun, and the join applies.
if (!fs.existsSync(path.join(revisionDir, "visual-analysis.json"))) {
entry.detail = "no discovery artifacts in this revision — not held";
return;
}
const checked = run(path.join(repoRoot, "scripts", "check-analysis.mjs"), [
"--project",
args.project,
"--revision",
args.revision,
"--root",
workspace.root,
"--for",
"authoring",
"--json",
]);
let analysis;
try {
analysis = JSON.parse(checked.stdout);
} catch {
// The barrier could not answer at all. Held, not waved through: a check
// that cannot run is not a check that passed.
throw new Error(`check-analysis did not answer:\n${excerptFailure(checked.output, 6)}`);
}
const held = analysis.artifacts.filter((a) => !a.ok).map((a) => `${a.name}: ${a.detail}`);
result.discovery = { complete: analysis.complete, held };
if (!analysis.complete) {
throw new Error(
`authoring started on unfinished discovery — ${held.join("; ")}\n` +
`run: node scripts/check-analysis.mjs --project ${args.project} --revision ${args.revision} --for authoring`,
);
}
entry.detail = `${analysis.artifacts.length} artifact(s) complete`;
});
// Before the compile, not after: a second copy of the template costs a pass its
// editing budget, and the cheapest moment to say so is before the budget is
// spent. See lib/template-source.mjs for the run that paid it four times.
step("template source", (entry) => {
const resolved = resolveTemplateSource({ revisionDir, canonicalName: declaredTemplateClass() });
if (!resolved.file) {
// No template yet is the ordinary state before the first authoring pass.
entry.detail = "not written yet";
return;
}
result.template = {
name: resolved.name,
ignored: resolved.ignored.map((c) => ({ name: c.name, divergent: c.divergent })),
divergent: resolved.divergent,
};
const note = describeIgnoredCopies(resolved);
entry.detail = note ? note : `${resolved.name} — one file, which is the build's`;
});
if (!args.skipRender) {
step("render", (entry) => {
const rendered = run(path.join(repoRoot, "scripts", "render.mjs"), [
args.project,
args.revision,
"--root",
workspace.root,
]);
if (rendered.status !== 0) {
throw new Error(`render failed:\n${excerptFailure(rendered.output, 15)}`);
}
entry.detail = "output.pdf + output.png (clean + debug)";
});
step("overflow fixture", (entry) => {
// A second dataset through the same template, when the revision carries
// one. It is the only place a flowing document's page break, repeated
// header and page numbering are ever rendered: the revision's own data
// mirrors the reference, and a reference is a sample that fits.
const fixture = overflowFixture();
if (!fixture) {
entry.detail = "none in this revision";
return;
}
const rendered = run(path.join(repoRoot, "scripts", "render.mjs"), [
args.project,
args.revision,
"--root",
workspace.root,
"--data-file",
fixture,
"--suffix",
"-overflow",
]);
if (rendered.status !== 0) {
throw new Error(`the overflow fixture did not render:\n${excerptFailure(rendered.output, 12)}`);
}
entry.detail = `${fixture} -> output-overflow.pdf`;
});
} else {
step("render (skipped)", (entry) => {
if (!fs.existsSync(path.join(revisionDir, "output.png"))) {
throw new Error("--skip-render was given but the revision has no output.png");
}
entry.detail = "reusing existing output.png";
});
}
step("diff", (entry) => {
let parentDir = null;
if (args.against === "parent") {
const revision = JSON.parse(fs.readFileSync(path.join(revisionDir, "revision.json"), "utf8"));
if (!revision.parentRevisionId) {
throw new Error("--against parent, but this revision has no parent");
}
parentDir = path.join(projectDir, "revisions", revision.parentRevisionId);
if (!fs.existsSync(path.join(parentDir, "output.png"))) {
throw new Error(`the parent revision has no output.png: ${parentDir}`);
}
}
// Every page the source has, not only the first. A reference can be a
// proposal or a book; comparing page 1 and stopping left the rest of the
// document measured by nobody, while both sides had rasters on disk.
const referenceDir = path.join(projectDir, "reference");
const pages = pagePairs({
referenceDir,
referenceImage: projectReferenceImage(),
revisionDir,
parentDir,
against: args.against,
});
if (!pages.pairs.length) {
throw new Error(
args.against === "parent"
? `the parent revision has no page to compare against: ${parentDir}`
: `no reference image found under ${referenceDir}`,
);
}
pagePair1 = pages.pairs.find((p) => p.page === 1) ?? null;
const perPage = [];
for (const pair of pages.pairs) {
// ALWAYS the source image, never the scaled copy a previous pass left.
//
// Preferring the persisted copy switched off the aspect-mismatch warning
// on every run after the first: `reference-scaled.png` already has the
// render's exact dimensions, so `--scale-reference` finds nothing to
// scale, skips the measurement, and rewrites visual-diff-stats.json
// WITHOUT `aspectMismatch`. The distortion was still in the pixels being
// compared — only the notice that it was there disappeared, on the second
// render of the same revision. A backstop that turns itself off on retry
// is worse than none, because the first run taught you to trust it.
//
// Costs nothing: the scaler is deterministic by design ("the same every
// run, so diff numbers are comparable across passes"), so re-scaling
// reproduces the identical file it would have reused.
const source = pair.reference;
const diffArgs = [source, pair.render, "--json", "--out", pair.diff];
if (pair.page === 1) {
// Page 1 alone writes the revision's diff artifacts. Everything
// downstream reads those names, and having page 3 overwrite them would
// make the recorded stats depend on page order.
diffArgs.push("--update-revision", revisionDir);
}
if (args.against === "reference") {
// Scaling is for the reference comparison only. A parent comparison is
// same-renderer, same-size by construction — if the sizes differ there,
// something real changed and the diff must fail loudly, not resample.
diffArgs.push("--scale-reference", "--save-scaled", pair.scaled);
} else {
// The parent gates are equality gates: `exact-diff` passes at AE == 0 and
// `region-diff` demands AE == 0 outside the changed regions. pixelmatch's
// default threshold (0.1) forgives small per-pixel colour distance, which
// is right against a rasterised design and wrong here — two renders of
// the same renderer differ only where something changed, and a gate that
// reads "0" while forgiving a shifted anti-aliased edge is not the gate
// the docs quote.
// And with anti-aliased pixels counted: pixelmatch's detector would
// otherwise discard a shifted glyph edge as "anti-aliasing", and an
// exact gate that forgives a one-pixel text shift is not exact.
diffArgs.push("--threshold", "0", "--include-aa");
}
const diffed = run(path.join(repoRoot, "tools", "visual-diff", "bin", "visual-diff.mjs"), diffArgs);
if (diffed.status !== 0) {
throw new Error(diffed.output.trim() || `diff failed on page ${pair.page}`);
}
const stats = JSON.parse(diffed.stdout);
perPage.push({
page: pair.page,
mismatchPx: stats.mismatchPx,
percent: stats.percent,
classification: stats.classification,
parityScore: stats.parityScore,
// The perceptual similarity beside the pixel count: SSIM over the
// downsampled luminance, which anti-aliasing does not inflate. Over the
// audited corpus the pixel count sat at 5–12% on every revision; this
// ordered them from 0.44 (one page against two) to 0.95 (approved).
perceptual: stats.perceptual
? { ssim: stats.perceptual.ssim, classification: stats.perceptual.classification, worstWindow: stats.perceptual.worstWindow ?? null }
: null,
diffImage: stats.diff,
// Present only when --scale-reference changed the reference's SHAPE and
// not just its size. It travels with the numbers because it is a fact
// about them: a stretched reference makes the mismatch look smaller than
// it is, so a percentage carrying this field understates the difference
// and cannot be classified until the page size is settled.
...(stats.aspectMismatch ? { aspectMismatch: stats.aspectMismatch } : {}),
});
}
const first = perPage[0];
// By share, not by raw pixel count. Pages are not obliged to be the same
// size — a continuation page rendered at a different dpi has a different
// denominator — and "worst" should mean the page furthest from its
// reference, not the page with the most pixels in it.
const worst = perPage.reduce((a, b) => (b.percent > a.percent ? b : a));
result.diff = {
against: args.against,
// Page 1's numbers keep their place at the top level: every consumer of
// this report already reads them there, and a multi-page document does not
// change what page 1 scored.
mismatchPx: first.mismatchPx,
percent: first.percent,
classification: first.classification,
parityScore: first.parityScore,
perceptual: first.perceptual ?? null,
diffImage: first.diffImage,
referencePages: pages.referencePages,
renderPages: pages.renderPages,
pages: perPage,
worstPage: worst.page,
missingFromRender: pages.missingFromRender,
extraInRender: pages.extraInRender,
// At the top level too, and as a list of pages rather than a boolean: a
// reader who takes `percent` from here and stops must still be told that
// the number was measured on a reference that had been stretched into a
// shape it did not have. The page size is wrong, not the layout, and the
// percentage is smaller than the truth. See docs/visual-accuracy-contract.md.
...(perPage.some((p) => p.aspectMismatch)
? {
aspectMismatchPages: perPage
.filter((p) => p.aspectMismatch)
.map((p) => ({ page: p.page, ...p.aspectMismatch })),
}
: {}),
};
entry.detail =
perPage.length === 1
? `${first.mismatchPx} px vs ${args.against} — ${first.classification}`
: `${perPage.length} pages vs ${args.against} — worst is page ${worst.page} ` +
`(${worst.mismatchPx} px, ${worst.classification})` +
(pages.missingFromRender.length
? `; page(s) ${pages.missingFromRender.join(", ")} missing from the render`
: "");
});
step("regions", (entry) => {
// Where the difference lives, not just how much of it there is.
//
// A whole-page percentage against a rasterised design reference is never
// zero and is dominated by glyph anti-aliasing, so the only thing a reviewer
// can do with it is explain it — and an explanation that covers 9.7% covers
// a structural defect hiding inside the same number. Regions disagree with
// each other, which is what makes them checkable: even wear puts every
// region's share of the diff near its share of the page, and a defect drives
// one of them well above it.
//
// Evidence, not a gate: the ranking says where to look. `region-diff
// --changed` is the gate, and it is what the data-only and asset-only scopes
// are supposed to end on.
const analysis = nearestVisualAnalysis(revisionDir);
if (!analysis) {
entry.detail = "no visual-analysis.json in this revision or its ancestors";
return;
}
const page1 = (result.diff?.pages ?? []).length ? pagePair1 : null;
if (!page1) {
entry.detail = "no page 1 pair to measure";
return;
}
const measured = run(
path.join(repoRoot, "tools", "visual-diff", "bin", "region-diff.mjs"),
[
"--reference",
// The scaled copy when the diff above made one: the regions must be cut
// from the same pair of images the page number came from, or the shares
// are computed against a denominator nobody else used.
args.against === "reference" && fs.existsSync(page1.scaled) ? page1.scaled : page1.reference,
"--output",
page1.render,
"--regions-file",
analysis,
"--write",
revisionDir,
"--json",
],
);
if (measured.status !== 0) {
// Never fatal. A region measurement that cannot run is a missing view of
// a comparison that already succeeded, and failing the pass here would
// make a diagnostic into a blocker.
entry.detail = `not measured: ${(measured.output || "").trim().split("\n")[0] || "failed"}`;
return;
}
let regions;
try {
regions = JSON.parse(measured.stdout);
} catch {
entry.detail = "not measured: region-diff produced no JSON";
return;
}
const ranked = regions.ranked
.map((id) => regions.regions.find((r) => r.id === id))
.filter((r) => r && r.mismatchPx > 0)
// A region covering the whole page restates the page figure; it is true
// and it is not a location, so it does not belong at the top of a list
// whose whole purpose is to point somewhere.
.filter((r) => r.shareOfPageArea < 90);
// Two rankings, because concentration alone sent the evidence to the wrong
// places. In 13 of 14 multi-revision projects the three "worst" regions
// were the same three hairline dividers every pass — each at 40–75% of its
// own area but 0.1–1.5% of the page's difference — while the regions
// carrying the mass (line items, a hero, a table) ranked below them.
// `ranked` keeps the concentration order for reading; `byMass` is what the
// evidence is built on: share of the page's difference × concentration,
// over regions large enough to be a layout rather than a rule. The rules go
// to `hairlines`, which check-border-topology exists for.
const HAIRLINE_AREA_PERCENT = 0.5;
const byMass = ranked
.filter((r) => r.shareOfPageArea >= HAIRLINE_AREA_PERCENT)
.map((r) => ({ id: r.id, mass: (r.shareOfPageMismatch ?? 0) * (r.concentration ?? 0) }))
.sort((a, b) => b.mass - a.mass)
.map((r) => r.id);
const hairlines = ranked.filter((r) => r.shareOfPageArea < HAIRLINE_AREA_PERCENT).map((r) => r.id);
result.regions = {
analysis: path.relative(revisionDir, analysis) || path.basename(analysis),
source: `${regions.width}x${regions.height}`,
pageMismatchPx: regions.pageMismatchPx,
stats: "region-diff-stats.json",
byMass,
hairlines,
ranked: ranked.map((r) => ({
id: r.id,
role: r.role ?? null,
mismatchPx: r.mismatchPx,
percentOfRegion: Number(r.percent.toFixed(2)),
shareOfPageMismatch: Number(r.shareOfPageMismatch.toFixed(1)),
shareOfPageArea: Number(r.shareOfPageArea.toFixed(1)),
concentration: r.concentration === null ? null : Number(r.concentration.toFixed(2)),
})),
};
const worst = ranked[0];
entry.detail = worst
? `${ranked.length} regions measured — worst is ${worst.id} at ` +
`${worst.concentration.toFixed(2)}x its share of the page ` +
`(${worst.mismatchPx} px, ${worst.percent.toFixed(2)}% of the region)`
: `${regions.regions.length} regions measured — none carries a difference`;
});
step("furniture", (entry) => {
// The page's edges, compared: the lowest band of ink in the reference's
// bottom strip against the render's, and the highest in the top strip.
// Four of sixteen audited invoices were approved with the page number lower
// than the reference's and corrected only when a person said so; a pixel
// percentage cannot see a dozen grey rows, and a region ranking will not
// rank a band nobody drew a region around. One subtraction sees it.
if (args.against !== "reference" || !pagePair1) {
entry.detail = "reference comparison only";
return;
}
const scaled = fs.existsSync(pagePair1.scaled) ? pagePair1.scaled : null;
if (!scaled) {
entry.detail = "no scaled reference to compare against";
return;
}
let compared;
try {
const { PNG } = createRequire(path.join(repoRoot, "tools", "visual-diff", "package.json"))("pngjs");
const reference = PNG.sync.read(fs.readFileSync(scaled));
const render = PNG.sync.read(fs.readFileSync(pagePair1.render));
compared = compareEdgeBands(reference, render);
} catch (err) {
entry.detail = `not compared: ${err.message}`;
return;
}
result.furniture = {
tolerancePx: compared.tolerancePx,
top: compared.top ? { delta: compared.top.delta, reference: compared.top.reference, render: compared.top.render } : null,
bottom: compared.bottom ? { delta: compared.bottom.delta, reference: compared.bottom.reference, render: compared.bottom.render } : null,
defects: compared.defects,
};
entry.detail = compared.defects.length
? compared.defects.map((d) => d.id).join(", ")
: `top ${compared.top?.delta ?? "—"}px, bottom ${compared.bottom?.delta ?? "—"}px — within ${compared.tolerancePx}px of the reference`;
});
step("source change", (entry) => {
// How much of the template stopped being the one that was reviewed.
//
// Two failures share this missing fact. A pass hit a write conflict, deleted
// the template and regenerated 1,103 lines; and a revision replaced the whole
// page construction — nested rows and a timeline for tables and an accent
// border — and was recorded as another visual change. On disk both look like
// any other pass. A share of the methods is the cheapest honest measure, and
// it needs no judgement about intent.
//
// Evidence, never a gate: a rewrite can be exactly the right thing to do. The
// point is that it should be visible in the chain rather than discovered
// later by reading the diff.
const changed = run(path.join(repoRoot, "scripts", "source.mjs"), [
"diff",
"--project", args.project,
"--revision", args.revision,
"--root", workspace.root,
"--json",
]);
if (changed.status !== 0) {
entry.detail = (changed.output || "").trim().split("\n")[0] || "not compared";
return;
}
let diff;
try {
diff = JSON.parse(changed.stdout);
} catch {
entry.detail = "not compared: source diff produced no JSON";
return;
}
result.source = {
against: diff.against,
touchedShare: diff.touchedShare,
changed: diff.changed,
added: diff.added,
removed: diff.removed,
unchangedCount: diff.unchanged.length,
rewroteMostOfIt: diff.rewroteMostOfIt,
};
entry.detail =
`${Math.round(diff.touchedShare * 100)}% of methods touched vs ${diff.against}` +
(diff.rewroteMostOfIt
? " — most of the template is not the one that was reviewed; a different construction " +
"belongs in a revision of its own"
: "");
});
step("evidence", (entry) => {
// What KIND of defect the worst regions are, produced as part of the pass
// rather than left for someone to remember to ask.
//
// `evidence.mjs` shipped a release ago to answer exactly this, and a create
// run afterwards invoked it zero times: 43 raw ImageMagick calls, 26
// hand-written patch scripts and 21 typography measurements did the work
// instead, arriving at "the box is in the right place, so this is the
// typeface" by hand, over an hour. The tool was not missing and the skill
// named it. Nothing produced it, so nothing read it.
//
// Evidence, never a gate: a classification that cannot be built is a missing
// view of a comparison that already succeeded.
if (!result.regions) {
entry.detail = "no measured regions to classify";
return;
}
// A render that matches is not a classification failure, and reporting it as
// one would teach a reader to skim this line.
if ((result.regions.ranked ?? []).length === 0) {
entry.detail = "no region carries a difference";
return;
}
// By mass when the ranking has one, so the packages describe the regions
// carrying the difference rather than the three thinnest rules on the page.
const targets = (result.regions.byMass ?? []).slice(0, EVIDENCE_REGIONS);
const built = run(path.join(repoRoot, "scripts", "evidence.mjs"), [
"--project", args.project,
"--revision", args.revision,
"--root", workspace.root,
...(targets.length ? ["--regions", targets.join(",")] : ["--worst", String(EVIDENCE_REGIONS)]),
"--out", path.join(revisionDir, "evidence.json"),
"--json",
]);
if (built.status !== 0) {
entry.detail = `not classified: ${(built.output || "").trim().split("\n")[0] || "failed"}`;
return;
}
let packages;
try {
packages = JSON.parse(built.stdout);
} catch {
entry.detail = "not classified: evidence produced no JSON";
return;
}
const list = Array.isArray(packages) ? packages : [packages];
result.evidence = {
file: "evidence.json",
packages: list.map((pkg) => ({
region: pkg.region?.id ?? null,
cause: pkg.cause ?? null,
candidates: pkg.causeCandidates ?? [],
owner: pkg.layout?.name ?? null,
})),
};
// The line that routes: a cause per region, in the pass output the loop
// already reads, before anything decides what to change.
entry.detail = list
.map((pkg) => `${pkg.region?.id ?? "?"}: ${pkg.cause ?? "?"}`)
.join(" · ");
});
step("links", (entry) => {
// The one defect the diff above is structurally blind to: a link annotation
// has no pixels, so a dead link scores exactly zero mismatch. Cheap enough to
// run every pass, and it never fails the pass on its own — what it finds is
// folded into the verdict below.
const checked = run(path.join(repoRoot, "scripts", "check-links.mjs"), [
"--project",
args.project,
"--revision",
args.revision,
"--root",
workspace.root,
"--json",
]);
let links;
try {
links = JSON.parse(checked.stdout);
} catch {
entry.detail = "not checked";
return;
}
result.links = {
checked: links.checked,
skipped: links.skipped,
rendered: links.rendered.linkAnnotations,
declared: links.declaredCount ?? 0,
missing: links.missing,
undeclared: links.undeclared,
};
entry.detail = links.checked
? `${links.rendered.linkAnnotations} in the render, ${links.declaredCount} declared` +
(links.missing.length ? ` — ${links.missing.length} MISSING` : "") +
(links.undeclared.length ? `, ${links.undeclared.length} link-shaped without an href` : "")
: `not checked — ${links.skipped}`;
});
step("document integrity", (entry) => {
// The other blind spot: page count, "Page N of M" and whether pagination ran
// at all are functional properties of a multi-page document, and each of them
// scores as a few dozen grey pixels in a diff — or as nothing, when the
// reference never had the page that went missing.
const checked = run(path.join(repoRoot, "scripts", "check-document-integrity.mjs"), [
"--project",
args.project,
"--revision",
args.revision,
"--root",
workspace.root,
"--json",
]);
let integrity;
try {
integrity = JSON.parse(checked.stdout);
} catch {
entry.detail = "not checked";
return;
}
result.document = {
checked: integrity.checked,
skipped: integrity.skipped,
pageCount: integrity.pageCount,
flow: integrity.flow?.kind ?? null,
defects: integrity.defects,
notes: integrity.notes,
};
entry.detail = integrity.checked
? `${integrity.pageCount} page(s)` +
(integrity.flow ? `, ${integrity.flow.kind}` : "") +
(integrity.defects.length ? ` — ${integrity.defects.length} DEFECT(S)` : "")
: `not checked — ${integrity.skipped}`;
});
step("region roles", (entry) => {
// The third blind spot, and the earliest one. A header drawn as body content
// appears on page one and nowhere else; a footer drawn with bleedToEdge
// floods the page; a table drawn as rows of shapes cannot break across one. On
// page one of a sample document each of those looks exactly right, which is
// why the diff above cannot see any of them.
const checked = run(path.join(repoRoot, "scripts", "check-region-primitives.mjs"), [
"--project",
args.project,
"--revision",
args.revision,
"--root",
workspace.root,
"--json",
]);
let roles;
try {
roles = JSON.parse(checked.stdout);
} catch {
// No analysis or no plan yet is the ordinary state early in a loop.
entry.detail = "not checked";
return;
}
result.roles = {
regions: roles.regions,
mapped: roles.mapped,
findings: roles.findings,
};
const contract = roles.findings.filter((f) => f.kind !== "role-missing");
const unroled = roles.findings.filter((f) => f.kind === "role-missing");
entry.detail =
`${roles.regions} region(s)` +
(contract.length ? ` — ${contract.length} built against their role` : "") +
(unroled.length ? `, ${unroled.length} state no role` : "") +
(!contract.length && !unroled.length ? " — each built the way its role says" : "");
});
step("structural smells", (entry) => {
// The fourth blind spot, and the only one that is invisible even to a perfect
// render. Three siblings each carrying the same margin look exactly like one
// parent carrying the equivalent spacing — the diff is zero — but the first
// is three numbers the next revision has to find and move together.
const checked = run(path.join(repoRoot, "scripts", "check-structural-smells.mjs"), [
"--project",
args.project,
"--revision",
args.revision,
"--root",
workspace.root,
"--json",
]);
let smells;
try {
smells = JSON.parse(checked.stdout);
} catch {
// No generated template yet is the ordinary state early in a loop.
entry.detail = "not checked";
return;
}
result.structure = { template: smells.template, findings: smells.findings };