diff --git a/scripts/__tests__/check-shell-escape-residue.test.ts b/scripts/__tests__/check-shell-escape-residue.test.ts index a1d16a639..0ebf5b675 100644 --- a/scripts/__tests__/check-shell-escape-residue.test.ts +++ b/scripts/__tests__/check-shell-escape-residue.test.ts @@ -9,6 +9,7 @@ import { fileURLToPath } from 'node:url'; // `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here — see // objectui#3494. import { + COVERAGE_TREES, FENCE_FLOOR, RESIDUE_PATTERNS, SCAN_ROOTS, @@ -64,6 +65,17 @@ import { REQUIRED_CONTEXTS } from '../dependabot-merge-gate.mjs'; * future widening of the scan surface turning this suite into the gate's own * first finding — and then PINS the constructed value against the shipped * `RESIDUE_PATTERNS` entry, so a typo in the source literal reddens here. + * + * ## Coverage, which is a different question from residue (objectui#7413) + * + * `SCAN_ROOTS` answers "what is scanned"; `COVERAGE_TREES` answers "is anything + * in the agent tree NOT scanned". The second question has no floor that can + * express it — a `minFiles` row detects a root that collapsed, never a document + * that was never on the surface — so the coverage block below carries its own + * ablation: the card's literal scenario, a planted `.claude/agents/reviewer.md`, + * which must be RED AND NAMED while every other signal in the same run reads + * healthy. That "everything else looks fine" assertion is the load-bearing half; + * without it the case would pass for a gate that reddened on anything. */ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); @@ -105,10 +117,23 @@ const FIXTURE_ROOTS = [ { spec: 'content/docs', kind: 'dir', minFiles: 1 }, ]; -const scanFixture = (root: string, roots = FIXTURE_ROOTS) => scan(root, { roots, fenceFloor: 0 }); +/** The fixture coverage trees, mirroring `COVERAGE_TREES` (objectui#7413). */ +const FIXTURE_COVERAGE_TREES = [ + { spec: '.claude', kind: 'dir' }, + { spec: 'skills', kind: 'dir' }, +]; + +const scanFixture = (root: string, roots = FIXTURE_ROOTS, extra: Record = {}) => + scan(root, { roots, fenceFloor: 0, coverageTrees: FIXTURE_COVERAGE_TREES, ...extra }); /** One `SCAN_ROOTS` entry, as declared. */ type DeclaredRoot = { spec: string; kind: string; minFiles: number }; +/** One `COVERAGE_TREES` entry, as declared. */ +type DeclaredTree = { spec: string; kind: string }; +/** One `census.coverage` row, as `scan` returns it. */ +type CoverageRow = { spec: string; documents: number; uncovered: number; resolved: boolean }; +/** One coverage finding: a document under a coverage tree that no root reached. */ +type Uncovered = { file: string; tree: string }; /** One `census.perRoot` row, as `scan` returns it. */ type RootRow = { spec: string; files: number; fences: number; minFiles: number; resolved: boolean }; /** A finding as `findResidue` returns it — no file, because it scans one source. */ @@ -234,20 +259,27 @@ describe('⛔ what this gate does NOT do — asserted by behaviour, not by prose expect(findResidue(alternative).hits).toEqual([]); }); - it('looks at nothing outside SCAN_ROOTS', () => { + it('JUDGES nothing outside SCAN_ROOTS — but says so, for the agent tree', () => { const root = fixtureTree({ 'AGENTS.md': fence('bash', 'echo ok'), 'CLAUDE.md': fence('bash', 'echo ok'), 'skills/s/SKILL.md': fence('bash', 'echo ok'), '.claude/skills/s/SKILL.md': fence('bash', 'echo ok'), 'content/docs/a.md': fence('bash', 'echo ok'), - // Out of scope by construction: not under any declared root. `.claude/` - // is only on the surface BELOW `skills/` — the rest of the agent tree - // (hooks, settings, agent definitions) is not markdown this gate reads. + // Residue under neither. `.claude/` is on the SCAN surface only BELOW + // `skills/`, so neither line is judged and `hits` stays empty for both. '.claude/hooks/notes.md': fence('bash', HISTORICAL_LINE), 'packages/thing/README.md': fence('bash', HISTORICAL_LINE), }); - expect(scanFixture(root).hits).toEqual([]); + const result = scanFixture(root); + expect(result.hits, 'residue outside every scan root is not judged').toEqual([]); + + // objectui#7413 — and the two files are not equivalent, which is the whole + // point of a coverage tree. `.claude/hooks/notes.md` is agent-tree markdown + // that no row reaches, so it is REPORTED; `packages/thing/README.md` is + // outside every coverage tree and stays out of this gate's business + // entirely. Before this, both were silent. + expect(result.uncovered).toEqual([{ file: '.claude/hooks/notes.md', tree: '.claude' }]); }); }); @@ -273,6 +305,7 @@ describe('⭐ ablation — objectui#5150 replanted in every scan root', () => { const result = scanFixture(fixtureTree(clean)); expect(result.hits).toEqual([]); expect(result.unresolved).toEqual([]); + expect(result.uncovered).toEqual([]); expect(result.vacuous).toEqual([]); expect(result.census.fences).toBe(5); }); @@ -315,12 +348,27 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', ( // A mistyped root and a clean root produce identical output otherwise, and // the mistyped one reads as coverage for as long as nobody checks. const root = fixtureTree({ 'AGENTS.md': fence('bash', 'echo ok') }); - const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 0 }); + const result = scanFixture(root); expect(result.unresolved.map((u: { spec: string }) => u.spec)).toEqual([ 'CLAUDE.md', 'skills', '.claude/skills', 'content/docs', + // objectui#7413 — a coverage tree gets the same treatment for the same + // reason: a mistyped `.claude` would cover nothing and read as coverage + // forever, which is precisely the failure the tree was added to stop. + '.claude', + 'skills', + ]); + // …and the two kinds are told apart, so the remedy text can name the right + // declaration list. + expect(result.unresolved.map((u: { role: string }) => u.role)).toEqual([ + 'scan root', + 'scan root', + 'scan root', + 'scan root', + 'coverage tree', + 'coverage tree', ]); expect(result.census.rootsResolved).toBe(1); }); @@ -342,7 +390,7 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', ( '.claude/skills/.keep': '', 'content/docs/.keep': '', }); - const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 0 }); + const result = scanFixture(root); // `.keep` is not a document, so all three directory roots resolve and walk to nothing. expect(result.vacuous.map((v: { what: string }) => v.what)).toEqual([ 'files under skills', @@ -359,7 +407,7 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', ( '.claude/skills/s/SKILL.md': '# no fences', 'content/docs/a.md': '# no fences', }); - const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 400 }); + const result = scanFixture(root, FIXTURE_ROOTS, { fenceFloor: 400 }); expect(result.vacuous).toEqual([{ what: 'fenced blocks examined', value: 1, floor: 400 }]); }); @@ -377,7 +425,7 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', ( '.claude/skills/.keep': '', // resolves, walks to no document at all 'content/docs/a.md': fence('bash', 'echo ok'), }); - const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 0 }); + const result = scanFixture(root); expect(result.unresolved, 'the root EXISTS — this is emptiness, not absence').toEqual([]); expect(result.vacuous).toEqual([{ what: 'files under .claude/skills', value: 0, floor: 1 }]); @@ -394,12 +442,137 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', ( it('exits 1 for a collapsed population even with nothing to report', () => { // The direction that matters: a broken walk must not be reported as OK. const root = fixtureTree({ 'AGENTS.md': fence('bash', 'echo ok') }); - const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 400 }); + const result = scanFixture(root, FIXTURE_ROOTS, { fenceFloor: 400 }); expect(result.hits).toEqual([]); expect(result.unresolved.length + result.vacuous.length).toBeGreaterThan(0); }); }); +// --------------------------------------------------------------------------- +// ⭐ Coverage — the half a floor cannot do (objectui#7413) +// --------------------------------------------------------------------------- + +describe('⭐ coverage — an agent-tree document under no scan root is RED (objectui#7413)', () => { + /** Every root populated, every agent-tree document covered. */ + const healthy = { + 'AGENTS.md': fence('bash', 'echo ok'), + 'CLAUDE.md': fence('bash', 'echo ok'), + 'skills/objectui/SKILL.md': fence('bash', 'echo ok'), + '.claude/skills/objectui-contributor/SKILL.md': fence('bash', 'echo ok'), + 'content/docs/a.md': fence('bash', 'echo ok'), + }; + + it('declares the two agent trees — and deliberately not the published docs tree', () => { + expect(COVERAGE_TREES.map((t: DeclaredTree) => t.spec)).toEqual(['.claude', 'skills']); + expect(COVERAGE_TREES.every((t: DeclaredTree) => t.kind === 'dir')).toBe(true); + // ⛔ Same mechanism, different claim: `content/docs` is published prose that + // shares this scan, not the agent surface objectui#5151 ruled. Declaring + // coverage over it would assert a completeness promise no card has made. + expect(COVERAGE_TREES.map((t: DeclaredTree) => t.spec)).not.toContain('content/docs'); + }); + + it('is GREEN on the healthy fixture — the control leg', () => { + const result = scanFixture(fixtureTree(healthy)); + expect(result.uncovered).toEqual([]); + expect(result.census.coverage).toEqual([ + { spec: '.claude', documents: 1, uncovered: 0, resolved: true }, + { spec: 'skills', documents: 1, uncovered: 0, resolved: true }, + ]); + }); + + it('⭐ REDS on this card\'s own scenario — a new .claude/agents/reviewer.md', () => { + // objectui#7413, verbatim: "The day someone adds `.claude/agents/reviewer.md` + // […] with a fenced shell example, it is agent-written and agent-read prose + // carrying fences that no root reaches — and nothing goes red." + const result = scanFixture( + fixtureTree({ ...healthy, '.claude/agents/reviewer.md': fence('bash', HISTORICAL_LINE, ' EOF') }), + ); + + const uncovered: Uncovered[] = result.uncovered; + expect(uncovered).toEqual([{ file: '.claude/agents/reviewer.md', tree: '.claude' }]); + + // ⭐ And every OTHER signal reads healthy — which is exactly why a floor + // could never have caught this. `.claude/skills` still returns its file, no + // root is unresolved, nothing collapsed, and the residue inside the planted + // fence is not even judged because the document is not on the surface. + expect(result.hits, 'the residue is UNJUDGED — that is the defect').toEqual([]); + expect(result.vacuous, 'no floor fires: nothing collapsed').toEqual([]); + expect(result.unresolved, 'no root is missing: nothing moved').toEqual([]); + expect(result.census.rootsResolved).toBe(FIXTURE_ROOTS.length); + expect(result.census.perRoot.find((r: RootRow) => r.spec === '.claude/skills')).toMatchObject({ + files: 1, + resolved: true, + }); + }); + + it('is GREEN again once the same document moves under a declared root — the restore leg', () => { + // Same bytes, same fences, one directory over. A finding that does not go + // away when the gap is closed is a finding about the wrong thing. + const result = scanFixture( + fixtureTree({ + ...healthy, + '.claude/skills/objectui-contributor/reviewer.md': fence('bash', HISTORICAL_LINE, ' EOF'), + }), + ); + expect(result.uncovered).toEqual([]); + // …and now that it IS on the surface, the residue in it is judged. + expect(result.hits.map((h: ScanHit) => h.file)).toEqual([ + '.claude/skills/objectui-contributor/reviewer.md', + '.claude/skills/objectui-contributor/reviewer.md', + ]); + }); + + it('⭐ REDS when a SCAN_ROOTS ROW IS DELETED — the shape with no other signal', () => { + // A root that VANISHES from disk is loud (`unresolved`). A root someone + // DELETES from the declaration list leaves nothing behind to be loud about: + // the remaining roots resolve, their floors are met, and a whole tree walks + // off the surface silently. This is why `skills` is a coverage tree even + // though its row covers the whole tree and the test is a tautology today. + const withoutSkills = FIXTURE_ROOTS.filter((r) => r.spec !== 'skills'); + const result = scanFixture(fixtureTree(healthy), withoutSkills); + + expect(result.unresolved, 'nothing is missing from DISK').toEqual([]); + expect(result.vacuous, 'every remaining floor is satisfied').toEqual([]); + expect(result.uncovered).toEqual([{ file: 'skills/objectui/SKILL.md', tree: 'skills' }]); + }); + + it('counts documents by the scan\'s OWN walk, so the two cannot disagree', () => { + // Coverage is membership in the set `listDocuments` returned, not a second + // glob and not a prefix match on the row spec. So `DOC_EXTENSIONS` decides + // for both: settings and hooks under `.claude/` are not documents and are + // never reported, while both markdown extensions are. + const root = fixtureTree({ + ...healthy, + '.claude/settings.json': '{}', + '.claude/hooks/guard.sh': '#!/bin/sh\n', + '.claude/launch.json': '{}', + '.claude/commands/ship.mdx': fence('bash', 'echo ship'), + }); + const result = scanFixture(root); + + expect(result.uncovered).toEqual([{ file: '.claude/commands/ship.mdx', tree: '.claude' }]); + // The membership claim itself, stated against the walk rather than inferred. + const walked: string[] = listDocuments(root, '.claude', 'dir'); + expect(walked).toEqual([ + '.claude/commands/ship.mdx', + '.claude/skills/objectui-contributor/SKILL.md', + ]); + }); + + it('⚠️ reports a coverage tree that does not resolve, rather than covering nothing quietly', () => { + // The `SCAN_ROOTS` rule applied to this list: a mistyped tree spec and a + // fully-covered tree are otherwise identical output, and the mistyped one + // reads as coverage forever. + const result = scanFixture(fixtureTree(healthy), FIXTURE_ROOTS, { + coverageTrees: [{ spec: '.cluade', kind: 'dir' }], + }); + expect(result.unresolved).toEqual([ + { spec: '.cluade', kind: 'dir', ok: false, problem: 'does not exist', role: 'coverage tree' }, + ]); + expect(result.census.coverage).toEqual([{ spec: '.cluade', documents: 0, uncovered: 0, resolved: false }]); + }); +}); + // --------------------------------------------------------------------------- // The tree // --------------------------------------------------------------------------- @@ -441,6 +614,28 @@ describe('repo state — the gate is green on this tree, over a real population' expect(row?.fences).toBeGreaterThan(0); }); + it('⭐ reaches every agent-tree document on this tree — asserted on the POPULATION (objectui#7413)', () => { + // Measured on `e1545cf`: 4 of 4 `.claude` documents under `.claude/skills`, + // 16 of 16 under `skills`. `uncovered` being empty is true both when every + // document is covered and when the coverage walk returned nothing at all — + // the same ambiguity objectui#7403 was filed about — so the population is + // asserted alongside it. + expect(result.uncovered).toEqual([]); + const rows: CoverageRow[] = result.census.coverage; + expect(rows.map((c) => c.spec)).toEqual(['.claude', 'skills']); + for (const row of rows) { + expect(row.resolved, `${row.spec} must resolve`).toBe(true); + expect(row.documents, `${row.spec} walked to nothing`).toBeGreaterThan(0); + } + // The `.claude` tree is the card's own case: its declared row is a PROPER + // SUBTREE, so this is the row that stops being a tautology first. + const claude = rows.find((c) => c.spec === '.claude'); + expect(claude?.documents).toBeGreaterThanOrEqual(4); + expect(listDocuments(repoRoot, '.claude', 'dir')).toEqual( + expect.arrayContaining(['.claude/skills/objectui-contributor/SKILL.md']), + ); + }); + it('has no residue in any fenced block', () => { expect( result.hits.map((h: ScanHit) => `${h.file}:${h.line}:${h.column}`), @@ -457,6 +652,7 @@ describe('repo state — the gate is green on this tree, over a real population' expect(result.census.files).toBeGreaterThan(100); expect(result.census.fences).toBeGreaterThan(FENCE_FLOOR); expect(result.vacuous).toEqual([]); + expect(result.uncovered).toEqual([]); }); it('puts the per-root census in the verdict, so a reader sees the population', () => { @@ -464,6 +660,11 @@ describe('repo state — the gate is green on this tree, over a real population' const line = summarise(result); for (const root of SCAN_ROOTS) expect(line).toContain(`${root.spec}: `); expect(line).toMatch(/\d+ file\(s\) and \d+ fenced block\(s\) examined/); + // objectui#7413 — the coverage figure rides in the verdict for the same + // reason the per-root census does: a green must show what it was computed + // over, or a reader cannot tell it from a green that examined nothing. + expect(line).toMatch(/coverage -- .*document\(s\) under a declared root/); + for (const tree of COVERAGE_TREES) expect(line).toContain(`${tree.spec}: `); const out = execFileSync('node', ['scripts/check-shell-escape-residue.mjs'], { cwd: repoRoot, encoding: 'utf8' }); expect(out).toMatch(/check-shell-escape-residue: OK/); diff --git a/scripts/check-shell-escape-residue.mjs b/scripts/check-shell-escape-residue.mjs index 7b1683887..57be819a9 100644 --- a/scripts/check-shell-escape-residue.mjs +++ b/scripts/check-shell-escape-residue.mjs @@ -43,7 +43,12 @@ * narrowing" below; the census counts those occurrences so the exclusion is * a number rather than a silence. * - * ⛔ It does not look outside `SCAN_ROOTS`. + * ⛔ It does not JUDGE text outside `SCAN_ROOTS`. It does make one narrow + * claim ABOUT what is outside them: every `.md`/`.mdx` under a + * `COVERAGE_TREES` entry must be reached by some declared root, so an + * agent-tree document no row covers is RED rather than a silent gap + * (objectui#7413). That is a membership test over the walk's own output -- + * it never reads a file the scan did not read, and it never judges one. * * The name is chosen to say all of that: `shell-escape-residue`, not * `shell-examples`. A gate named for a general property while checking a literal @@ -236,7 +241,10 @@ const DOC_EXTENSIONS = ['.mdx', '.md']; * narrower one was declared because it is the subtree whose contents are * agent-read prose by construction. ⛔ It therefore does NOT reach a future * `.claude/agents/*.md` or `.claude/commands/*.md` -- which is this card's own - * class one step out, and is objectui#7413 rather than pre-solved here. + * class one step out. That is objectui#7413, and it is closed by + * `COVERAGE_TREES` below rather than by widening this row: the row keeps + * declaring what is SCANNED, and the coverage tree turns a document the row + * does not reach into a RED gate that names the file. * * ⚠️ `AGENTS.md`, `CLAUDE.md`, `skills/**` and `.claude/**` are GOVERNED SURFACE * (AGENTS.md §受管面). This gate READS them and never writes: a finding in one of @@ -251,6 +259,68 @@ export const SCAN_ROOTS = Object.freeze([ { spec: 'content/docs', kind: 'dir', minFiles: 100 }, ]); +/** + * ## objectui#7413 -- coverage, which no floor can express + * + * Trees in which EVERY `.md`/`.mdx` must be reached by some `SCAN_ROOTS` row. A + * document under one of these that no row reaches is a FAILURE naming the file. + * + * ⭐ This is the second half of objectui#7403's lesson, and the half a floor + * cannot do. `minFiles` asks "did this root still return enough files?" -- a + * COLLAPSE detector, and collapse is only one of the two ways coverage is lost: + * + * root walked to zero -> `minFiles` fires (objectui#7251's move) + * root declared, file NEW -> nothing fires (this card) + * root ROW DELETED -> nothing fires (this card, worse) + * + * On the day `.claude/agents/reviewer.md` is written with a fenced shell + * example, `.claude/skills` still returns its 4 files, every floor is satisfied, + * every root resolves, and the new document is simply not on the surface. That + * is objectui#5151's own defect one level up -- a green that means "nothing was + * looked at" -- and it is only ever noticed by someone going and looking. Row + * deletion is the same shape with no signal at all: an unresolved root is loud, + * but a row someone REMOVED leaves nothing behind to be loud about. + * + * ## ⛔ Why it cannot disagree with the scan about what a document is + * + * Coverage is a set-membership test over the walk's OWN output: `scan` collects + * every path `listDocuments` returned for every resolved root, then asks the + * same `listDocuments` for the tree. One walk, one `DOC_EXTENSIONS`, one + * definition of "document". A re-derivation -- a second glob, a prefix match on + * the row's `spec` -- would be a second answer to one fact, free to drift from + * the first (objectui#3261/#3279), and the drift would land on the side that + * reads as coverage. ⛔ Do not reintroduce one. + * + * ## The rows, and why `skills` is here while `content/docs` is not + * + * `.claude` is the card's own case: the declared row is a PROPER SUBTREE of it, + * so the tree can grow documents outside the row. Measured on `e1545cf`: 4 of 4 + * `.claude` documents under `.claude/skills`, so this row is green today by + * construction and its whole job is the next file. + * + * `skills` is the generalisation objectui#7413 asked to be measured. Its row is + * the WHOLE tree, so today the test is a tautology -- 16 of 16 -- and it costs + * one extra walk of 16 files and nothing else. It is declared anyway because a + * tautology is exactly what it stops being under the one edit nothing else here + * catches: NARROWING the `skills` row to a subtree, or deleting it. Neither + * shows up as unresolved, and both leave the published skills tree unscanned. + * + * ⛔ `content/docs` is deliberately NOT a coverage tree. Same mechanics, but a + * different claim: this gate's roots are the agent-facing surface (objectui#5151 + * ruled it, objectui#7403 widened it), and `content/docs` is published prose + * that happens to share the scan. Declaring coverage over it would assert a + * completeness promise about the docs tree that no card has ruled. If that is + * ever wanted, rule it on its own card -- the mechanism takes one row. + * + * ⛔ This is not an allowlist and there is no per-file opt-out: the two ways to + * clear a finding are to move the document under a declared root, or to declare + * a new root for it. Both are the deliberate decision the gap deserves. + */ +export const COVERAGE_TREES = Object.freeze([ + { spec: '.claude', kind: 'dir' }, + { spec: 'skills', kind: 'dir' }, +]); + /** * Literals OBSERVED to be machine-produced and shipped. One entry; see the * header for why a second one needs an observed instance rather than an @@ -376,26 +446,34 @@ export function findResidue(source) { * here, so the tests exercise the real code path rather than an imitation. * * @param {string} root Repository root to scan. - * @param {{ roots?: ReadonlyArray, fenceFloor?: number }} [options] + * @param {{ roots?: ReadonlyArray, fenceFloor?: number, + * coverageTrees?: ReadonlyArray }} [options] * `roots` overrides `SCAN_ROOTS` (fixtures declare their own); `fenceFloor` * overrides `FENCE_FLOOR` -- pass 0 for a fixture tree, which is legitimately - * far below any repo floor. + * far below any repo floor; `coverageTrees` overrides `COVERAGE_TREES`. */ -export function scan(root, { roots = SCAN_ROOTS, fenceFloor = FENCE_FLOOR } = {}) { +export function scan(root, { roots = SCAN_ROOTS, fenceFloor = FENCE_FLOOR, coverageTrees = COVERAGE_TREES } = {}) { const perRoot = []; const unresolved = []; const hits = []; + /** + * Every document the walk reached, repo-relative. objectui#7413's coverage + * test is membership in THIS set -- the scan's own output, never a second + * derivation of it. + */ + const scanned = new Set(); let outsideFences = 0; for (const declared of roots) { const resolved = resolveRoot(root, declared); if (!resolved.ok) { - unresolved.push(resolved); + unresolved.push({ ...resolved, role: 'scan root' }); perRoot.push({ spec: declared.spec, files: 0, fences: 0, minFiles: declared.minFiles, resolved: false }); continue; } const documents = listDocuments(root, declared.spec, declared.kind); + for (const rel of documents) scanned.add(rel); let fences = 0; for (const rel of documents) { let source; @@ -419,6 +497,25 @@ export function scan(root, { roots = SCAN_ROOTS, fenceFloor = FENCE_FLOOR } = {} }); } + // objectui#7413 -- every document under a coverage tree must have been reached + // by some declared root above. A tree that does not resolve is LOUD for the + // same reason a scan root is: a mistyped coverage tree covers nothing, and + // reads as coverage forever. + const uncovered = []; + const coverage = []; + for (const tree of coverageTrees) { + const resolved = resolveRoot(root, tree); + if (!resolved.ok) { + unresolved.push({ ...resolved, role: 'coverage tree' }); + coverage.push({ spec: tree.spec, documents: 0, uncovered: 0, resolved: false }); + continue; + } + const documents = listDocuments(root, tree.spec, tree.kind); + const missing = documents.filter((rel) => !scanned.has(rel)); + for (const rel of missing) uncovered.push({ file: rel, tree: tree.spec }); + coverage.push({ spec: tree.spec, documents: documents.length, uncovered: missing.length, resolved: true }); + } + const census = { roots: roots.length, rootsResolved: perRoot.filter((r) => r.resolved).length, @@ -426,6 +523,7 @@ export function scan(root, { roots = SCAN_ROOTS, fenceFloor = FENCE_FLOOR } = {} fences: perRoot.reduce((n, r) => n + r.fences, 0), perRoot, outsideFences, + coverage, }; // The population, checked for collapse. See "GREEN AT REST" in the header. @@ -439,7 +537,7 @@ export function scan(root, { roots = SCAN_ROOTS, fenceFloor = FENCE_FLOOR } = {} vacuous.push({ what: 'fenced blocks examined', value: census.fences, floor: fenceFloor }); } - return { census, hits, unresolved, vacuous }; + return { census, hits, unresolved, uncovered, vacuous }; } /** The census, as one line, for the verdict. */ @@ -447,18 +545,24 @@ export function summarise({ census }) { const per = census.perRoot .map((r) => `${r.spec}: ${r.resolved ? `${r.files} file(s), ${r.fences} fence(s)` : 'UNRESOLVED'}`) .join('; '); + // The coverage figure is in the verdict for the same reason the per-root + // census is: a reader has to see the population a green was computed over. + const cover = census.coverage + .map((c) => `${c.spec}: ${c.resolved ? `${c.documents - c.uncovered}/${c.documents}` : 'UNRESOLVED'}`) + .join('; '); return ( `${census.rootsResolved}/${census.roots} root(s) resolved -- ${per}; ` + `${census.files} file(s) and ${census.fences} fenced block(s) examined in total; ` + - `${census.outsideFences} occurrence(s) outside a fence (counted, not judged)` + `${census.outsideFences} occurrence(s) outside a fence (counted, not judged); ` + + `coverage -- ${cover} document(s) under a declared root` ); } function main() { const result = scan(repoRoot()); - const { hits, unresolved, vacuous } = result; + const { hits, unresolved, uncovered, vacuous } = result; - if (hits.length === 0 && unresolved.length === 0 && vacuous.length === 0) { + if (hits.length === 0 && unresolved.length === 0 && uncovered.length === 0 && vacuous.length === 0) { console.log(`✅ check-shell-escape-residue: OK (${summarise(result)}).`); process.exit(0); } @@ -488,13 +592,30 @@ examples are executable -- nothing does. See this script's header.`); } if (unresolved.length > 0) { - console.error('\n❌ check-shell-escape-residue: a declared scan root did not resolve\n'); - for (const u of unresolved) console.error(` - ${u.spec} (declared ${u.kind}) ${u.problem}`); + console.error('\n❌ check-shell-escape-residue: a declared root did not resolve\n'); + for (const u of unresolved) console.error(` - ${u.spec} (${u.role}, declared ${u.kind}) ${u.problem}`); console.error(` A root that is gone is reported rather than skipped, because a MISTYPED root and a CLEAN root produce identical output otherwise -- and the mistyped one reads as coverage for as long as nobody checks. If the file genuinely moved, move it in -\`SCAN_ROOTS\` in the same change.`); +\`SCAN_ROOTS\` -- or in \`COVERAGE_TREES\` for a coverage tree -- in the same +change.`); + } + + if (uncovered.length > 0) { + const plural = uncovered.length === 1 ? 'document is' : 'documents are'; + console.error(`\n❌ check-shell-escape-residue: ${uncovered.length} agent-tree ${plural} on no scan root\n`); + for (const u of uncovered) console.error(` - ${u.file} (under the coverage tree \`${u.tree}\`)`); + console.error(` +These files exist, they are markdown in a tree this gate claims, and NO +\`SCAN_ROOTS\` row reaches them -- so their fenced blocks are unjudged and every +floor above is still satisfied by the files that ARE covered (objectui#7413). +A \`minFiles\` floor detects a root that COLLAPSED; it cannot detect a document +that was never on the surface, which is why this is a separate judgement. + +Two ways to clear it, both deliberate -- there is no per-file opt-out: + - move the document under a root \`SCAN_ROOTS\` already declares, or + - add a \`SCAN_ROOTS\` row for its location, in the same change that adds it.`); } if (vacuous.length > 0) { @@ -522,13 +643,17 @@ Census: ${summarise(result)}`); if (isEntrypoint(import.meta.url)) { if (process.argv.includes('--json')) { const result = scan(repoRoot()); - console.log(JSON.stringify({ census: result.census, hits: result.hits, unresolved: result.unresolved, vacuous: result.vacuous }, null, 2)); + console.log(JSON.stringify({ census: result.census, hits: result.hits, unresolved: result.unresolved, uncovered: result.uncovered, vacuous: result.vacuous }, null, 2)); } else if (process.argv.includes('--list')) { const result = scan(repoRoot()); for (const r of result.census.perRoot) { console.log(`${r.resolved ? 'ok ' : 'UNRESOLVED'} ${r.spec.padEnd(16)} ${r.files} file(s), ${r.fences} fence(s)`); } + for (const c of result.census.coverage) { + console.log(`${c.resolved ? 'coverage ' : 'UNRESOLVED'} ${c.spec.padEnd(16)} ${c.documents - c.uncovered}/${c.documents} document(s) under a declared root`); + } for (const hit of result.hits) console.log(`RESIDUE ${hit.file}:${hit.line}:${hit.column} ${hit.patternId}`); + for (const u of result.uncovered) console.log(`UNCOVERED ${u.file} (tree ${u.tree})`); console.log(`\n${summarise(result)}`); } else { main();