Skip to content

test(scripts): check-shell-escape-residue reds on an agent-tree document no scan root reaches (#7413) #8010

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
baozhoutao merged 1 commit into from
Sep 6, 2026
+351 −25
Merged

test(scripts): check-shell-escape-residue reds on an agent-tree document no scan root reaches (#7413) #8010

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 212 additions & 11 deletions scripts/__tests__/check-shell-escape-residue.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { fileURLToPath } from 'node:url';
// `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here — see
// objectui#3494.
import {
COVERAGE_TREES,
FENCE_FLOOR,
RESIDUE_PATTERNS,
SCAN_ROOTS,
Expand Down Expand Up @@ -64,6 +65,17 @@ import { REQUIRED_CONTEXTS } from '../dependabot-merge-gate.mjs';
* future widening of the scan surface turning this suite into the gate's own
* first finding — and then PINS the constructed value against the shipped
* `RESIDUE_PATTERNS` entry, so a typo in the source literal reddens here.
*
* ## Coverage, which is a different question from residue (objectui#7413)
*
* `SCAN_ROOTS` answers "what is scanned"; `COVERAGE_TREES` answers "is anything
* in the agent tree NOT scanned". The second question has no floor that can
* express it — a `minFiles` row detects a root that collapsed, never a document
* that was never on the surface — so the coverage block below carries its own
* ablation: the card's literal scenario, a planted `.claude/agents/reviewer.md`,
* which must be RED AND NAMED while every other signal in the same run reads
* healthy. That "everything else looks fine" assertion is the load-bearing half;
* without it the case would pass for a gate that reddened on anything.
*/

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
Expand Down Expand Up @@ -105,10 +117,23 @@ const FIXTURE_ROOTS = [
{ spec: 'content/docs', kind: 'dir', minFiles: 1 },
];

const scanFixture = (root: string, roots = FIXTURE_ROOTS) => scan(root, { roots, fenceFloor: 0 });
/** The fixture coverage trees, mirroring `COVERAGE_TREES` (objectui#7413). */
const FIXTURE_COVERAGE_TREES = [
{ spec: '.claude', kind: 'dir' },
{ spec: 'skills', kind: 'dir' },
];

const scanFixture = (root: string, roots = FIXTURE_ROOTS, extra: Record<string, unknown> = {}) =>
scan(root, { roots, fenceFloor: 0, coverageTrees: FIXTURE_COVERAGE_TREES, ...extra });

/** One `SCAN_ROOTS` entry, as declared. */
type DeclaredRoot = { spec: string; kind: string; minFiles: number };
/** One `COVERAGE_TREES` entry, as declared. */
type DeclaredTree = { spec: string; kind: string };
/** One `census.coverage` row, as `scan` returns it. */
type CoverageRow = { spec: string; documents: number; uncovered: number; resolved: boolean };
/** One coverage finding: a document under a coverage tree that no root reached. */
type Uncovered = { file: string; tree: string };
/** One `census.perRoot` row, as `scan` returns it. */
type RootRow = { spec: string; files: number; fences: number; minFiles: number; resolved: boolean };
/** A finding as `findResidue` returns it — no file, because it scans one source. */
Expand Down Expand Up @@ -234,20 +259,27 @@ describe('⛔ what this gate does NOT do — asserted by behaviour, not by prose
expect(findResidue(alternative).hits).toEqual([]);
});

it('looks at nothing outside SCAN_ROOTS', () => {
it('JUDGES nothing outside SCAN_ROOTS — but says so, for the agent tree', () => {
const root = fixtureTree({
'AGENTS.md': fence('bash', 'echo ok'),
'CLAUDE.md': fence('bash', 'echo ok'),
'skills/s/SKILL.md': fence('bash', 'echo ok'),
'.claude/skills/s/SKILL.md': fence('bash', 'echo ok'),
'content/docs/a.md': fence('bash', 'echo ok'),
// Out of scope by construction: not under any declared root. `.claude/`
// is only on the surface BELOW `skills/` — the rest of the agent tree
// (hooks, settings, agent definitions) is not markdown this gate reads.
// Residue under neither. `.claude/` is on the SCAN surface only BELOW
// `skills/`, so neither line is judged and `hits` stays empty for both.
'.claude/hooks/notes.md': fence('bash', HISTORICAL_LINE),
'packages/thing/README.md': fence('bash', HISTORICAL_LINE),
});
expect(scanFixture(root).hits).toEqual([]);
const result = scanFixture(root);
expect(result.hits, 'residue outside every scan root is not judged').toEqual([]);

// objectui#7413 — and the two files are not equivalent, which is the whole
// point of a coverage tree. `.claude/hooks/notes.md` is agent-tree markdown
// that no row reaches, so it is REPORTED; `packages/thing/README.md` is
// outside every coverage tree and stays out of this gate's business
// entirely. Before this, both were silent.
expect(result.uncovered).toEqual([{ file: '.claude/hooks/notes.md', tree: '.claude' }]);
});
});

Expand All @@ -273,6 +305,7 @@ describe('⭐ ablation — objectui#5150 replanted in every scan root', () => {
const result = scanFixture(fixtureTree(clean));
expect(result.hits).toEqual([]);
expect(result.unresolved).toEqual([]);
expect(result.uncovered).toEqual([]);
expect(result.vacuous).toEqual([]);
expect(result.census.fences).toBe(5);
});
Expand Down Expand Up @@ -315,12 +348,27 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', (
// A mistyped root and a clean root produce identical output otherwise, and
// the mistyped one reads as coverage for as long as nobody checks.
const root = fixtureTree({ 'AGENTS.md': fence('bash', 'echo ok') });
const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 0 });
const result = scanFixture(root);
expect(result.unresolved.map((u: { spec: string }) => u.spec)).toEqual([
'CLAUDE.md',
'skills',
'.claude/skills',
'content/docs',
// objectui#7413 — a coverage tree gets the same treatment for the same
// reason: a mistyped `.claude` would cover nothing and read as coverage
// forever, which is precisely the failure the tree was added to stop.
'.claude',
'skills',
]);
// …and the two kinds are told apart, so the remedy text can name the right
// declaration list.
expect(result.unresolved.map((u: { role: string }) => u.role)).toEqual([
'scan root',
'scan root',
'scan root',
'scan root',
'coverage tree',
'coverage tree',
]);
expect(result.census.rootsResolved).toBe(1);
});
Expand All @@ -342,7 +390,7 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', (
'.claude/skills/.keep': '',
'content/docs/.keep': '',
});
const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 0 });
const result = scanFixture(root);
// `.keep` is not a document, so all three directory roots resolve and walk to nothing.
expect(result.vacuous.map((v: { what: string }) => v.what)).toEqual([
'files under skills',
Expand All @@ -359,7 +407,7 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', (
'.claude/skills/s/SKILL.md': '# no fences',
'content/docs/a.md': '# no fences',
});
const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 400 });
const result = scanFixture(root, FIXTURE_ROOTS, { fenceFloor: 400 });
expect(result.vacuous).toEqual([{ what: 'fenced blocks examined', value: 1, floor: 400 }]);
});

Expand All @@ -377,7 +425,7 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', (
'.claude/skills/.keep': '', // resolves, walks to no document at all
'content/docs/a.md': fence('bash', 'echo ok'),
});
const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 0 });
const result = scanFixture(root);

expect(result.unresolved, 'the root EXISTS — this is emptiness, not absence').toEqual([]);
expect(result.vacuous).toEqual([{ what: 'files under .claude/skills', value: 0, floor: 1 }]);
Expand All @@ -394,12 +442,137 @@ describe('non-vacuity — zero roots or zero files is a failure, not a green', (
it('exits 1 for a collapsed population even with nothing to report', () => {
// The direction that matters: a broken walk must not be reported as OK.
const root = fixtureTree({ 'AGENTS.md': fence('bash', 'echo ok') });
const result = scan(root, { roots: FIXTURE_ROOTS, fenceFloor: 400 });
const result = scanFixture(root, FIXTURE_ROOTS, { fenceFloor: 400 });
expect(result.hits).toEqual([]);
expect(result.unresolved.length + result.vacuous.length).toBeGreaterThan(0);
});
});

// ---------------------------------------------------------------------------
// ⭐ Coverage — the half a floor cannot do (objectui#7413)
// ---------------------------------------------------------------------------

describe('⭐ coverage — an agent-tree document under no scan root is RED (objectui#7413)', () => {
/** Every root populated, every agent-tree document covered. */
const healthy = {
'AGENTS.md': fence('bash', 'echo ok'),
'CLAUDE.md': fence('bash', 'echo ok'),
'skills/objectui/SKILL.md': fence('bash', 'echo ok'),
'.claude/skills/objectui-contributor/SKILL.md': fence('bash', 'echo ok'),
'content/docs/a.md': fence('bash', 'echo ok'),
};

it('declares the two agent trees — and deliberately not the published docs tree', () => {
expect(COVERAGE_TREES.map((t: DeclaredTree) => t.spec)).toEqual(['.claude', 'skills']);
expect(COVERAGE_TREES.every((t: DeclaredTree) => t.kind === 'dir')).toBe(true);
// ⛔ Same mechanism, different claim: `content/docs` is published prose that
// shares this scan, not the agent surface objectui#5151 ruled. Declaring
// coverage over it would assert a completeness promise no card has made.
expect(COVERAGE_TREES.map((t: DeclaredTree) => t.spec)).not.toContain('content/docs');
});

it('is GREEN on the healthy fixture — the control leg', () => {
const result = scanFixture(fixtureTree(healthy));
expect(result.uncovered).toEqual([]);
expect(result.census.coverage).toEqual([
{ spec: '.claude', documents: 1, uncovered: 0, resolved: true },
{ spec: 'skills', documents: 1, uncovered: 0, resolved: true },
]);
});

it('⭐ REDS on this card\'s own scenario — a new .claude/agents/reviewer.md', () => {
// objectui#7413, verbatim: "The day someone adds `.claude/agents/reviewer.md`
// […] with a fenced shell example, it is agent-written and agent-read prose
// carrying fences that no root reaches — and nothing goes red."
const result = scanFixture(
fixtureTree({ ...healthy, '.claude/agents/reviewer.md': fence('bash', HISTORICAL_LINE, ' EOF') }),
);

const uncovered: Uncovered[] = result.uncovered;
expect(uncovered).toEqual([{ file: '.claude/agents/reviewer.md', tree: '.claude' }]);

// ⭐ And every OTHER signal reads healthy — which is exactly why a floor
// could never have caught this. `.claude/skills` still returns its file, no
// root is unresolved, nothing collapsed, and the residue inside the planted
// fence is not even judged because the document is not on the surface.
expect(result.hits, 'the residue is UNJUDGED — that is the defect').toEqual([]);
expect(result.vacuous, 'no floor fires: nothing collapsed').toEqual([]);
expect(result.unresolved, 'no root is missing: nothing moved').toEqual([]);
expect(result.census.rootsResolved).toBe(FIXTURE_ROOTS.length);
expect(result.census.perRoot.find((r: RootRow) => r.spec === '.claude/skills')).toMatchObject({
files: 1,
resolved: true,
});
});

it('is GREEN again once the same document moves under a declared root — the restore leg', () => {
// Same bytes, same fences, one directory over. A finding that does not go
// away when the gap is closed is a finding about the wrong thing.
const result = scanFixture(
fixtureTree({
...healthy,
'.claude/skills/objectui-contributor/reviewer.md': fence('bash', HISTORICAL_LINE, ' EOF'),
}),
);
expect(result.uncovered).toEqual([]);
// …and now that it IS on the surface, the residue in it is judged.
expect(result.hits.map((h: ScanHit) => h.file)).toEqual([
'.claude/skills/objectui-contributor/reviewer.md',
'.claude/skills/objectui-contributor/reviewer.md',
]);
});

it('⭐ REDS when a SCAN_ROOTS ROW IS DELETED — the shape with no other signal', () => {
// A root that VANISHES from disk is loud (`unresolved`). A root someone
// DELETES from the declaration list leaves nothing behind to be loud about:
// the remaining roots resolve, their floors are met, and a whole tree walks
// off the surface silently. This is why `skills` is a coverage tree even
// though its row covers the whole tree and the test is a tautology today.
const withoutSkills = FIXTURE_ROOTS.filter((r) => r.spec !== 'skills');
const result = scanFixture(fixtureTree(healthy), withoutSkills);

expect(result.unresolved, 'nothing is missing from DISK').toEqual([]);
expect(result.vacuous, 'every remaining floor is satisfied').toEqual([]);
expect(result.uncovered).toEqual([{ file: 'skills/objectui/SKILL.md', tree: 'skills' }]);
});

it('counts documents by the scan\'s OWN walk, so the two cannot disagree', () => {
// Coverage is membership in the set `listDocuments` returned, not a second
// glob and not a prefix match on the row spec. So `DOC_EXTENSIONS` decides
// for both: settings and hooks under `.claude/` are not documents and are
// never reported, while both markdown extensions are.
const root = fixtureTree({
...healthy,
'.claude/settings.json': '{}',
'.claude/hooks/guard.sh': '#!/bin/sh\n',
'.claude/launch.json': '{}',
'.claude/commands/ship.mdx': fence('bash', 'echo ship'),
});
const result = scanFixture(root);

expect(result.uncovered).toEqual([{ file: '.claude/commands/ship.mdx', tree: '.claude' }]);
// The membership claim itself, stated against the walk rather than inferred.
const walked: string[] = listDocuments(root, '.claude', 'dir');
expect(walked).toEqual([
'.claude/commands/ship.mdx',
'.claude/skills/objectui-contributor/SKILL.md',
]);
});

it('⚠️ reports a coverage tree that does not resolve, rather than covering nothing quietly', () => {
// The `SCAN_ROOTS` rule applied to this list: a mistyped tree spec and a
// fully-covered tree are otherwise identical output, and the mistyped one
// reads as coverage forever.
const result = scanFixture(fixtureTree(healthy), FIXTURE_ROOTS, {
coverageTrees: [{ spec: '.cluade', kind: 'dir' }],
});
expect(result.unresolved).toEqual([
{ spec: '.cluade', kind: 'dir', ok: false, problem: 'does not exist', role: 'coverage tree' },
]);
expect(result.census.coverage).toEqual([{ spec: '.cluade', documents: 0, uncovered: 0, resolved: false }]);
});
});

// ---------------------------------------------------------------------------
// The tree
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -441,6 +614,28 @@ describe('repo state — the gate is green on this tree, over a real population'
expect(row?.fences).toBeGreaterThan(0);
});

it('⭐ reaches every agent-tree document on this tree — asserted on the POPULATION (objectui#7413)', () => {
// Measured on `e1545cf`: 4 of 4 `.claude` documents under `.claude/skills`,
// 16 of 16 under `skills`. `uncovered` being empty is true both when every
// document is covered and when the coverage walk returned nothing at all —
// the same ambiguity objectui#7403 was filed about — so the population is
// asserted alongside it.
expect(result.uncovered).toEqual([]);
const rows: CoverageRow[] = result.census.coverage;
expect(rows.map((c) => c.spec)).toEqual(['.claude', 'skills']);
for (const row of rows) {
expect(row.resolved, `${row.spec} must resolve`).toBe(true);
expect(row.documents, `${row.spec} walked to nothing`).toBeGreaterThan(0);
}
// The `.claude` tree is the card's own case: its declared row is a PROPER
// SUBTREE, so this is the row that stops being a tautology first.
const claude = rows.find((c) => c.spec === '.claude');
expect(claude?.documents).toBeGreaterThanOrEqual(4);
expect(listDocuments(repoRoot, '.claude', 'dir')).toEqual(
expect.arrayContaining(['.claude/skills/objectui-contributor/SKILL.md']),
);
});

it('has no residue in any fenced block', () => {
expect(
result.hits.map((h: ScanHit) => `${h.file}:${h.line}:${h.column}`),
Expand All @@ -457,13 +652,19 @@ describe('repo state — the gate is green on this tree, over a real population'
expect(result.census.files).toBeGreaterThan(100);
expect(result.census.fences).toBeGreaterThan(FENCE_FLOOR);
expect(result.vacuous).toEqual([]);
expect(result.uncovered).toEqual([]);
});

it('puts the per-root census in the verdict, so a reader sees the population', () => {
// "OK" alone is what a gate that does nothing also prints.
const line = summarise(result);
for (const root of SCAN_ROOTS) expect(line).toContain(`${root.spec}: `);
expect(line).toMatch(/\d+ file\(s\) and \d+ fenced block\(s\) examined/);
// objectui#7413 — the coverage figure rides in the verdict for the same
// reason the per-root census does: a green must show what it was computed
// over, or a reader cannot tell it from a green that examined nothing.
expect(line).toMatch(/coverage -- .*document\(s\) under a declared root/);
for (const tree of COVERAGE_TREES) expect(line).toContain(`${tree.spec}: `);

const out = execFileSync('node', ['scripts/check-shell-escape-residue.mjs'], { cwd: repoRoot, encoding: 'utf8' });
expect(out).toMatch(/check-shell-escape-residue: OK/);
Expand Down
Loading
Loading