diff --git a/CHANGELOG.md b/CHANGELOG.md index 64597979..5757dfad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -407,6 +407,18 @@ true until the next version shipped. ### Fixed +- `pgc_ledger.py gate` refuses a census that contradicts its own ledger (#952). + + It printed `never observed red=N` and never compared that N to + `checks_never_observed_red`. A 20-row ledger with a budget claiming 5 returned + rc=0. Two PRs that each rewrote the census from the same base composed into a + ledger holding both sets of rows while the budget kept whichever side won, and + the tool certified the pair. + + The census is not a ceiling and this does not make it one. The refusal is only + that the two numbers describe the same file and disagree, which needs no prior. + A matching pair still passes. + - A loop that never ran asserted nothing, and half of that was already refused by a mechanism nobody had recorded covered it (#432). diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 422ae9f0..059ad07b 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -607,6 +607,7 @@ harness_selftest 400-a-check-result-must-be-machine the sweep finds a check insi harness_selftest 410-a-check-must-have-been-red --mutation across two failing checks in one run is refused never - harness_selftest 410-a-check-must-have-been-red a before-log and an after-log together are refused, not silently empty never - harness_selftest 410-a-check-must-have-been-red a budget that does not exist at the prior is a note, not a refusal never - +harness_selftest 410-a-check-must-have-been-red a budget that understates the ledger census is refused never - harness_selftest 410-a-check-must-have-been-red a check in an UNCOVERED suite is not refused never - harness_selftest 410-a-check-must-have-been-red a check merely added is not reported as a rename never - harness_selftest 410-a-check-must-have-been-red a check observed red gains the date it was seen never - @@ -666,6 +667,7 @@ harness_selftest 410-a-check-must-have-been-red and the comparison passes when t harness_selftest 410-a-check-must-have-been-red and the history it is about to lose travels with it never - harness_selftest 410-a-check-must-have-been-red and the message says how to fix it, because regenerating is the intended action never - harness_selftest 410-a-check-must-have-been-red and the refusal names both values never - +harness_selftest 410-a-check-must-have-been-red and the refusal names both values the budget and the ledger hold never - harness_selftest 410-a-check-must-have-been-red and the refusal names how many failed, so the author can narrow the run never - harness_selftest 410-a-check-must-have-been-red and the refusal names the check that reddened never - harness_selftest 410-a-check-must-have-been-red and the refusal names the raise never - @@ -677,6 +679,7 @@ harness_selftest 410-a-check-must-have-been-red both failure arms fail the major harness_selftest 410-a-check-must-have-been-red but that suite is counted as not covered, which is the debt never - harness_selftest 410-a-check-must-have-been-red control: a well-formed log still merges never - harness_selftest 410-a-check-must-have-been-red control: an all-PASS log still merges with no flag at all never - +harness_selftest 410-a-check-must-have-been-red control: the same ledger passes when the census matches never - harness_selftest 410-a-check-must-have-been-red control: the same log merges with a different reason, so the refusal above is --mutation-across-two-checks and not the log never - harness_selftest 410-a-check-must-have-been-red each says what was wrong with the input never - harness_selftest 410-a-check-must-have-been-red every committed row has five fields never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 08560d1b..2f226bb2 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -34,4 +34,4 @@ suites_not_covered 250 # Without that it is a hand-maintained count that drifts, which is the failure # this repository has spent a day proving. It is not a ceiling; it is a # measurement that must be true. -checks_never_observed_red 762 +checks_never_observed_red 765 diff --git a/test/pgc_ledger.py b/test/pgc_ledger.py index 1b6b8648..2ec90c29 100755 --- a/test/pgc_ledger.py +++ b/test/pgc_ledger.py @@ -35,6 +35,10 @@ ledger is regenerated, which is a reviewable one-line diff and the INTENDED action rather than a forbidden one. +A budget whose `checks_never_observed_red` disagrees with the ledger it was handed. +Printing the census is not enforcing it (#952). The census is not a ceiling; the +refusal is only that the two numbers describe the same file and disagree. + WHAT FEEDS IT ------------- `run_all_versions.sh` merges every suite's log before it removes the build @@ -561,6 +565,23 @@ def cmd_gate(args): print(f" ledger census: rows={len(rows)} | never observed red={never}, " f"ever red={len(rows) - never}, new this run={len(unknown)}") + # A CENSUS MUST MATCH THE LEDGER IT DESCRIBES (#952). Printing the count is + # not enforcing it: a 20-row ledger with a budget claiming 5 returned rc=0, + # and composing two PRs that each rewrote the census from the same base left + # the ledger holding both sets of rows while the budget kept whichever side + # won. The comparison lives one layer out today, in the selftest arm, so a + # merge commit is what first notices. The tool that CI runs should refuse + # the pair itself. + # + # Not a ceiling. Bounding this number deadlocks; the refusal is only that + # these two numbers describe the same file and disagree, which needs no prior. + # A budget that omits the key is left to the committed-file arm; the lie this + # closes is a number that is present and wrong. + stated = budget.get("checks_never_observed_red") + if stated is not None and stated != never: + print(f" checks_never_observed_red: budget states {stated}, ledger has {never}") + rc = 1 + if not args.registered: raise LedgerError( "--registered is required: without the registered suite list the coverage " diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index d6b10b8f..cace7a61 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -2150,6 +2150,18 @@ If they disagree, one was edited by hand. `suites_not_covered` is 250 of 251, so gate cannot refuse a new check in 250 suites — a real limit, counted rather than hidden, which falls as suites are seeded. +### `test_the_gate_refuses_a_census_that_contradicts_its_ledger` + +`gate` printed the census and did not compare it to the budget. A 20-row ledger +with `checks_never_observed_red 5` returned rc=0. Composing two PRs that each +rewrote the census from the same base left the ledger holding both sets of rows +while the budget kept whichever side won, and the tool certified the lie (#952). + +The census is not a ceiling — bounding it deadlocks. The refusal is only that the +two numbers describe the same file and disagree. An all-matching pair still +passes, which is the control. Independent of the shell fixture, which uses six +rows claiming two. + ## 24. test_loop_coverage_premise.py: a loop that never ran asserted nothing diff --git a/test/pytest/test_mutation_ledger.py b/test/pytest/test_mutation_ledger.py index ae6a90b6..bd24d840 100644 --- a/test/pytest/test_mutation_ledger.py +++ b/test/pytest/test_mutation_ledger.py @@ -432,3 +432,38 @@ def test_a_reconciling_log_with_a_red_is_not_evidence_on_its_own(tmp_path, expec green = _w(tmp_path, "green.log", GREEN) expect.num(_run("merge", "--ledger", led4, "--date", "2026-09-10", green)[1], 0, "control: an all-PASS log still merges with no flag at all") + +def test_the_gate_refuses_a_census_that_contradicts_its_ledger(tmp_path, expect): + """#952: reporting the census is not enforcing it. + + `gate` prints `ledger census: rows=N | never observed red=N` and never + compares that N to `checks_never_observed_red` in the budget. Measured on + main: a 20-row ledger with a budget claiming 5 returned rc=0. Found by + @OffgridwithJD composing #943 and #947: the ledger took both sets of rows + while the budget kept whichever side won, and the tool certified the lie. + + The census is not a ceiling and must not become one. The refusal is only + that these two numbers describe the same file and disagree, which is + decidable from the two inputs with no prior. + """ + lines = ["demo\tp\tc%02d\tnever\t-\n" % i for i in range(20)] + ledger = _w(tmp_path, "l.tsv", "".join(lines)) + log = _w(tmp_path, "g.log", + "".join("RESULT\tdemo\tp\tc%02d\tPASS\t\n" % i for i in range(20)) + + "checks run: 20\n") + reg = _w(tmp_path, "reg", "demo\n") + + lie = _w(tmp_path, "lie.txt", + "suites_not_covered 0\nchecks_never_observed_red 5\n") + out, rc = _run("gate", "--ledger", ledger, "--budget", lie, + "--registered", reg, log) + expect.num(rc, 1, "a budget that understates the ledger census is refused") + expect.at_least(out.count("5"), 1, "and the refusal names the budget value") + expect.at_least(out.count("20"), 1, "and it names the ledger value") + + ok = _w(tmp_path, "ok.txt", + "suites_not_covered 0\nchecks_never_observed_red 20\n") + expect.num(_run("gate", "--ledger", ledger, "--budget", ok, + "--registered", reg, log)[1], 0, + "control: the same ledger passes when the census matches") + diff --git a/test/selftest/410-a-check-must-have-been-red.sh b/test/selftest/410-a-check-must-have-been-red.sh index e3427a5f..30ddd05e 100644 --- a/test/selftest/410-a-check-must-have-been-red.sh +++ b/test/selftest/410-a-check-must-have-been-red.sh @@ -691,3 +691,36 @@ check "and a genuine observation says so with --reds-are-real" \ check "control: an all-PASS log still merges with no flag at all" \ "$(_led_rc merge --ledger "$_r946/d.tsv" --date 2026-09-10 "$_r946/green.log")" "0" unset _r946 + +# ---- the gate must refuse a census that contradicts its ledger (#952) -------- +# +# `gate` prints `ledger census: rows=N | never observed red=N` and does not +# compare that N to the budget. A six-row ledger with a budget claiming 2 is +# a four-row lie; today it is rc=0. The composed-tree case @OffgridwithJD +# measured was 775 rows against a budget of 769, same shape. +# +# The census is not a ceiling. The refusal is only that the two numbers +# describe the same file and disagree. + +_r952="$PGC_WORKDIR/r952"; mkdir -p "$_r952" +: > "$_r952/l.tsv" +: > "$_r952/g.log" +i=0 +while [ "$i" -lt 6 ]; do + printf 'demo\tq\trow-%s\tnever\t-\n' "$i" >> "$_r952/l.tsv" + printf 'RESULT\tdemo\tq\trow-%s\tPASS\t\n' "$i" >> "$_r952/g.log" + i=$((i + 1)) +done +printf 'checks run: 6\n' >> "$_r952/g.log" +printf 'demo\n' > "$_r952/reg" +printf 'suites_not_covered 0\nchecks_never_observed_red 2\n' > "$_r952/lie.txt" +printf 'suites_not_covered 0\nchecks_never_observed_red 6\n' > "$_r952/ok.txt" + +check "a budget that understates the ledger census is refused" \ + "$(_led_rc gate --ledger "$_r952/l.tsv" --budget "$_r952/lie.txt" --registered "$_r952/reg" "$_r952/g.log")" "1" +check "and the refusal names both values the budget and the ledger hold" \ + "$(_led_run gate --ledger "$_r952/l.tsv" --budget "$_r952/lie.txt" --registered "$_r952/reg" "$_r952/g.log" \ + | grep -c 'budget states 2, ledger has 6')" "1" +check "control: the same ledger passes when the census matches" \ + "$(_led_rc gate --ledger "$_r952/l.tsv" --budget "$_r952/ok.txt" --registered "$_r952/reg" "$_r952/g.log")" "0" +unset _r952