diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d59b7f0d..aeb65f35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -387,6 +387,21 @@ jobs: steps: - uses: actions/checkout@v4 + # THE LEDGER GATE NEEDS THE BASE BRANCH TO EXIST (#918). It compares the + # tracked ceiling in `test/check_ledger_budget.txt` against the one already + # on the PR's target, and it FAILS CLOSED when no trustworthy prior can be + # read rather than quietly enforcing less. + # + # actions/checkout fetches one ref at depth 1, so without this the base is + # absent and the gate stops the run -- correctly, but for a reason that is + # this workflow's to fix rather than the author's. Depth 1 is enough: only + # the file at that commit is read. + - name: fetch the PR base, for the ledger ceiling comparison + if: github.base_ref != '' + run: | + git fetch --depth=1 origin \ + "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}" + # The repository and the package lists first, on their own, because the # cache key below is derived from the versions apt resolves and cannot be # computed before the lists exist. @@ -539,7 +554,10 @@ jobs: - name: Collect logs on failure if: failure() run: | - for f in /tmp/pgcolumnar-matrix-*/*.log; do + # /tmp/pgcolumnar-logs, not the build directory: the runner removes + # each builddir at the end of its major, so this step used to search a + # path that no longer existed and print nothing. + for f in /tmp/pgcolumnar-logs/*.log /tmp/pgcolumnar-matrix-*/*.log; do [ -e "$f" ] || continue echo "===== $f =====" tail -60 "$f" diff --git a/CHANGELOG.md b/CHANGELOG.md index bd1f25f3..add322c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,78 @@ true until the next version shipped. ### Added +- Every check result is machine-readable, and counting a check is the same + operation as recording it (#917). + + `check`, `check_num` and `check_text` printed `PASS` or `FAIL` and nothing + else, so proving that a mutation reddened one NAMED check meant grepping prose + and retyping the result. That is also how a reverted guard once reported plain + green while the check count fell from 190 to 186: the suite passed, and the only + evidence anything had changed was a number nobody compared. + + `lib.sh` had eleven places that bumped `PGC_CHECKS`, each with its own outcome + line beside it, which is eleven chances to add a twelfth and forget the line. + `projections.sh` did exactly that with an `expect_fail` at ten call sites, for + as long as it existed. There is now one, `pgc_record`, so a helper cannot report + an outcome without being counted and cannot be counted without reporting one. + Every human line is byte-identical; 3,762 call sites is past what a careful + refactor can be trusted on, so both harnesses pin the exact strings. + + Each record names the suite, the part, the check, the verdict and the reason. + The part matters because `harness_selftest` sources 40-odd parts into one shell + and phrases its premises to be copied, so a key of suite and name is a key of + check NAMES rather than of checks: 583 records give 579 distinct pairs against + 582 distinct triples. It is derived from `BASH_SOURCE`, not from a convention. + + A skipped wall-clock check is a fourth counted outcome. Under + `PGC_SKIP_TIMING`, `check_timing` and `check_ratio_needs_quiet_machine` printed + a `SKIP` line a reader sees while leaving the count at zero and emitting no + record, in branches no arm reached. `checks run:` now reports the checks a suite + encountered rather than the ones it evaluated, and the summary reconciles four + counters against it. A suite that skipped every check reports `SKIPPED` rather + than `PASSED`, which the old zero-check condition caught only by accident. + + The matrix reconciles each suite's records against the count its log states, and + names the cause rather than the arithmetic: more records than counted is a check + that ran in a subshell, fewer is a counter bumped outside `pgc_record`. +- A ledger of which checks have ever been seen red, and under what (#918). + + Nothing recorded it. That is the gap that let 39 checks across 35 suites ship + unable to fail, three of them inside the suite whose whole purpose is to stop + exactly that: the gate answered "did anything print FAIL" and had never + answered "could anything print FAIL". + + It records that a named check WAS OBSERVED RED in a recorded run. It does not + claim the check is proven able to fail, which needs a named mutation applied + deliberately; conflating the two would put a claim in the ledger that nothing + measured. It is fed by every real failure, not only by deliberate mutation + runs. + + `run_all_versions.sh` runs the gate before it removes the build directory, + which is the only place a matrix run can reach every suite's log. What the gate + refuses is a check the committed ledger has never seen, in a suite the ledger + covers. Regenerating the ledger is the intended fix and a reviewable diff. + + The two tracked numbers are different kinds of thing, and the first design + treated both as ceilings and deadlocked. `checks_never_observed_red` is a + CENSUS: every new check enters as `never`, so bounding it means every added + check breaks the gate and the only way to land one is to raise a number the + design says may only fall. It shipped that way once, at 614 rows, 614 never, + ceiling 614. `suites_not_covered` IS a ceiling, because adding a check to a + covered suite does not move it, and the gate refuses to see it raised above its + previously committed value rather than leaving that to review. + + The row is keyed on suite, part and check name. The part matters because + `harness_selftest` sources 40-odd parts into one shell and phrases its premises + to be copied, so a name-only key is a key of check NAMES: 583 records give 579 + distinct pairs against 582 distinct triples. A rename is detected and named + rather than silently resetting a check's history to `never`, and the mutation + column accumulates a set rather than keeping only the most recent attack. + + Bad input fails closed. An unreadable log, an empty one, and a record missing + its verdict each returned success before, which is worse than no gate because + it certifies. + - A write that wrote no rows no longer passes as a fixture that built something (#432). diff --git a/CONTEXT.md b/CONTEXT.md index c43f816e..a92126b3 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -209,8 +209,7 @@ this was first counted. The file is the stable unit, so each file below is named with the mechanism that makes it a reference -- which is also what has to change for it to stop being one. -**The debt this starts with, on 2026-09-10: 3 python files and 7 shell files**, -with a fourth python file arriving in PR #923. +**The debt this starts with, on 2026-09-10: 3 python files and 7 shell files.** Python that reaches into shell: @@ -222,8 +221,6 @@ Python that reaches into shell: - `test_suite_accounting.py` -- reads `run_all_versions.sh`'s text, sources the real `lib.sh` from a suite it writes, and executes the real runner. - `pgc_cluster.py` -- sources the real `test/lib.sh`. -- `test_check_results_are_machine_readable.py` -- sources `./lib.sh`. Arrives - with PR #923; not on `main` yet. Shell whose subject is python: `lib.sh`, and `selftest/030`, `040`, `350`, `360`, `370`, `380`. diff --git a/test/advisory_lock_class.sh b/test/advisory_lock_class.sh index f69659af..a6c17bd1 100755 --- a/test/advisory_lock_class.sh +++ b/test/advisory_lock_class.sh @@ -127,7 +127,7 @@ if [ -n "$LK_CLASSID" ] && [ "$LK_CLASSID" -le 2147483647 ] && [ "$LK_OBJID" -le sleep 0.2 done else - echo "SKIP classid $LK_CLASSID or objid $LK_OBJID exceeds int4, so the SQL form cannot address it" + check_skip "the SQL form of the advisory lock" "SKIP classid $LK_CLASSID or objid $LK_OBJID exceeds int4, so the SQL form cannot address it" "classid or objid exceeds int4" fi # --------------------------------------------------------------------------- diff --git a/test/analyze_differential.sh b/test/analyze_differential.sh index 16eaa8e7..a2e66de3 100755 --- a/test/analyze_differential.sh +++ b/test/analyze_differential.sh @@ -54,13 +54,12 @@ ROWS=${PGC_ANALYZE_DIFF_ROWS:-50000} # The major is asserted first so an unreadable version is not mistaken for an old # one and reported as "supported, skipped". if ! pgc_is_number "${PGC_MAJOR:-}"; then - echo "FAIL could not read the server major, so the gate below cannot be trusted: got [${PGC_MAJOR:-}]" - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 + pgc_fail "could not read the server major, so the gate below cannot be trusted" \ + "got [${PGC_MAJOR:-}]" pgc_summary fi if [ "$PGC_MAJOR" -lt 18 ]; then - echo "SKIP pgcolumnar.analyze() needs pg_restore_attribute_stats (PG18+); this server is $PGC_MAJOR" + check_skip "the differential analyze path" "SKIP pgcolumnar.analyze() needs pg_restore_attribute_stats (PG18+); this server is $PGC_MAJOR" "needs pg_restore_attribute_stats, PG18+" pgc_summary fi diff --git a/test/analyze_function.sh b/test/analyze_function.sh index d491e1ef..63efdb1d 100755 --- a/test/analyze_function.sh +++ b/test/analyze_function.sh @@ -63,13 +63,12 @@ ROWS=${PGC_ANALYZE_ROWS:-500000} # The major is asserted first. An unreadable version must not be mistaken for an # old one, or a broken environment would report SKIP and look supported. if ! pgc_is_number "${PGC_MAJOR:-}"; then - echo "FAIL could not read the server major, so the gate below cannot be trusted: got [${PGC_MAJOR:-}]" - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 + pgc_fail "could not read the server major, so the gate below cannot be trusted" \ + "got [${PGC_MAJOR:-}]" pgc_summary fi if [ "$PGC_MAJOR" -lt 18 ]; then - echo "SKIP pgcolumnar.analyze() needs pg_restore_attribute_stats (PG18+); this server is $PGC_MAJOR" + check_skip "pgcolumnar.analyze()" "SKIP pgcolumnar.analyze() needs pg_restore_attribute_stats (PG18+); this server is $PGC_MAJOR" "needs pg_restore_attribute_stats, PG18+" pgc_summary fi diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv new file mode 100644 index 00000000..7f10fbaa --- /dev/null +++ b/test/check_ledger.tsv @@ -0,0 +1,826 @@ +harness_selftest 030-assertions nothing leaked into the squatter never - +harness_selftest 030-assertions pgc_port_free says the squatter's port is busy never - +harness_selftest 030-assertions squatter survived untouched never - +harness_selftest 030-assertions suite did not settle on the squatter's port never - +harness_selftest 030-assertions suite's cluster is its own never - +harness_selftest 030-assertions suite's own objects are visible to it never - +harness_selftest 040-the-detection-primitive-itself detection distinguishes it from ours never - +harness_selftest 040-the-detection-primitive-itself detection reports a foreign cluster's directory never - +harness_selftest 040-the-detection-primitive-itself guard accepts our own cluster never - +harness_selftest 040-the-detection-primitive-itself guard rejects a foreign cluster never - +harness_selftest 040-the-detection-primitive-itself premise: the runner answered --list-suites, so the two checks below mean something never - +harness_selftest 050-the-list-must-be-read-the a name after the array's closing paren is not read as a registered suite never - +harness_selftest 050-the-list-must-be-read-the and the mistake empties the whole array rather than appending to it never - +harness_selftest 050-the-list-must-be-read-the positive control: and it is a whole list, not one lucky line never - +harness_selftest 050-the-list-must-be-read-the positive control: the real runner's list is read, and contains isolation never - +harness_selftest 050-the-list-must-be-read-the premise: the fixture really does carry the stray name never - +harness_selftest 060-the-list-stays-sorted-which-is premise: C collation puts sort_status before sorted_projection never - +harness_selftest 060-the-list-stays-sorted-which-is the suite list is sorted in C order, so two new suites land in different places never - +harness_selftest 070-and-comm-s-two-inputs-must a file that uses comm pins the collation of every sort feeding it never - +harness_selftest 070-and-comm-s-two-inputs-must and a prefix of a registered name is not treated as registered never - +harness_selftest 070-and-comm-s-two-inputs-must every registered suite has a file never - +harness_selftest 070-and-comm-s-two-inputs-must every suite is registered in run_all_versions.sh never - +harness_selftest 070-and-comm-s-two-inputs-must negative control: and does not find one that is not never - +harness_selftest 070-and-comm-s-two-inputs-must positive control: the membership test finds a name that is registered never - +harness_selftest 070-and-comm-s-two-inputs-must premise: some suite still uses comm, or the check below is vacuous never - +harness_selftest 080-no-suite-pipes-a-captured-string a comment ending in a pipe is not a producer never - +harness_selftest 080-no-suite-pipes-a-captured-string a comment naming a TERMINATED tag still opens no heredoc, so the lines below it are seen never - +harness_selftest 080-no-suite-pipes-a-captured-string a comment naming a heredoc exempts nothing, so the line below it is still seen never - +harness_selftest 080-no-suite-pipes-a-captured-string an opener on a line of CODE with no terminator exempts nothing either never - +harness_selftest 080-no-suite-pipes-a-captured-string an opener with no terminator exempts nothing never - +harness_selftest 080-no-suite-pipes-a-captured-string and a pipeline split BEFORE the pipe, with the operator at column 1 never - +harness_selftest 080-no-suite-pipes-a-captured-string and an indented split is reported twice, at the reader and at the producer never - +harness_selftest 080-no-suite-pipes-a-captured-string and bench/ was in the scan, which is the hole this rule had never - +harness_selftest 080-no-suite-pipes-a-captured-string and does not cover the live one above it never - +harness_selftest 080-no-suite-pipes-a-captured-string and it follows the pipe across blank and comment lines never - +harness_selftest 080-no-suite-pipes-a-captured-string and reports it at the PRODUCER's line, which is the line that has to change never - +harness_selftest 080-no-suite-pipes-a-captured-string and selftest/ was in the scan, which is the hole that reddened #923 never - +harness_selftest 080-no-suite-pipes-a-captured-string and the heredoc exemption covers the one inside the heredoc, not the other never - +harness_selftest 080-no-suite-pipes-a-captured-string and the planted violation in that file is found by the pattern never - +harness_selftest 080-no-suite-pipes-a-captured-string and the scan examined the suites rather than finding nothing to read never - +harness_selftest 080-no-suite-pipes-a-captured-string and the sweep counts it once, because a hit is keyed on file:line never - +harness_selftest 080-no-suite-pipes-a-captured-string control: a real heredoc, opened in code and terminated, still exempts its body never - +harness_selftest 080-no-suite-pipes-a-captured-string control: piping a large string into grep -q reports a match as absent never - +harness_selftest 080-no-suite-pipes-a-captured-string no suite pipes a captured string into an early-exit reader never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and bash agrees -- with no command to continue, the reader alone will not parse never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and bash runs that file as one assignment -- no reader, no output, rc 0 never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and its second line really does hold the reader it must not flag never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and that body line is the one the pattern would otherwise flag never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and the exemption covers a minority of them, not the corpus never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and the line below it is what the pattern would flag never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and the line really does hold the reader it must not flag never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and the old echo/printf pattern did NOT catch it never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and the physical-line pattern alone saw neither of those two never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and the violation between that comment and its tag is what the pattern would flag never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: and without -H it reports a line number where the key wants a path never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: bash joins a pipeline across blank and comment lines, which is why the joiner must never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: indent that same reader by a tab and the physical-line pattern sees it unaided never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: no backslash continuation is followed by a blank or a comment never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: no line opens a heredoc AND continues, which is what lets the joiner ignore bodies never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: that shape matches in both streams at one line, which is what the key is for never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the candidate is on a line of code, not a comment never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the continuation detector found continuations to classify never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the file the old filename exclusion named holds no reader to exclude never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the fixture really has two backslash continuations to classify never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the fixture really has two heredoc openers for the detector to see never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the heredoc exemption found heredoc lines to exempt never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the joiner joined continuations, so the sweep is not still line-oriented never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the opener detector found heredocs to classify never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the physical stream names the file even when it reads only one of them never - +harness_selftest 080-no-suite-pipes-a-captured-string premise: the sweep read lines to classify never - +harness_selftest 080-no-suite-pipes-a-captured-string the continuing-opener detector finds an opener that ends in a pipe, and one that ends in a backslash never - +harness_selftest 080-no-suite-pipes-a-captured-string the gap detector finds a continuation followed by a comment, and one followed by a blank never - +harness_selftest 080-no-suite-pipes-a-captured-string the join CREATES a hit on a two-line quoted string, the sweep's one false positive never - +harness_selftest 080-no-suite-pipes-a-captured-string the joiner does not mistake a two-line || fallback for a split pipeline never - +harness_selftest 080-no-suite-pipes-a-captured-string the sweep catches a producer that is neither echo nor printf never - +harness_selftest 080-no-suite-pipes-a-captured-string the sweep does not mistake the || operator for a pipe never - +harness_selftest 080-no-suite-pipes-a-captured-string the sweep sees a pipeline split AFTER the pipe, which all six live sites were never - +harness_selftest 080-no-suite-pipes-a-captured-string the sweep's pattern sees both lines of the probe never - +harness_selftest 090-no-suite-hands-every-run-the no suite hands every run the same default port never - +harness_selftest 090-no-suite-hands-every-run-the no test picks a port from inside the ephemeral range never - +harness_selftest 100-the-assertions-that-refuse-an-empty check compares two empty strings and passes, which is why the rest exist never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num accepts a decimal and a sign never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num refuses a psql error message never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num refuses an md5, which is why check_text exists never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num refuses the word a yes/no check would produce never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num refuses two empty strings never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num still compares two real numbers never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_num still fails two unequal numbers never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_ratio fails a ratio outside its bound never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_ratio passes a ratio inside its bound never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_ratio refuses a zero denominator rather than dividing by it never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_ratio refuses a zero numerator, which is inside every bound never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_ratio refuses an empty measurement never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_text compares two md5 hashes, which check_num cannot never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_text refuses one empty side never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_text refuses two empty strings, where plain check passes never - +harness_selftest 100-the-assertions-that-refuse-an-empty check_text still fails two different strings never - +harness_selftest 100-the-assertions-that-refuse-an-empty pgc_require_tools fails on one that does not never - +harness_selftest 100-the-assertions-that-refuse-an-empty pgc_require_tools passes on tools that exist never - +harness_selftest 110-the-harness-must-say-which-binary pgc_setup reports the installed .so never - +harness_selftest 110-the-harness-must-say-which-binary the installed .so is the one this run built never - +harness_selftest 120-a-failing-suite-must-surface-the a failing suite names the first fatal event in its log never - +harness_selftest 120-a-failing-suite-must-surface-the premise: the 40-line tail is filler, not the marker never - +harness_selftest 120-a-failing-suite-must-surface-the premise: the sub-suite failed, so its summary ran never - +harness_selftest 130-the-sanitizer-subset-must-cover-the premise: at least one suite drives the C-level encoding selftest never - +harness_selftest 130-the-sanitizer-subset-must-cover-the premise: run_san.sh's default subset was found and is non-empty never - +harness_selftest 130-the-sanitizer-subset-must-cover-the the sanitizer subset runs every suite that drives the encoding selftest never - +harness_selftest 140-a-cluster-that-will-not-start and still matches a PANIC never - +harness_selftest 140-a-cluster-that-will-not-start and still matches a signal death never - +harness_selftest 140-a-cluster-that-will-not-start and still matches an AddressSanitizer report never - +harness_selftest 140-a-cluster-that-will-not-start but not a routine statement error never - +harness_selftest 140-a-cluster-that-will-not-start nor an ordinary log line never - +harness_selftest 140-a-cluster-that-will-not-start premise: the harness exposes its fatal pattern to be judged never - +harness_selftest 140-a-cluster-that-will-not-start the fatal pattern matches a library that will not load never - +harness_selftest 150-the-verdict-must-not-assert-a a mixed run reports both causes and neither as the whole story never - +harness_selftest 150-the-verdict-must-not-assert-a and is NOT made when our own postmaster died, which is the #537 case never - +harness_selftest 150-the-verdict-must-not-assert-a and so is the attempt count never - +harness_selftest 150-the-verdict-must-not-assert-a and the no-squatter verdict points at the server log never - +harness_selftest 150-the-verdict-must-not-assert-a premise: the verdict is composed somewhere it can be judged never - +harness_selftest 150-the-verdict-must-not-assert-a the ownership claim is made when a squatter held the port every time never - +harness_selftest 150-the-verdict-must-not-assert-a the port is named either way never - +harness_selftest 160-and-the-log-report-must-show and a log with no fatal line still reports rather than staying silent never - +harness_selftest 160-and-the-log-report-must-show premise: the log report is a function that can be fed a fixture never - +harness_selftest 160-and-the-log-report-must-show the report names the symbol that was actually missing never - +harness_selftest 170-and-lib-sh-must-ask-these and asks pgc_start_failure_message for the verdict never - +harness_selftest 170-and-lib-sh-must-ask-these and the old start-failure verdict is not echoed inline anywhere never - +harness_selftest 170-and-lib-sh-must-ask-these and the start path asks pgc_start_fatal_pattern, its deliberately wider one never - +harness_selftest 170-and-lib-sh-must-ask-these premise: lib.sh is readable, or every grep below approves nothing never - +harness_selftest 170-and-lib-sh-must-ask-these premise: the start-failure path still exists to be judged never - +harness_selftest 170-and-lib-sh-must-ask-these the failure path asks pgc_start_log_report for the reason never - +harness_selftest 170-and-lib-sh-must-ask-these the summary path asks pgc_fatal_pattern rather than hardcoding it never - +harness_selftest 180-the-port-walk-must-wrap-not a free port beyond the old 300-probe bound is still found never - +harness_selftest 180-the-port-walk-must-wrap-not a seed at the ceiling wraps past a busy top and still finds a port never - +harness_selftest 180-the-port-walk-must-wrap-not an entirely busy band reports itself full and terminates never - +harness_selftest 180-the-port-walk-must-wrap-not and the port it found is below the busy region, which is where wrapping lands never - +harness_selftest 180-the-port-walk-must-wrap-not premise: and really does allow the bottom never - +harness_selftest 180-the-port-walk-must-wrap-not premise: the auxiliary band has a width to wrap within never - +harness_selftest 180-the-port-walk-must-wrap-not premise: the real prober was restored, or every check after this lies never - +harness_selftest 180-the-port-walk-must-wrap-not premise: the stub frees exactly one port, 500 past the floor never - +harness_selftest 180-the-port-walk-must-wrap-not premise: the stub really does refuse the top of the band never - +harness_selftest 190-an-in-tree-build-must-not an unknown provenance is not reported as a major never - +harness_selftest 190-an-in-tree-build-must-not an unparseable stamp cleans rather than guessing never - +harness_selftest 190-an-in-tree-build-must-not and an empty WANT is refused rather than compared never - +harness_selftest 190-an-in-tree-build-must-not and in the other direction too never - +harness_selftest 190-an-in-tree-build-must-not and it is 3 bytes, not an escaped literal never - +harness_selftest 190-an-in-tree-build-must-not and it says plainly that no major was recorded never - +harness_selftest 190-an-in-tree-build-must-not building a DIFFERENT major needs a clean, which is the #536 case never - +harness_selftest 190-an-in-tree-build-must-not building the same major again needs no clean never - +harness_selftest 190-an-in-tree-build-must-not but a tree with no objects at all needs nothing, stamp or not never - +harness_selftest 190-an-in-tree-build-must-not objects with NO stamp are unknown provenance and must be cleaned never - +harness_selftest 190-an-in-tree-build-must-not premise: the build-stamp decision is exposed to be judged never - +harness_selftest 190-an-in-tree-build-must-not premise: the stamp writer is a function that can be exercised never - +harness_selftest 190-an-in-tree-build-must-not the build path asks pgc_build_needs_clean rather than merely naming it never - +harness_selftest 190-an-in-tree-build-must-not the stamp lib.sh writes is exactly the major never - +harness_selftest 200-additions-go-in-their-own-file premise: the parts directory exists and was sourced never - +harness_selftest 200-additions-go-in-their-own-file the driver holds no checks; they all live in parts never - +harness_selftest 200-additions-go-in-their-own-file the driver sources the parts by glob, not by a list never - +harness_selftest 210-no-suite-assigns-a-bash-special control: reads, longer names, and the deliberate RANDOM/SECONDS seeds are not flagged never - +harness_selftest 210-no-suite-assigns-a-bash-special control: the sweep catches an assignment to a bash special never - +harness_selftest 210-no-suite-assigns-a-bash-special no suite assigns to a bash special variable never - +harness_selftest 220-an-opt-in-upgrade-guard-must CI runs the extension-upgrade guard somewhere (#741) never - +harness_selftest 220-an-opt-in-upgrade-guard-must every PGC_RUN_UPGRADE-gated suite is excluded from the coverage runner (#741) never - +harness_selftest 220-an-opt-in-upgrade-guard-must premise: both not_a_suite definitions were found never - +harness_selftest 220-an-opt-in-upgrade-guard-must premise: not_a_suite says no to an ordinary suite, so its yes means something never - +harness_selftest 220-an-opt-in-upgrade-guard-must premise: run_coverage.sh defines not_a_suite and calls it never - +harness_selftest 220-an-opt-in-upgrade-guard-must premise: the PGC_RUN_UPGRADE block was found and names at least one suite never - +harness_selftest 220-an-opt-in-upgrade-guard-must premise: the population is the real test directory, not an empty glob never - +harness_selftest 220-an-opt-in-upgrade-guard-must premise: there are workflow files to search never - +harness_selftest 220-an-opt-in-upgrade-guard-must the coverage runner's not_a_suite agrees with the selftest's, both ways (#741) never - +harness_selftest 230-a-suite-connecting-by-socket-must every suite that connects by socket path sets unix_socket_directories never - +harness_selftest 230-a-suite-connecting-by-socket-must premise: at least one suite connects by a socket path, so this is not vacuous never - +harness_selftest 230-a-suite-connecting-by-socket-must premise: no TCP suite is counted as a socket user never - +harness_selftest 240-the-nightly-enumeration-must-not every nightly job is named in docs/testing.md (#741) never - +harness_selftest 240-the-nightly-enumeration-must-not premise: at least three nightly jobs were parsed, so the list is real never - +harness_selftest 240-the-nightly-enumeration-must-not premise: the nightly paragraph was located and is not empty never - +harness_selftest 240-the-nightly-enumeration-must-not premise: the nightly workflow and the testing doc are both present never - +harness_selftest 250-the-coverage-runner-must-refuse GCOV_PREFIX is exported before the suites run (#740) never - +harness_selftest 250-the-coverage-runner-must-refuse premise: both stray-counter probes were located never - +harness_selftest 250-the-coverage-runner-must-refuse premise: both the counter refusal and the lcov capture were located never - +harness_selftest 250-the-coverage-runner-must-refuse premise: the containment and the copy were both located never - +harness_selftest 250-the-coverage-runner-must-refuse premise: the coverage runner is present and parses never - +harness_selftest 250-the-coverage-runner-must-refuse premise: the guard's count directory and the capture's were both located never - +harness_selftest 250-the-coverage-runner-must-refuse premise: the redirect, the suite invocation and the copy-back were located never - +harness_selftest 250-the-coverage-runner-must-refuse the copy-back refuses a destination outside the tree (#740) never - +harness_selftest 250-the-coverage-runner-must-refuse the counters are returned beside their objects before the refusal (#740) never - +harness_selftest 250-the-coverage-runner-must-refuse the coverage runner refuses zero counters before it calls lcov (#740) never - +harness_selftest 250-the-coverage-runner-must-refuse the refusal looks in GCOV_PREFIX before the tree-wide walk (#740) never - +harness_selftest 250-the-coverage-runner-must-refuse the zero-counter guard counts the directory lcov captures (#740) never - +harness_selftest 260-an-ordered-comparison-must-use-the and it is the same suites, not merely the same count never - +harness_selftest 260-an-ordered-comparison-must-use-the every diff_query_ordered site actually names an ORDER BY never - +harness_selftest 260-an-ordered-comparison-must-use-the every suite using the ordered oracle asserts its premise never - +harness_selftest 260-an-ordered-comparison-must-use-the no diff_query site names an ORDER BY it cannot test (use diff_query_ordered) never - +harness_selftest 260-an-ordered-comparison-must-use-the premise: both comparison helpers are present never - +harness_selftest 260-an-ordered-comparison-must-use-the premise: both oracles are present never - +harness_selftest 260-an-ordered-comparison-must-use-the premise: some suite uses the ordered oracle, or the next check is vacuous never - +harness_selftest 260-an-ordered-comparison-must-use-the premise: the tree really contains continued diff_query calls to join never - +harness_selftest 260-an-ordered-comparison-must-use-the sorted_projection's two comparisons are ordered, its subject being order never - +harness_selftest 260-an-ordered-comparison-must-use-the the ordered oracle keeps the empty-result sentinel never - +harness_selftest 260-an-ordered-comparison-must-use-the the ordered oracle keeps the unique query-error sentinel never - +harness_selftest 260-an-ordered-comparison-must-use-the the ordered oracle numbers the rows as they arrive never - +harness_selftest 260-an-ordered-comparison-must-use-the the ordered oracle orders by row_number, so it keeps the query's order never - +harness_selftest 260-an-ordered-comparison-must-use-the the set oracle orders by the rendered row, so it is order-blind never - +harness_selftest 270-a-set-options-call-must-use-values no suite calls set_options with a value it will reject never - +harness_selftest 270-a-set-options-call-must-use-values premise: and does NOT fire when the error is the point, across a continuation never - +harness_selftest 270-a-set-options-call-must-use-values premise: every file containing a set_options call is in the sweep never - +harness_selftest 270-a-set-options-call-must-use-values premise: the detector fires on an out-of-range value that is NOT expect_error never - +harness_selftest 270-a-set-options-call-must-use-values premise: the set_options sweep read a substantial number of calls never - +harness_selftest 280-the-shared-cluster-config-must-not premise: and it is the right block (it sets the port and the preload) never - +harness_selftest 280-the-shared-cluster-config-must-not premise: the cluster-config block was located in lib.sh never - +harness_selftest 280-the-shared-cluster-config-must-not premise: the detector fires on the line that caused #799 never - +harness_selftest 280-the-shared-cluster-config-must-not the per-suite escape hatch PGC_EXTRA_CONF is still applied to the config never - +harness_selftest 280-the-shared-cluster-config-must-not the shared cluster config sets no pgcolumnar.* GUC never - +harness_selftest 290-a-preflight-that-built-nothing a preflight that built nothing does not report PASSED never - +harness_selftest 290-a-preflight-that-built-nothing a preflight that built nothing exits non-zero never - +harness_selftest 290-a-preflight-that-built-nothing a preflight that built nothing says how many it built never - +harness_selftest 290-a-preflight-that-built-nothing premise: and it built none of them never - +harness_selftest 290-a-preflight-that-built-nothing premise: the probe run skipped every major never - +harness_selftest 300-a-test-script-must-be-runnable and every executable script declares one never - +harness_selftest 300-a-test-script-must-be-runnable and every script a document names exists never - +harness_selftest 300-a-test-script-must-be-runnable and every script a document names is executable never - +harness_selftest 300-a-test-script-must-be-runnable control: an interpreter declared without the bit is caught never - +harness_selftest 300-a-test-script-must-be-runnable control: and a sourced fragment, with neither, is correct never - +harness_selftest 300-a-test-script-must-be-runnable control: and the same file with the bit is not never - +harness_selftest 300-a-test-script-must-be-runnable control: cp -a preserves the execute bit, so a staged tree reads the same never - +harness_selftest 300-a-test-script-must-be-runnable control: the bit without an interpreter is caught too never - +harness_selftest 300-a-test-script-must-be-runnable every script that declares an interpreter is executable never - +harness_selftest 300-a-test-script-must-be-runnable premise: a runnable script one level down is inside the population never - +harness_selftest 300-a-test-script-must-be-runnable premise: and bench/ is in the population never - +harness_selftest 300-a-test-script-must-be-runnable premise: and so are the fixture host tools never - +harness_selftest 300-a-test-script-must-be-runnable premise: and they name at least one command in every swept directory never - +harness_selftest 300-a-test-script-must-be-runnable premise: the documents name a population of commands, not none never - +harness_selftest 300-a-test-script-must-be-runnable premise: the sourced parts are inside the population, not pruned never - +harness_selftest 300-a-test-script-must-be-runnable premise: the sweep reads a population of scripts, not an empty find never - +harness_selftest 310-a-compiled-artifact-must-not-be and a compiled artifact written beside its source never - +harness_selftest 310-a-compiled-artifact-must-not-be and the tracked list names none of them never - +harness_selftest 310-a-compiled-artifact-must-not-be and the tree ignores the directory Python writes them to never - +harness_selftest 310-a-compiled-artifact-must-not-be no compiled Python artifact is tracked never - +harness_selftest 310-a-compiled-artifact-must-not-be premise: and git ls-files sees the harness it is being asked about never - +harness_selftest 310-a-compiled-artifact-must-not-be premise: and that a tracked source file is not never - +harness_selftest 310-a-compiled-artifact-must-not-be premise: check-ignore agrees a build object is already ignored never - +harness_selftest 310-a-compiled-artifact-must-not-be premise: the source tree is a git checkout never - +harness_selftest 320-a-check-that-could-not-run 67 without its line is a failure, not an INCOMPLETE taken on trust never - +harness_selftest 320-a-check-that-could-not-run a counter that drifts is caught rather than absorbed never - +harness_selftest 320-a-check-that-could-not-run a failure outranks an unrunnable check, and both are still counted never - +harness_selftest 320-a-check-that-could-not-run a passing ratio check is counted as a pass, not a failure never - +harness_selftest 320-a-check-that-could-not-run a suite of nothing but unrunnable checks is INCOMPLETE, not SKIPPED never - +harness_selftest 320-a-check-that-could-not-run a suite whose checks all passed still exits 0 PASSED never - +harness_selftest 320-a-check-that-could-not-run a suite with none says so as zero rather than staying silent never - +harness_selftest 320-a-check-that-could-not-run an INCOMPLETE suite fails its major never - +harness_selftest 320-a-check-that-could-not-run an unrunnable check counts toward checks run never - +harness_selftest 320-a-check-that-could-not-run an unrunnable reason outside the enum fails rather than being accepted never - +harness_selftest 320-a-check-that-could-not-run and 66 with its line a skip never - +harness_selftest 320-a-check-that-could-not-run and 67 with its line INCOMPLETE, which is not a pass never - +harness_selftest 320-a-check-that-could-not-run and a failing suite still does never - +harness_selftest 320-a-check-that-could-not-run and a skip does not, which is the one that must stay true never - +harness_selftest 320-a-check-that-could-not-run and a suite with no unrunnable checks reconciles too never - +harness_selftest 320-a-check-that-could-not-run and allows one that does, which is what it was written to allow never - +harness_selftest 320-a-check-that-could-not-run and an ordinary failure is still a failure never - +harness_selftest 320-a-check-that-could-not-run and both exempt a file that keeps its own counter without lib.sh never - +harness_selftest 320-a-check-that-could-not-run and it is not reported as having run no checks never - +harness_selftest 320-a-check-that-could-not-run and no write-only failure flag survives in the runner never - +harness_selftest 320-a-check-that-could-not-run and the suite holding it fails rather than reporting PASSED never - +harness_selftest 320-a-check-that-could-not-run and the suite that holds it still passes never - +harness_selftest 320-a-check-that-could-not-run and the unrunnable ones are reported as their own count never - +harness_selftest 320-a-check-that-could-not-run every direct write to PGC_CHECKS records an outcome too never - +harness_selftest 320-a-check-that-could-not-run lib.sh defines check_unrunnable never - +harness_selftest 320-a-check-that-could-not-run lib.sh defines the INCOMPLETE exit status never - +harness_selftest 320-a-check-that-could-not-run no non-zero status is classified as a pass never - +harness_selftest 320-a-check-that-could-not-run no suite that uses lib.sh's accounting writes PGC_CHECKS directly never - +harness_selftest 320-a-check-that-could-not-run one unrunnable check makes the suite INCOMPLETE, not passed never - +harness_selftest 320-a-check-that-could-not-run premise: the classifier evalled out of the runner is callable never - +harness_selftest 320-a-check-that-could-not-run premise: the fixtures carry the shapes these rules are about never - +harness_selftest 320-a-check-that-could-not-run premise: the harness library is where this part thinks it is never - +harness_selftest 320-a-check-that-could-not-run premise: the major-verdict mapping evalled out of the runner is callable never - +harness_selftest 320-a-check-that-could-not-run premise: the runner defines the classifier this part is about to eval never - +harness_selftest 320-a-check-that-could-not-run premise: the sweep read the corpus and found sites to classify never - +harness_selftest 320-a-check-that-could-not-run the original rule flags a bump that records no outcome never - +harness_selftest 320-a-check-that-could-not-run the runner calls a clean exit a pass never - +harness_selftest 320-a-check-that-could-not-run the runner's INCOMPLETE branch calls the mapping rather than a local flag never - +harness_selftest 320-a-check-that-could-not-run the stronger rule flags that same allowed bump, which is the change never - +harness_selftest 320-a-check-that-could-not-run the summary reconciles the three states against the total never - +harness_selftest 320-a-check-that-could-not-run the unrunnable check names itself, its reason code and its detail never - +harness_selftest 320-a-check-that-could-not-run while a pass does not never - +harness_selftest 330-the-incomplete-path-must-run-whole a suite with an unrunnable check exits 67 never - +harness_selftest 330-the-incomplete-path-must-run-whole an INCOMPLETE suite sets the flag the major verdict actually reads never - +harness_selftest 330-the-incomplete-path-must-run-whole and appears in the results string as INCOMPLETE never - +harness_selftest 330-the-incomplete-path-must-run-whole and counted as incomplete, so the tally can say so never - +harness_selftest 330-the-incomplete-path-must-run-whole and counts both suites as having run never - +harness_selftest 330-the-incomplete-path-must-run-whole and exactly one of them as incomplete never - +harness_selftest 330-the-incomplete-path-must-run-whole and is counted as having run never - +harness_selftest 330-the-incomplete-path-must-run-whole and is not counted as skipped, nor is the skip count disturbed never - +harness_selftest 330-the-incomplete-path-must-run-whole and its log carries the INCOMPLETE line the classifier needs never - +harness_selftest 330-the-incomplete-path-must-run-whole and neither as skipped never - +harness_selftest 330-the-incomplete-path-must-run-whole and no longer counts incompletes inline beside it never - +harness_selftest 330-the-incomplete-path-must-run-whole and records each suite's own verdict in the results string never - +harness_selftest 330-the-incomplete-path-must-run-whole and reprints the suite's own UNRUN line beneath it never - +harness_selftest 330-the-incomplete-path-must-run-whole and the UNRUN line the runner prints into the matrix output never - +harness_selftest 330-the-incomplete-path-must-run-whole and the run's overall status is failure never - +harness_selftest 330-the-incomplete-path-must-run-whole and the summary line carries the incomplete count a reader needs never - +harness_selftest 330-the-incomplete-path-must-run-whole and the tally announces it, with the reason lifted from the log never - +harness_selftest 330-the-incomplete-path-must-run-whole control fixture: a suite whose checks all ran exits 0 never - +harness_selftest 330-the-incomplete-path-must-run-whole control: and leaves the run's overall status alone never - +harness_selftest 330-the-incomplete-path-must-run-whole control: and still announces it never - +harness_selftest 330-the-incomplete-path-must-run-whole control: and still records that it ran, and how never - +harness_selftest 330-the-incomplete-path-must-run-whole control: and the major reports PASS never - +harness_selftest 330-the-incomplete-path-must-run-whole control: the same loop leaves a passing suite passing never - +harness_selftest 330-the-incomplete-path-must-run-whole every function the collect loop calls is defined here never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: all five runner functions were extracted, not empty ranges never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: all three runner functions were extracted, not empty ranges never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: and all five are callable never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: and all three are callable never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: and each extraction ends at its own closing brace never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: the major-verdict branch was extracted, not an empty range never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: the runner's collect loop was extracted, not an empty range never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: the selftest has a workdir to build fixtures in never - +harness_selftest 330-the-incomplete-path-must-run-whole premise: the sweep found the loop's callers to check never - +harness_selftest 330-the-incomplete-path-must-run-whole running the real loop over both fixtures fails the major never - +harness_selftest 330-the-incomplete-path-must-run-whole the loop delegates each verdict to pgc_tally_suite never - +harness_selftest 330-the-incomplete-path-must-run-whole the runner classifies the file that suite actually produced never - +harness_selftest 330-the-incomplete-path-must-run-whole with an incomplete suite in the tally the major reports FAIL never - +harness_selftest 340-the-binary-must-be-built-from PREMISE and the target really holds sources find would otherwise hash never - +harness_selftest 340-the-binary-must-be-built-from PREMISE the Makefile's recursion was actually parsed never - +harness_selftest 340-the-binary-must-be-built-from PREMISE the copy discovers the same build directories as the real tree never - +harness_selftest 340-the-binary-must-be-built-from PREMISE the fingerprint covers at least src never - +harness_selftest 340-the-binary-must-be-built-from PREMISE the fixture's src really is a symlink never - +harness_selftest 340-the-binary-must-be-built-from a /./ segment hashes the same tree the same way never - +harness_selftest 340-the-binary-must-be-built-from a /src/.. segment hashes the same tree the same way never - +harness_selftest 340-the-binary-must-be-built-from a caller passing a major is caught never - +harness_selftest 340-the-binary-must-be-built-from a file that is not a build input does not move it never - +harness_selftest 340-the-binary-must-be-built-from a fingerprint different from the record is stale never - +harness_selftest 340-the-binary-must-be-built-from a fingerprint equal to the record is fresh never - +harness_selftest 340-the-binary-must-be-built-from a fingerprint is 12 hex characters never - +harness_selftest 340-the-binary-must-be-built-from a library newer than the running server is REFUSED never - +harness_selftest 340-the-binary-must-be-built-from a library older than the running server is accepted never - +harness_selftest 340-the-binary-must-be-built-from a missing binary timestamp is unknown, not predates never - +harness_selftest 340-the-binary-must-be-built-from a missing postmaster timestamp is unknown, not predates never - +harness_selftest 340-the-binary-must-be-built-from a new source file under objstore moves the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from a relative path hashes the same tree the same way never - +harness_selftest 340-the-binary-must-be-built-from a server older than the binary predates it never - +harness_selftest 340-the-binary-must-be-built-from a server started after the binary is fresh never - +harness_selftest 340-the-binary-must-be-built-from a server started at the same second is fresh never - +harness_selftest 340-the-binary-must-be-built-from a symlink to the tree hashes it the same way never - +harness_selftest 340-the-binary-must-be-built-from a symlinked src contributes nothing, as find -P contributes nothing never - +harness_selftest 340-the-binary-must-be-built-from a trailing slash hashes the same tree the same way never - +harness_selftest 340-the-binary-must-be-built-from a tree with no hashable file yields no fingerprint never - +harness_selftest 340-the-binary-must-be-built-from adding a source file moves it never - +harness_selftest 340-the-binary-must-be-built-from an added file appears in the manifest by name never - +harness_selftest 340-the-binary-must-be-built-from an added file shows up in the report never - +harness_selftest 340-the-binary-must-be-built-from an empty manifest is reported as empty, not as silence never - +harness_selftest 340-the-binary-must-be-built-from an unhashable tree has an empty manifest never - +harness_selftest 340-the-binary-must-be-built-from an unreadable b.c yields no fingerprint, not a wrong one never - +harness_selftest 340-the-binary-must-be-built-from an unreadable c.c yields no fingerprint, not a wrong one never - +harness_selftest 340-the-binary-must-be-built-from an unreadable library is not a failure never - +harness_selftest 340-the-binary-must-be-built-from and a non-numeric timestamp is unknown rather than compared as text never - +harness_selftest 340-the-binary-must-be-built-from and an uncomputable current fingerprint is unknown, not stale never - +harness_selftest 340-the-binary-must-be-built-from and comparing two manifests names it rather than saying 'changed' never - +harness_selftest 340-the-binary-must-be-built-from and it says so rather than staying silent never - +harness_selftest 340-the-binary-must-be-built-from and removing it restores the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from and restoring it restores the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from and restoring the partition restores the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from and the major is still readable in the name never - +harness_selftest 340-the-binary-must-be-built-from and the reader reads back the fingerprint the writer recorded never - +harness_selftest 340-the-binary-must-be-built-from and the refusal says the server must be restarted never - +harness_selftest 340-the-binary-must-be-built-from and two pg_configs for one prefix share a stamp, keyed on pkglibdir never - +harness_selftest 340-the-binary-must-be-built-from but it says which question went unanswered never - +harness_selftest 340-the-binary-must-be-built-from control: a caller passing a pg_config is not flagged never - +harness_selftest 340-the-binary-must-be-built-from control: a readable run still reads fresh never - +harness_selftest 340-the-binary-must-be-built-from control: a real content change still moves the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from control: a real src directory is still hashed never - +harness_selftest 340-the-binary-must-be-built-from control: and it still succeeds on a writable one never - +harness_selftest 340-the-binary-must-be-built-from control: and restoring the content restores the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from control: and the tree fingerprints again once it is readable never - +harness_selftest 340-the-binary-must-be-built-from control: the same pg_config twice gives the same path never - +harness_selftest 340-the-binary-must-be-built-from control: writing the value it was given never - +harness_selftest 340-the-binary-must-be-built-from each manifest line is a tree-relative path and a digest never - +harness_selftest 340-the-binary-must-be-built-from editing a source file moves the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from every directory the Makefile builds from is in the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from moving bytes between files moves the fingerprint never - +harness_selftest 340-the-binary-must-be-built-from no caller passes a major where a pg_config belongs never - +harness_selftest 340-the-binary-must-be-built-from no record at all is unknown, not fresh never - +harness_selftest 340-the-binary-must-be-built-from one tree, one fingerprint, whatever the locale never - +harness_selftest 340-the-binary-must-be-built-from premise: and the stamp really was not written, so the arm is not vacuous never - +harness_selftest 340-the-binary-must-be-built-from premise: at least two locales are installed to compare never - +harness_selftest 340-the-binary-must-be-built-from premise: both fake configs report the same major, which is the whole point never - +harness_selftest 340-the-binary-must-be-built-from premise: every locale produced a fingerprint never - +harness_selftest 340-the-binary-must-be-built-from premise: the argument parser reads the second argument at all never - +harness_selftest 340-the-binary-must-be-built-from premise: the build path ran to completion, so a stamp was due never - +harness_selftest 340-the-binary-must-be-built-from premise: the partition fixture fingerprints at all never - +harness_selftest 340-the-binary-must-be-built-from premise: the same function returns a fingerprint for a real tree never - +harness_selftest 340-the-binary-must-be-built-from premise: the spelling fixture fingerprints at all never - +harness_selftest 340-the-binary-must-be-built-from premise: the sweep finds the call sites it is meant to police never - +harness_selftest 340-the-binary-must-be-built-from premise: the tree fingerprints to something when it is readable never - +harness_selftest 340-the-binary-must-be-built-from premise: the unprivileged read agrees while everything is readable never - +harness_selftest 340-the-binary-must-be-built-from premise: the writer wrote a stamp at all never - +harness_selftest 340-the-binary-must-be-built-from renaming a source file moves the fingerprint too never - +harness_selftest 340-the-binary-must-be-built-from so the tree still fingerprints from its root files alone never - +harness_selftest 340-the-binary-must-be-built-from so the verdict is fresh, not unknown never - +harness_selftest 340-the-binary-must-be-built-from so the verdict is unknown -- UNVERIFIED -- and never stale never - +harness_selftest 340-the-binary-must-be-built-from the fingerprint is the hash of the manifest never - +harness_selftest 340-the-binary-must-be-built-from the fixed fingerprint equals what the previous implementation produced never - +harness_selftest 340-the-binary-must-be-built-from the manifest is tree-relative, never absolute never - +harness_selftest 340-the-binary-must-be-built-from the manifest names every file the fingerprint hashes never - +harness_selftest 340-the-binary-must-be-built-from the probe is written outside the live source tree never - +harness_selftest 340-the-binary-must-be-built-from the report names each hashed file never - +harness_selftest 340-the-binary-must-be-built-from the report states how many files it hashed never - +harness_selftest 340-the-binary-must-be-built-from the same tree fingerprints the same twice never - +harness_selftest 340-the-binary-must-be-built-from the stamp writer reports failure on an unwritable target never - +harness_selftest 340-the-binary-must-be-built-from the writer writes the file the reader looks for never - +harness_selftest 340-the-binary-must-be-built-from two installations of one major get different stamp paths never - +harness_selftest 340-the-binary-must-be-built-from two unreadable pg_configs do not alias onto one stamp never - +harness_selftest 350-the-pytest-corpus-must-be README.md quotes the number of modes the inventory names as refused never - +harness_selftest 350-the-pytest-corpus-must-be TESTS.md states no totals line for a merge to get wrong never - +harness_selftest 350-the-pytest-corpus-must-be TESTS.md states the counted number as well never - +harness_selftest 350-the-pytest-corpus-must-be a documented file that does not exist is named never - +harness_selftest 350-the-pytest-corpus-must-be a documented test that does not exist is named, not passed over never - +harness_selftest 350-the-pytest-corpus-must-be a name defined in two files is named, not passed over never - +harness_selftest 350-the-pytest-corpus-must-be a prose total that disagrees with the ids is visible never - +harness_selftest 350-the-pytest-corpus-must-be a stated total that disagrees with disk is visible never - +harness_selftest 350-the-pytest-corpus-must-be a stated total that disagrees with the ids is visible never - +harness_selftest 350-the-pytest-corpus-must-be an absent prose total is empty rather than a stray number never - +harness_selftest 350-the-pytest-corpus-must-be an absent total is empty rather than a number that happens to match never - +harness_selftest 350-the-pytest-corpus-must-be an anchor that strips the underscores is named, not passed over never - +harness_selftest 350-the-pytest-corpus-must-be an id named twice counts once never - +harness_selftest 350-the-pytest-corpus-must-be an id of fewer than three words is not counted as a mode never - +harness_selftest 350-the-pytest-corpus-must-be an unbackticked name in prose is not treated as a claim never - +harness_selftest 350-the-pytest-corpus-must-be an undocumented file is caught along with the tests inside it never - +harness_selftest 350-the-pytest-corpus-must-be an undocumented test is named rather than passed over never - +harness_selftest 350-the-pytest-corpus-must-be and derives the pins from requirements-test.txt never - +harness_selftest 350-the-pytest-corpus-must-be and it derives the file list rather than repeating it never - +harness_selftest 350-the-pytest-corpus-must-be and the same comparison agrees on the fixture that is right never - +harness_selftest 350-the-pytest-corpus-must-be control: a document naming only what exists is clean never - +harness_selftest 350-the-pytest-corpus-must-be control: a fully documented corpus reports nothing missing never - +harness_selftest 350-the-pytest-corpus-must-be control: an anchor that keeps the underscores resolves never - +harness_selftest 350-the-pytest-corpus-must-be control: distinct names in the same corpus report no duplicate never - +harness_selftest 350-the-pytest-corpus-must-be every in-document link in README.md reaches a heading never - +harness_selftest 350-the-pytest-corpus-must-be every in-document link in TESTS.md reaches a heading never - +harness_selftest 350-the-pytest-corpus-must-be every in-document link in VACUITY_MODES.md reaches a heading never - +harness_selftest 350-the-pytest-corpus-must-be every test file and every test in the corpus is named in TESTS.md never - +harness_selftest 350-the-pytest-corpus-must-be every test the document names exists in the corpus never - +harness_selftest 350-the-pytest-corpus-must-be no test name is defined twice in the corpus never - +harness_selftest 350-the-pytest-corpus-must-be premise: the CI workflow is where this part thinks it is never - +harness_selftest 350-the-pytest-corpus-must-be premise: the corpus carries the documentation this part polices never - +harness_selftest 350-the-pytest-corpus-must-be premise: the counting rule finds modes at all never - +harness_selftest 350-the-pytest-corpus-must-be premise: the link sweep saw the directory's documents never - +harness_selftest 350-the-pytest-corpus-must-be premise: the mode inventory is where this part thinks it is never - +harness_selftest 350-the-pytest-corpus-must-be premise: the pytest corpus is where this part thinks it is never - +harness_selftest 350-the-pytest-corpus-must-be premise: the reader still finds a totals line when one is there never - +harness_selftest 350-the-pytest-corpus-must-be premise: the reverse sweep reads backticked names at all never - +harness_selftest 350-the-pytest-corpus-must-be premise: the sweep found the corpus rather than an empty glob never - +harness_selftest 350-the-pytest-corpus-must-be premise: the sweep reads the contents list's links never - +harness_selftest 350-the-pytest-corpus-must-be section 1a's document total is the sum of its two sections never - +harness_selftest 350-the-pytest-corpus-must-be section 1a's not-refused total is the count of ids in section 3 never - +harness_selftest 350-the-pytest-corpus-must-be section 1a's refused total is the count of ids in section 2 never - +harness_selftest 350-the-pytest-corpus-must-be section 2's opening states the counted number of refused modes never - +harness_selftest 350-the-pytest-corpus-must-be the admitted gap is the run total minus what is written down never - +harness_selftest 350-the-pytest-corpus-must-be the anchor rule drops punctuation and keeps underscores never - +harness_selftest 350-the-pytest-corpus-must-be the closing paragraph states the counted number too never - +harness_selftest 350-the-pytest-corpus-must-be the counter counts a fixture's section 2 never - +harness_selftest 350-the-pytest-corpus-must-be the counter counts a fixture's section 3 never - +harness_selftest 350-the-pytest-corpus-must-be the counter stops at the next heading never - +harness_selftest 350-the-pytest-corpus-must-be the gate runs the harness guards never - +harness_selftest 350-the-pytest-corpus-must-be the row's value is read, not a digit inside its label never - +harness_selftest 350-the-pytest-corpus-must-be the sweep counts the fixture's tests and files never - +harness_selftest 360-an-unrunnable-pytest-test-must a comparison on the exit status is not counted as an assignment never - +harness_selftest 360-an-unrunnable-pytest-test-must a drifted exit code is visible rather than absorbed never - +harness_selftest 360-an-unrunnable-pytest-test-must a write-only unrunnable field is caught never - +harness_selftest 360-an-unrunnable-pytest-test-must an unconditional exit override is caught by the dominance arm never - +harness_selftest 360-an-unrunnable-pytest-test-must and only ever moves a run off zero, so a failure still dominates never - +harness_selftest 360-an-unrunnable-pytest-test-must and something READS it, rather than only writing it never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: and that same fixture does show the write, so the arm is not blind never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: lib.sh states an INCOMPLETE exit code this part could read never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: the harness library is where this part thinks it is never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: the pytest layer is where this part thinks it is never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: the pytest layer states one too never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: while a real assignment on the same line shape IS counted never - +harness_selftest 360-an-unrunnable-pytest-test-must premise: while the real layer satisfies that same arm never - +harness_selftest 360-an-unrunnable-pytest-test-must the layer ends a session by setting its exit status never - +harness_selftest 360-an-unrunnable-pytest-test-must the layer prints the unrunnable reason in lib.sh's shape never - +harness_selftest 360-an-unrunnable-pytest-test-must the layer still writes the unrunnable state never - +harness_selftest 360-an-unrunnable-pytest-test-must the two harnesses agree on the INCOMPLETE exit code never - +harness_selftest 370-the-plan-marker-guard-must a neutered absent arm is caught never - +harness_selftest 370-the-plan-marker-guard-must a neutered empty-plan refusal is caught never - +harness_selftest 370-the-plan-marker-guard-must a neutered present arm is caught never - +harness_selftest 370-the-plan-marker-guard-must and that refusal is a VacuityError, not an ordinary assertion never - +harness_selftest 370-the-plan-marker-guard-must plan_marker keeps the arm that fails when the key is absent never - +harness_selftest 370-the-plan-marker-guard-must plan_marker keeps the arm that fails when the key is present never - +harness_selftest 370-the-plan-marker-guard-must plan_marker refuses a plan with no nodes at all never - +harness_selftest 370-the-plan-marker-guard-must premise: both line numbers were found, so the ordering arm can mean something never - +harness_selftest 370-the-plan-marker-guard-must premise: plan_marker's body was actually cut out of the file never - +harness_selftest 370-the-plan-marker-guard-must premise: the pytest layer is where this part thinks it is never - +harness_selftest 370-the-plan-marker-guard-must premise: while the real body satisfies all three, so the greps work never - +harness_selftest 370-the-plan-marker-guard-must the empty-plan refusal precedes the arm it protects never - +harness_selftest 380-the-pytest-cluster-helpers a caller that reimplements the digest is caught never - +harness_selftest 380-the-pytest-cluster-helpers a fingerprint that reads src only is caught never - +harness_selftest 380-the-pytest-cluster-helpers a make_cluster with no cleanup is caught never - +harness_selftest 380-the-pytest-cluster-helpers an import from the pytest tree is caught never - +harness_selftest 380-the-pytest-cluster-helpers and a hard-coded module list is caught by the name arm never - +harness_selftest 380-the-pytest-cluster-helpers and it catches BaseException, so an interrupt cleans up too never - +harness_selftest 380-the-pytest-cluster-helpers and it names no module directory, so it is a derivation and not a list never - +harness_selftest 380-the-pytest-cluster-helpers and it stops a partially started cluster before removing the tree never - +harness_selftest 380-the-pytest-cluster-helpers and no longer mixes in the bare filename never - +harness_selftest 380-the-pytest-cluster-helpers and the original error is re-raised rather than swallowed never - +harness_selftest 380-the-pytest-cluster-helpers and the shell keeps none either never - +harness_selftest 380-the-pytest-cluster-helpers make_cluster removes its tree when setup raises never - +harness_selftest 380-the-pytest-cluster-helpers premise: and the real helper still carries its cleanup never - +harness_selftest 380-the-pytest-cluster-helpers premise: make_cluster's body was actually cut out of the file never - +harness_selftest 380-the-pytest-cluster-helpers premise: the one fingerprint implementation is where this part thinks it is never - +harness_selftest 380-the-pytest-cluster-helpers premise: the pytest cluster helper is where this part thinks it is never - +harness_selftest 380-the-pytest-cluster-helpers premise: while the real module satisfies the derivation arm never - +harness_selftest 380-the-pytest-cluster-helpers the fingerprint derives its build directories from a Makefile on disk never - +harness_selftest 380-the-pytest-cluster-helpers the hash mixes in each file's path relative to the tree, not its name never - +harness_selftest 380-the-pytest-cluster-helpers the module imports nothing from the pytest tree never - +harness_selftest 380-the-pytest-cluster-helpers the pytest helper keeps no private fingerprint implementation never - +harness_selftest 390-a-registered-suite-must-account a NEW unaccounted suite fails even while the known debt is excused never - +harness_selftest 390-a-registered-suite-must-account a comment mentioning pgc_summary is not a declaration never - +harness_selftest 390-a-registered-suite-must-account a declared suite that produced no accounting is caught never - +harness_selftest 390-a-registered-suite-must-account a declared suite the driver never dispatched reconciles never - +harness_selftest 390-a-registered-suite-must-account a file that does not exist is reported absent, not exempt never - +harness_selftest 390-a-registered-suite-must-account a hash inside a word does not hide the call after it never - +harness_selftest 390-a-registered-suite-must-account a log carrying lib.sh's accounting line is accounted never - +harness_selftest 390-a-registered-suite-must-account a log carrying neither is not accounted never - +harness_selftest 390-a-registered-suite-must-account a log claiming PASSED without the accounting line shows none never - +harness_selftest 390-a-registered-suite-must-account a long suite that calls pgc_summary still declares accounting never - +harness_selftest 390-a-registered-suite-must-account a longer name containing pgc_summary is not a declaration never - +harness_selftest 390-a-registered-suite-must-account a passing log shows accounting never - +harness_selftest 390-a-registered-suite-must-account a registered suite that is accounted by nothing FAILS never - +harness_selftest 390-a-registered-suite-must-account a suite recorded as known debt passes never - +harness_selftest 390-a-registered-suite-must-account a suite recorded as never dispatched that DID account is caught never - +harness_selftest 390-a-registered-suite-must-account a suite that accounted passes never - +harness_selftest 390-a-registered-suite-must-account a suite that calls pgc_summary declares accounting never - +harness_selftest 390-a-registered-suite-must-account a suite that never calls it does not never - +harness_selftest 390-a-registered-suite-must-account a suite that now accounts but is still listed as debt is reported never - +harness_selftest 390-a-registered-suite-must-account a suite the driver never dispatched passes never - +harness_selftest 390-a-registered-suite-must-account a trailing comment after the call does not hide it never - +harness_selftest 390-a-registered-suite-must-account an absent log shows no accounting rather than erroring never - +harness_selftest 390-a-registered-suite-must-account an accounting line that does not start its line is refused never - +harness_selftest 390-a-registered-suite-must-account an indented comment is still a comment never - +harness_selftest 390-a-registered-suite-must-account an undeclared suite that DID account is caught too never - +harness_selftest 390-a-registered-suite-must-account and a failed population reconciliation fails the major never - +harness_selftest 390-a-registered-suite-must-account and a failed reconciliation sets the per-major failure flag never - +harness_selftest 390-a-registered-suite-must-account and a log carrying only its OWN checks-run line is accounted too never - +harness_selftest 390-a-registered-suite-must-account and a reworded producer line is refused, so the arm can fail never - +harness_selftest 390-a-registered-suite-must-account and a skip, which reached the summary and counted zero never - +harness_selftest 390-a-registered-suite-must-account and absent is distinguishable from a present file that does not declare never - +harness_selftest 390-a-registered-suite-must-account and an incomplete never - +harness_selftest 390-a-registered-suite-must-account and debt naming a suite that is not registered is reported too never - +harness_selftest 390-a-registered-suite-must-account and it agrees with the real reader on a SHORT file, which is why it survived review never - +harness_selftest 390-a-registered-suite-must-account and it is NAMED, so the reader does not have to diff two lists never - +harness_selftest 390-a-registered-suite-must-account and it is named as that fault, not as one of the other two never - +harness_selftest 390-a-registered-suite-must-account and it is named as the opposite fault, not the same one never - +harness_selftest 390-a-registered-suite-must-account and it is named, which the symmetry check could never do never - +harness_selftest 390-a-registered-suite-must-account and prose containing the word does not count as the line never - +harness_selftest 390-a-registered-suite-must-account and so does a failing one, which is the point never - +harness_selftest 390-a-registered-suite-must-account and the excused one is not named as a failure never - +harness_selftest 390-a-registered-suite-must-account and the reader answers no on it, which is the wrong answer the arm catches never - +harness_selftest 390-a-registered-suite-must-account and the real function reconciles the same input, so the arm is not noise never - +harness_selftest 390-a-registered-suite-must-account and the reconciliation is given that record never - +harness_selftest 390-a-registered-suite-must-account and the registered file is written from the SUITES array itself never - +harness_selftest 390-a-registered-suite-must-account and without that record the same run is still caught never - +harness_selftest 390-a-registered-suite-must-account equal sets reconcile never - +harness_selftest 390-a-registered-suite-must-account every registered suite has a file never - +harness_selftest 390-a-registered-suite-must-account nor no for every one of them never - +harness_selftest 390-a-registered-suite-must-account opposite errors do not cancel: both directions are reported never - +harness_selftest 390-a-registered-suite-must-account premise: and produced exactly one accounting line to be read never - +harness_selftest 390-a-registered-suite-must-account premise: and that count excludes the definition line, which mentions it never - +harness_selftest 390-a-registered-suite-must-account premise: and the accounted reader that feeds it never - +harness_selftest 390-a-registered-suite-must-account premise: and the real function still does never - +harness_selftest 390-a-registered-suite-must-account premise: pipefail is on, which is the condition the bug needs never - +harness_selftest 390-a-registered-suite-must-account premise: the declaration reader evalled out of the runner is callable never - +harness_selftest 390-a-registered-suite-must-account premise: the drift changed the line the reader looks for never - +harness_selftest 390-a-registered-suite-must-account premise: the fixture carries a well-formed accounting line, just indented never - +harness_selftest 390-a-registered-suite-must-account premise: the fixture is long enough to lose the race never - +harness_selftest 390-a-registered-suite-must-account premise: the fixture really does hide its call from the stripper never - +harness_selftest 390-a-registered-suite-must-account premise: the mutation applied -- the twin no longer sorts its inputs never - +harness_selftest 390-a-registered-suite-must-account premise: the observation reader evalled out of the runner is callable never - +harness_selftest 390-a-registered-suite-must-account premise: the population reconciliation is callable never - +harness_selftest 390-a-registered-suite-must-account premise: the real suite ran and reached its summary never - +harness_selftest 390-a-registered-suite-must-account premise: the reconciliation evalled out of the runner is callable never - +harness_selftest 390-a-registered-suite-must-account premise: the registered list is not empty, so the partition means something never - +harness_selftest 390-a-registered-suite-must-account premise: the runner defines the declaration reader this part evals never - +harness_selftest 390-a-registered-suite-must-account premise: the runner defines the observation reader this part evals never - +harness_selftest 390-a-registered-suite-must-account premise: the runner defines the population reconciliation never - +harness_selftest 390-a-registered-suite-must-account premise: the runner defines the reconciliation this part evals never - +harness_selftest 390-a-registered-suite-must-account premise: the twin script was written and is runnable never - +harness_selftest 390-a-registered-suite-must-account premise: the unsorted twin is callable never - +harness_selftest 390-a-registered-suite-must-account the debt file is in the tree never - +harness_selftest 390-a-registered-suite-must-account the grep -q shape is the one that gets this wrong under pipefail never - +harness_selftest 390-a-registered-suite-must-account the identity catches comm reading unsorted input never - +harness_selftest 390-a-registered-suite-must-account the partition over the real suite list adds up never - +harness_selftest 390-a-registered-suite-must-account the population partitions, and prints inputs == sum(buckets) never - +harness_selftest 390-a-registered-suite-must-account the reader accepts the line the producer actually emits never - +harness_selftest 390-a-registered-suite-must-account the reader does not answer yes for every registered suite never - +harness_selftest 390-a-registered-suite-must-account the reconciliation prints inputs == sum(buckets) never - +harness_selftest 390-a-registered-suite-must-account the record cannot introduce a suite the source never declared never - +harness_selftest 390-a-registered-suite-must-account the runner calls the population reconciliation never - +harness_selftest 390-a-registered-suite-must-account the runner calls the reconciliation, not merely defines it never - +harness_selftest 390-a-registered-suite-must-account the skip branch records the suite it did not dispatch never - +harness_selftest 390-a-registered-suite-must-account the stripper hides no pgc_summary call in any registered suite never - +harness_selftest 400-a-check-result-must-be-machine a bogus reason code records FAIL, not UNRUN never - +harness_selftest 400-a-check-result-must-be-machine a failing check emits exactly one record never - +harness_selftest 400-a-check-result-must-be-machine a failing check still prints its old line never - +harness_selftest 400-a-check-result-must-be-machine a log that never stated a count is not silently accepted never - +harness_selftest 400-a-check-result-must-be-machine a log whose records match its stated count reconciles never - +harness_selftest 400-a-check-result-must-be-machine a log with fewer records than it claims is caught never - +harness_selftest 400-a-check-result-must-be-machine a log with more records than it claims is caught too never - +harness_selftest 400-a-check-result-must-be-machine a passing check emits exactly one record never - +harness_selftest 400-a-check-result-must-be-machine a passing check still prints its old line never - +harness_selftest 400-a-check-result-must-be-machine a record carrying extra fields does not reconcile never - +harness_selftest 400-a-check-result-must-be-machine a record missing fields does not reconcile never - +harness_selftest 400-a-check-result-must-be-machine a skipped ratio check emits exactly one record never - +harness_selftest 400-a-check-result-must-be-machine a skipped timing check emits exactly one record never - +harness_selftest 400-a-check-result-must-be-machine a suite that skipped every check did not pass never - +harness_selftest 400-a-check-result-must-be-machine a verdict pgc_record cannot emit does not reconcile never - +harness_selftest 400-a-check-result-must-be-machine an empty check name does not reconcile never - +harness_selftest 400-a-check-result-must-be-machine an unrunnable check emits exactly one record never - +harness_selftest 400-a-check-result-must-be-machine an unrunnable check still prints its old line never - +harness_selftest 400-a-check-result-must-be-machine and is counted as a pass, not a skip never - +harness_selftest 400-a-check-result-must-be-machine and it is counted never - +harness_selftest 400-a-check-result-must-be-machine and it is counted, so checks run: reports it never - +harness_selftest 400-a-check-result-must-be-machine and its human line is unchanged never - +harness_selftest 400-a-check-result-must-be-machine and its name field is the check's name, spaces intact never - +harness_selftest 400-a-check-result-must-be-machine and its verdict field says FAIL never - +harness_selftest 400-a-check-result-must-be-machine and its verdict field says PASS never - +harness_selftest 400-a-check-result-must-be-machine and its verdict field says UNRUN, which is neither of the other two never - +harness_selftest 400-a-check-result-must-be-machine and its verdict is SKIP never - +harness_selftest 400-a-check-result-must-be-machine and leaves one whose variable is assigned earlier never - +harness_selftest 400-a-check-result-must-be-machine and leaves one whose variable is the loop it sits in never - +harness_selftest 400-a-check-result-must-be-machine and names its own cause instead never - +harness_selftest 400-a-check-result-must-be-machine and not a check after a one-line piped loop that already closed never - +harness_selftest 400-a-check-result-must-be-machine and not a piped loop with no check in it never - +harness_selftest 400-a-check-result-must-be-machine and not one inside a process-substitution loop, which keeps its shell never - +harness_selftest 400-a-check-result-must-be-machine and records FAIL never - +harness_selftest 400-a-check-result-must-be-machine and records FAIL, because nothing was compared never - +harness_selftest 400-a-check-result-must-be-machine and records PASS when the ratio is inside the bound never - +harness_selftest 400-a-check-result-must-be-machine and still reports the two numbers never - +harness_selftest 400-a-check-result-must-be-machine and that place is pgc_record never - +harness_selftest 400-a-check-result-must-be-machine and the REASON_CODE travels in the reason field, not in prose never - +harness_selftest 400-a-check-result-must-be-machine and the two numbers are named, not just the verdict never - +harness_selftest 400-a-check-result-must-be-machine and with timing enabled the skipped term is zero, not absent never - +harness_selftest 400-a-check-result-must-be-machine check_num on a non-number emits one record never - +harness_selftest 400-a-check-result-must-be-machine check_num's non-measurement line is unchanged never - +harness_selftest 400-a-check-result-must-be-machine check_ratio on a non-number emits one record never - +harness_selftest 400-a-check-result-must-be-machine check_ratio that forms a ratio emits one record never - +harness_selftest 400-a-check-result-must-be-machine check_ratio with a zero side emits one record never - +harness_selftest 400-a-check-result-must-be-machine check_text on an empty side emits one record never - +harness_selftest 400-a-check-result-must-be-machine check_text's empty-side line is unchanged never - +harness_selftest 400-a-check-result-must-be-machine control: a well-formed record still reconciles never - +harness_selftest 400-a-check-result-must-be-machine fewer records than counted checks is not described as a subshell never - +harness_selftest 400-a-check-result-must-be-machine lib.sh bumps PGC_CHECKS in exactly one place never - +harness_selftest 400-a-check-result-must-be-machine more records than counted checks names the cause, not just the arithmetic never - +harness_selftest 400-a-check-result-must-be-machine no check_skip reads a name its own file never assigns never - +harness_selftest 400-a-check-result-must-be-machine no file that calls check prints a SKIP outcome the count cannot see never - +harness_selftest 400-a-check-result-must-be-machine no suite calls a check inside a piped loop never - +harness_selftest 400-a-check-result-must-be-machine pgc_fail emits one record never - +harness_selftest 400-a-check-result-must-be-machine pgc_pass emits one record never - +harness_selftest 400-a-check-result-must-be-machine premise: and it would name a file that calls check and echoes a SKIP never - +harness_selftest 400-a-check-result-must-be-machine premise: and that is this fragment, not the suite never - +harness_selftest 400-a-check-result-must-be-machine premise: it is callable never - +harness_selftest 400-a-check-result-must-be-machine premise: it names a check_skip reading an unassigned variable never - +harness_selftest 400-a-check-result-must-be-machine premise: lib.sh is where the check helpers live never - +harness_selftest 400-a-check-result-must-be-machine premise: the fixtures carry the shapes this sweep is about never - +harness_selftest 400-a-check-result-must-be-machine premise: the probe ran every helper shape once never - +harness_selftest 400-a-check-result-must-be-machine premise: the sweep classified a corpus of check-calling files never - +harness_selftest 400-a-check-result-must-be-machine premise: while a file that calls no check is not its business never - +harness_selftest 400-a-check-result-must-be-machine the accounting line reconciles four outcomes against the count never - +harness_selftest 400-a-check-result-must-be-machine the reconciliation accepts the verdict FAIL, which pgc_record emits never - +harness_selftest 400-a-check-result-must-be-machine the reconciliation accepts the verdict PASS, which pgc_record emits never - +harness_selftest 400-a-check-result-must-be-machine the reconciliation accepts the verdict SKIP, which pgc_record emits never - +harness_selftest 400-a-check-result-must-be-machine the reconciliation accepts the verdict UNRUN, which pgc_record emits never - +harness_selftest 400-a-check-result-must-be-machine the record count equals the counter the summary reports never - +harness_selftest 400-a-check-result-must-be-machine the record names the part the check was asked from never - +harness_selftest 400-a-check-result-must-be-machine the runner calls the record reconciliation, not merely defines it never - +harness_selftest 400-a-check-result-must-be-machine the runner defines the record reconciliation never - +harness_selftest 400-a-check-result-must-be-machine the same ratio check, ENABLED, emits one record and passes never - +harness_selftest 400-a-check-result-must-be-machine the same timing check, ENABLED, emits one record and passes never - +harness_selftest 400-a-check-result-must-be-machine the sweep finds a check inside a PIPED loop never - +harness_selftest 410-a-check-must-have-been-red --mutation across two failing checks in one run is refused never - +harness_selftest 410-a-check-must-have-been-red a before-log and an after-log together are refused, not silently empty never - +harness_selftest 410-a-check-must-have-been-red a budget that does not exist at the prior is a note, not a refusal never - +harness_selftest 410-a-check-must-have-been-red a check in an UNCOVERED suite is not refused never - +harness_selftest 410-a-check-must-have-been-red a check merely added is not reported as a rename never - +harness_selftest 410-a-check-must-have-been-red a check observed red gains the date it was seen never - +harness_selftest 410-a-check-must-have-been-red a check the ledger has never seen is refused never - +harness_selftest 410-a-check-must-have-been-red a clean status says nothing and does not fail the major never - +harness_selftest 410-a-check-must-have-been-red a date that is not a date is refused rather than stored never - +harness_selftest 410-a-check-must-have-been-red a gate over a nonexistent log is an integrity failure, not a pass never - +harness_selftest 410-a-check-must-have-been-red a later green run does not erase an observation never - +harness_selftest 410-a-check-must-have-been-red a name that appeared while another disappeared is reported as a rename never - +harness_selftest 410-a-check-must-have-been-red a named mutation is recorded against the check that reddened never - +harness_selftest 410-a-check-must-have-been-red a real refusal is a different status from an integrity failure never - +harness_selftest 410-a-check-must-have-been-red a ref that does not resolve is an integrity failure, not a bootstrap never - +harness_selftest 410-a-check-must-have-been-red a ref that exists with the budget still compares never - +harness_selftest 410-a-check-must-have-been-red a ref that exists without the budget is still the bootstrap case never - +harness_selftest 410-a-check-must-have-been-red a refusal keeps the regenerate-the-ledger wording never - +harness_selftest 410-a-check-must-have-been-red a rename in one part survives an addition in another never - +harness_selftest 410-a-check-must-have-been-red a run whose checks are all ledgered passes the gate never - +harness_selftest 410-a-check-must-have-been-red a second mutation ACCUMULATES rather than replacing the first never - +harness_selftest 410-a-check-must-have-been-red a stale prior is named WITH its distance from HEAD never - +harness_selftest 410-a-check-must-have-been-red a verdict pgc_record cannot emit is an integrity failure never - +harness_selftest 410-a-check-must-have-been-red an absolute budget path resolves against git rather than shrugging never - +harness_selftest 410-a-check-must-have-been-red an empty check name is one too, because it names no check never - +harness_selftest 410-a-check-must-have-been-red an empty log is one too, because there is nothing to reconcile never - +harness_selftest 410-a-check-must-have-been-red an integrity failure says regenerating will not help never - +harness_selftest 410-a-check-must-have-been-red an older observation does not overwrite a newer one never - +harness_selftest 410-a-check-must-have-been-red and CI collects from the retained path rather than the deleted one never - +harness_selftest 410-a-check-must-have-been-red and a level prior carries no distance, so zero is silent never - +harness_selftest 410-a-check-must-have-been-red and a log that does not reconcile with its own checks run: never - +harness_selftest 410-a-check-must-have-been-red and a newer one does never - +harness_selftest 410-a-check-must-have-been-red and a raise against that base ref is refused never - +harness_selftest 410-a-check-must-have-been-red and a record missing its verdict never - +harness_selftest 410-a-check-must-have-been-red and an empty mutation is a placeholder, not an empty last field never - +harness_selftest 410-a-check-must-have-been-red and an undated merge does not erase a known date never - +harness_selftest 410-a-check-must-have-been-red and it copies them BEFORE removing the build directory, which is the only order that works never - +harness_selftest 410-a-check-must-have-been-red and it entered as debt, not as an observation nothing made never - +harness_selftest 410-a-check-must-have-been-red and it is a different sentence from the refusal, not the same one twice never - +harness_selftest 410-a-check-must-have-been-red and it is named, so the author knows which one never - +harness_selftest 410-a-check-must-have-been-red and it is that case that says the change introduces the file never - +harness_selftest 410-a-check-must-have-been-red and it is the new one that is named, not the one already ledgered never - +harness_selftest 410-a-check-must-have-been-red and it names the verdict, so the author knows which record never - +harness_selftest 410-a-check-must-have-been-red and it runs before the build directory is removed, which is the only place it can never - +harness_selftest 410-a-check-must-have-been-red and it says the change introduces the file rather than raising anything never - +harness_selftest 410-a-check-must-have-been-red and it says the ref does not resolve, rather than claiming the file is new never - +harness_selftest 410-a-check-must-have-been-red and it says why, rather than falling back to something weaker never - +harness_selftest 410-a-check-must-have-been-red and never says a change introduces a file at a ref that is not there never - +harness_selftest 410-a-check-must-have-been-red and none of them ends in a tab never - +harness_selftest 410-a-check-must-have-been-red and not against one that stayed green never - +harness_selftest 410-a-check-must-have-been-red and one that stayed green keeps its debt never - +harness_selftest 410-a-check-must-have-been-red and only when there is a base, so a push build does not fail on it never - +harness_selftest 410-a-check-must-have-been-red and records neither as ever having been red never - +harness_selftest 410-a-check-must-have-been-red and the addition in the other part is not called a rename never - +harness_selftest 410-a-check-must-have-been-red and the comparison passes when the ceiling did not rise never - +harness_selftest 410-a-check-must-have-been-red and the history it is about to lose travels with it never - +harness_selftest 410-a-check-must-have-been-red and the message says how to fix it, because regenerating is the intended action never - +harness_selftest 410-a-check-must-have-been-red and the refusal names both values never - +harness_selftest 410-a-check-must-have-been-red and the refusal names how many failed, so the author can narrow the run never - +harness_selftest 410-a-check-must-have-been-red and the refusal names the raise never - +harness_selftest 410-a-check-must-have-been-red and the runner names no remote at that call site never - +harness_selftest 410-a-check-must-have-been-red auto refuses when GITHUB_BASE_REF names a ref that is not here never - +harness_selftest 410-a-check-must-have-been-red auto uses the base ref when it resolves, and names it never - +harness_selftest 410-a-check-must-have-been-red auto with no base ref and no upstream is an integrity failure never - +harness_selftest 410-a-check-must-have-been-red both failure arms fail the major never - +harness_selftest 410-a-check-must-have-been-red but that suite is counted as not covered, which is the debt never - +harness_selftest 410-a-check-must-have-been-red control: a well-formed log still merges never - +harness_selftest 410-a-check-must-have-been-red control: the same log merges without --mutation never - +harness_selftest 410-a-check-must-have-been-red each says what was wrong with the input never - +harness_selftest 410-a-check-must-have-been-red every committed row has five fields never - +harness_selftest 410-a-check-must-have-been-red every row has five fields and no trailing tab never - +harness_selftest 410-a-check-must-have-been-red lowering it is allowed, which is the direction the burn-down goes never - +harness_selftest 410-a-check-must-have-been-red merging a green run records both checks never - +harness_selftest 410-a-check-must-have-been-red nor is one merely removed never - +harness_selftest 410-a-check-must-have-been-red once the suite is covered, a new check in it IS refused never - +harness_selftest 410-a-check-must-have-been-red one --mutation cannot be attributed across several runs at once never - +harness_selftest 410-a-check-must-have-been-red premise: and it is a file git has never seen, which is the case that used to fail open never - +harness_selftest 410-a-check-must-have-been-red premise: and the nobudget branch does not, which is the bootstrap shape never - +harness_selftest 410-a-check-must-have-been-red premise: the budget is a tracked file too never - +harness_selftest 410-a-check-must-have-been-red premise: the budget was restored byte-exact never - +harness_selftest 410-a-check-must-have-been-red premise: the check has history before the rename never - +harness_selftest 410-a-check-must-have-been-red premise: the fixture log names checks the real ledger already knows never - +harness_selftest 410-a-check-must-have-been-red premise: the gate's status block was found in the runner never - +harness_selftest 410-a-check-must-have-been-red premise: the hasbudget branch carries the budget never - +harness_selftest 410-a-check-must-have-been-red premise: the ledger is not empty, so the partition means something never - +harness_selftest 410-a-check-must-have-been-red premise: the ledger itself is a tracked file, not a variable never - +harness_selftest 410-a-check-must-have-been-red premise: the ledger tool exists never - +harness_selftest 410-a-check-must-have-been-red premise: the raised copy really does carry a higher ceiling never - +harness_selftest 410-a-check-must-have-been-red premise: the real budget is inside a git repository never - +harness_selftest 410-a-check-must-have-been-red premise: the scratch prior really is three commits behind never - +harness_selftest 410-a-check-must-have-been-red premise: the scratch repo has a committed ceiling and no upstream never - +harness_selftest 410-a-check-must-have-been-red premise: the scratch repo has a prior ceiling committed never - +harness_selftest 410-a-check-must-have-been-red premise: the workflow file is where this part thinks it is never - +harness_selftest 410-a-check-must-have-been-red raising the ceiling above its committed value is refused never - +harness_selftest 410-a-check-must-have-been-red raising the ceiling in the tracked file is refused never - +harness_selftest 410-a-check-must-have-been-red regenerating the ledger lets the new check through never - +harness_selftest 410-a-check-must-have-been-red the budget names a ceiling and a census, and says which is which never - +harness_selftest 410-a-check-must-have-been-red the ceiling refuses being exceeded never - +harness_selftest 410-a-check-must-have-been-red the committed census matches the committed ledger never - +harness_selftest 410-a-check-must-have-been-red the distance travels with a refusal too, not only with a pass never - +harness_selftest 410-a-check-must-have-been-red the gate refuses to run without the registered suite list never - +harness_selftest 410-a-check-must-have-been-red the ledger partitions into observed and never never - +harness_selftest 410-a-check-must-have-been-red the runner asks the tool to resolve the prior rather than naming one never - +harness_selftest 410-a-check-must-have-been-red the runner captures the gate's status rather than only its success never - +harness_selftest 410-a-check-must-have-been-red the runner invokes the ledger gate never - +harness_selftest 410-a-check-must-have-been-red the runner keeps the logs somewhere that outlives the build directory never - +harness_selftest 410-a-check-must-have-been-red the runner passes --against to the gate never - +harness_selftest 410-a-check-must-have-been-red the same check in two logs is two runs, not a duplicate never - +harness_selftest 410-a-check-must-have-been-red the same name twice in ONE log is a duplicate, and is named never - +harness_selftest 410-a-check-must-have-been-red the stable check is not reported never - +harness_selftest 410-a-check-must-have-been-red the suites job fetches the PR base for the ceiling comparison never - +harness_selftest 420-a-deleted-part-must-be-visible and every name in the manifest is a part on disk, so a deletion reddens never - +harness_selftest 420-a-deleted-part-must-be-visible every part on disk is named in the manifest never - +harness_selftest 420-a-deleted-part-must-be-visible premise: a part present in the manifest and missing on disk is named never - +harness_selftest 420-a-deleted-part-must-be-visible premise: and a part added without listing it is named too never - +harness_selftest 420-a-deleted-part-must-be-visible premise: the manifest names something, so the comparison has two sides never - +harness_selftest 420-a-deleted-part-must-be-visible the parts manifest exists, because without it a deletion is invisible never - +harness_selftest 420-a-deleted-part-must-be-visible the three buckets account for every part on disk never - +harness_selftest 430-the-self-test-must-not-report a pg_config whose --bindir is empty is refused too, not run with a broken PATH never - +harness_selftest 430-the-self-test-must-not-report and it does not pretend to have run checks never - +harness_selftest 430-the-self-test-must-not-report and pairs each with the marker the runners require beside the status never - +harness_selftest 430-the-self-test-must-not-report and part 010 exits with exactly that status on its skip paths never - +harness_selftest 430-the-self-test-must-not-report and the refusal is the guard's own, naming the path it could not use never - +harness_selftest 430-the-self-test-must-not-report control: and the second, so a good pg_config is not refused never - +harness_selftest 430-the-self-test-must-not-report control: the pg_config this run was handed satisfies the first predicate never - +harness_selftest 430-the-self-test-must-not-report handed a pg_config that does not exist, the self-test refuses instead of exiting 0 never - +harness_selftest 430-the-self-test-must-not-report no selftest part exits 0, which would exit the driver before its summary never - +harness_selftest 430-the-self-test-must-not-report part 010's skip status is the one lib.sh calls PGC_EXIT_SKIPPED never - +harness_selftest 430-the-self-test-must-not-report premise: and a BARE exit too, which ends with the last status and is usually 0 never - +harness_selftest 430-the-self-test-must-not-report premise: and it answers --bindir with nothing never - +harness_selftest 430-the-self-test-must-not-report premise: and it finds a planted exit 0 outside a heredoc never - +harness_selftest 430-the-self-test-must-not-report premise: the driver this part is about is where it is expected never - +harness_selftest 430-the-self-test-must-not-report premise: the stub is executable, so an -x test alone would accept it never - +harness_selftest 430-the-self-test-must-not-report premise: the sweep reads every selftest part never - +harness_selftest 430-the-self-test-must-not-report premise: while a deliberate non-zero exit is not an offence never - +harness_selftest 430-the-self-test-must-not-report premise: while a fixture script ending in exit 0 inside a heredoc is not an offence never - +harness_selftest 440-a-count-grep-never-produced an empty count is not 'present' under a numeric comparison never - +harness_selftest 440-a-count-grep-never-produced and the numeric form says so on stderr rather than silently never - +harness_selftest 440-a-count-grep-never-produced and the string comparison it replaces WOULD have said present never - +harness_selftest 440-a-count-grep-never-produced no count from grep -c is compared as a string, which answers present when grep could not answer never - +harness_selftest 440-a-count-grep-never-produced premise: a real count compares the same both ways, so the conversion is behaviour-preserving never - +harness_selftest 440-a-count-grep-never-produced premise: and a zero count does too never - +harness_selftest 440-a-count-grep-never-produced premise: and it finds a planted string comparison on a grep -c never - +harness_selftest 440-a-count-grep-never-produced premise: and it finds one whose PATTERN contains a parenthesis, which the first version could not never - +harness_selftest 440-a-count-grep-never-produced premise: and the = 0 spelling too, which fails the same way never - +harness_selftest 440-a-count-grep-never-produced premise: grep -c prints nothing at all on a pattern that does not compile never - +harness_selftest 440-a-count-grep-never-produced premise: nor is one inside a generated fixture script never - +harness_selftest 440-a-count-grep-never-produced premise: the sweep has a corpus to read never - +harness_selftest 440-a-count-grep-never-produced premise: while a numeric comparison is not an offence never - +harness_selftest 440-a-count-grep-never-produced premise: while a valid pattern prints a number never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt new file mode 100644 index 00000000..01a8c04f --- /dev/null +++ b/test/check_ledger_budget.txt @@ -0,0 +1,37 @@ +# The ledger's debt, in a tracked file so a change to it is a diff a reviewer +# sees. An environment variable would not be: PGC_SKIP_TIMING is the precedent, +# set in two workflow files, suppressing whole suites for months with no diff +# ever showing it. +# +# THE TWO NUMBERS ARE DIFFERENT KINDS OF THING. The first version of this file +# treated both as ceilings and deadlocked, so the distinction is written down. +# +# suites_not_covered -- A CEILING, and monotone. +# Registered suites with no rows in the ledger at all. Their checks are +# invisible to the gate: it cannot refuse a new check in a suite it has never +# seen. It falls as suites are seeded, and it MAY ONLY FALL -- the gate compares +# this value against the previously committed one and refuses an increase, so +# widening the debt is an edit a reviewer sees AND a gate refuses. +# +# Adding a check to a suite that is already covered does not move it, which is +# what makes it safe to bound. +suites_not_covered 250 +# +# checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become +# one. +# Every new check enters the ledger as `never`, so bounding this number means +# every added check breaks the gate, and the only way to land one is to raise a +# number the design says may only fall. That is a deadlock rather than a budget, +# and the first version of this file shipped it: 614 rows, 614 never, ceiling +# 614. +# +# It is reported by the gate on every run and falls as checks are attacked. What +# the gate REFUSES is a check the ledger has never seen, which regenerating the +# ledger fixes -- a reviewable one-line diff, and the intended action rather +# than a forbidden one. +# +# The number here is ASSERTED to match the committed ledger, in both harnesses. +# Without that it is a hand-maintained count that drifts, which is the failure +# this repository has spent a day proving. It is not a ceiling; it is a +# measurement that must be true. +checks_never_observed_red 826 diff --git a/test/lib.sh b/test/lib.sh index 0edd36df..16947cb0 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -28,7 +28,17 @@ # failure and continue. PGC_FAIL=0 +# The suite's own name, resolved ONCE at load rather than per check: pgc_record +# runs at every one of 3,762 call sites, and a basename fork at each of them is +# 3,762 forks a suite does not need. +PGC_SUITE="$(basename "$0" .sh)" PGC_CHECKS=0 +# A FOURTH OUTCOME. A check deliberately not asked -- a wall-clock measurement on +# a shared runner under PGC_SKIP_TIMING -- is not a pass, not a failure, and not +# unrunnable. It is counted, so `checks run:` reports the checks a suite +# ENCOUNTERED rather than the ones it managed to evaluate, and pgc_summary +# reconciles four counters against that count instead of three. +PGC_SKIPPED=0 # The status pgc_summary uses for "ran no checks". # @@ -969,17 +979,113 @@ psql_file() { # only supported way to add a check from outside this file, and selftest part 320 # sweeps for direct PGC_CHECKS writes so the next expect_fail is caught when it # is written rather than when it reddens something. -pgc_pass() { # pgc_pass NAME +# ---- one place that counts a check, and it is the same place that records it - +# +# lib.sh had ELEVEN sites bumping PGC_CHECKS, each with its own outcome line +# beside it. That is eleven chances to add a twelfth and forget the line, which +# is exactly what projections.sh's expect_fail did with ten call sites for as +# long as it existed. +# +# Counting and recording are therefore ONE operation. A helper cannot report an +# outcome without being counted, and cannot be counted without reporting one, +# because there is no code path that does either alone. `checks run: N` and the +# N record lines are the same increment seen twice. +# +# DISPLAY is passed whole rather than composed here, so every existing human line +# stays byte-identical: suites, selftests and CI all grep `^PASS` and `^FAIL`, +# and 3,762 call sites is far past what a careful refactor can be trusted on. +# +# The record is tab separated, five columns after the RESULT marker: +# +# RESULT suite part name verdict reason +# +# so a check name containing spaces survives. The reason carries the REASON_CODE +# #915 introduced, which is what makes this more than a reformat: an unrunnable +# check is distinguishable from a passing one without parsing prose. The verdict +# is one of PASS, FAIL, UNRUN or SKIP. +# +# There is no mutation column here. That one belongs to the LEDGER (#918), which +# keys on (suite, part, name) and records which mutation reddened a check; a +# record is one observation, not a history. +pgc_record() { # pgc_record VERDICT NAME DISPLAY [REASON] + local _v="$1" _name="$2" _display="$3" _reason="${4:-}" PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_PASSED=$((PGC_PASSED + 1)) - echo "PASS $1" + case "$_v" in + PASS) PGC_PASSED=$((PGC_PASSED + 1)) ;; + FAIL) PGC_FAILED=$((PGC_FAILED + 1)); PGC_FAIL=1 ;; + UNRUN) PGC_UNRUN=$((PGC_UNRUN + 1)) ;; + SKIP) PGC_SKIPPED=$((PGC_SKIPPED + 1)) ;; + *) + # An unknown verdict is a failure of the harness, not a check to + # drop. Dropping it would leave PGC_CHECKS bumped with no outcome + # recorded, which is the reconciliation failure pgc_summary refuses. + PGC_FAILED=$((PGC_FAILED + 1)); PGC_FAIL=1 + _display="FAIL $_name: pgc_record was given the verdict [$_v], which is not PASS, FAIL, UNRUN or SKIP" + _v=FAIL + ;; + esac + # WHICH PART asked this question, derived from the call stack. + # + # The suite is not enough. harness_selftest sources 40-odd parts into one + # shell, and its premises are phrased to be COPIED -- "premise: the pytest + # layer is where THIS PART thinks it is" says "this part" precisely so the + # same sentence works in any of them. main carries two copies of that one and + # two of another, and the number grows with every part anyone adds. + # + # So a key of (suite, name) is not a key of checks, it is a key of check + # NAMES, and they differ by however many parts share a boilerplate premise. + # One of them going red would then mark every sharer as observed red -- a + # claim about a check nothing attacked. Found by OffgridwithJD, who noticed + # that all six of their own branches added more. + # + # BASH_SOURCE, not a convention change, so the next part written the same way + # is keyed correctly without anyone remembering. Parameter expansion only: no + # basename fork, at 3,762 call sites. + local _part="" _bs + for _bs in "${BASH_SOURCE[@]}"; do + case "$_bs" in */lib.sh|lib.sh) continue ;; esac + _part="${_bs##*/}"; _part="${_part%.sh}" + break + done + + printf '%s\n' "$_display" + # Tabs in a field would split it, and a NEWLINE splits the whole record just + # as completely -- it ends the line, so what follows becomes a second line the + # reader cannot key. Nothing in the tree puts either in a check name, and this + # makes that true rather than assumed. The tab was stripped here from the + # first version; the newline was not, which @linuxhikerpm named on #917: the + # reason already written for the tab is the reason for both. Measured before + # the fix, a newline in the name gave a record of 4 fields plus two stray + # lines; after it, 6 fields and one line. + # + # PARAMETER EXPANSION, not `printf | tr` in a command substitution. The first + # version paid four forks per record -- two subshells and two tr processes -- + # in the function that runs at every one of 3,762 check sites, and whose own + # comment hoists PGC_SUITE out of the body on exactly that ground. Measured on + # an idle box, 2,000 calls, identical output on every input including a real + # tab: 3.1577 ms per call against 0.0096 ms, 331x, or 11.9 seconds of pure + # fork overhead across a full suite against 36 ms. Reported by OffgridwithJD. + local _nl_name="${_name//$'\t'/ }" _nl_reason="${_reason//$'\t'/ }" + _nl_name="${_nl_name//$'\n'/ }"; _nl_reason="${_nl_reason//$'\n'/ }" + _nl_name="${_nl_name//$'\r'/ }"; _nl_reason="${_nl_reason//$'\r'/ }" + printf 'RESULT\t%s\t%s\t%s\t%s\t%s\n' \ + "${PGC_SUITE:-unknown}" \ + "${_part:-${PGC_SUITE:-unknown}}" \ + "${_nl_name}" \ + "$_v" \ + "${_nl_reason}" +} + +pgc_pass() { # pgc_pass NAME + pgc_record PASS "$1" "PASS $1" } pgc_fail() { # pgc_fail NAME DETAIL - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 - if [ -n "${2:-}" ]; then echo "FAIL $1: $2"; else echo "FAIL $1"; fi + if [ -n "${2:-}" ]; then + pgc_record FAIL "$1" "FAIL $1: $2" + else + pgc_record FAIL "$1" "FAIL $1" + fi } # A check that could not be evaluated is a third state, not a pass. @@ -998,30 +1104,23 @@ pgc_fail() { # pgc_fail NAME DETAIL # FAILED, because the failure is the more urgent fact. check_unrunnable() { # check_unrunnable NAME REASON_CODE DETAIL local name="$1" reason="${2:-}" detail="${3:-}" - PGC_CHECKS=$((PGC_CHECKS + 1)) case " $PGC_UNRUN_REASONS " in *" $reason "*) ;; *) - echo "FAIL $name: unrunnable reason [$reason] is not one of: $PGC_UNRUN_REASONS" - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) + pgc_record FAIL "$name" \ + "FAIL $name: unrunnable reason [$reason] is not one of: $PGC_UNRUN_REASONS" return ;; esac - PGC_UNRUN=$((PGC_UNRUN + 1)) - echo "UNRUN $name: $reason: $detail" + pgc_record UNRUN "$name" "UNRUN $name: $reason: $detail" "$reason" } check() { local name="$1" got="$2" want="$3" - PGC_CHECKS=$((PGC_CHECKS + 1)) if [ "$got" = "$want" ]; then - PGC_PASSED=$((PGC_PASSED + 1)) - echo "PASS $name" + pgc_record PASS "$name" "PASS $name" else - echo "FAIL $name: got [$got] want [$want]" - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) + pgc_record FAIL "$name" "FAIL $name: got [$got] want [$want]" fi } @@ -1071,11 +1170,8 @@ pgc_is_number() { # $1 -> 0 when $1 is a number check_text() { local name="$1" got="$2" want="$3" if [ -z "$got" ] || [ -z "$want" ]; then - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) - echo "FAIL $name: a side is empty, so nothing was compared:" \ - "got [$got] want [$want]" + pgc_record FAIL "$name" \ + "FAIL $name: a side is empty, so nothing was compared: got [$got] want [$want]" return 1 fi check "$name" "$got" "$want" @@ -1085,11 +1181,8 @@ check_text() { check_num() { local name="$1" got="$2" want="$3" if ! pgc_is_number "$got" || ! pgc_is_number "$want"; then - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) - echo "FAIL $name: not a measurement, so nothing was compared:" \ - "got [$got] want [$want]" + pgc_record FAIL "$name" \ + "FAIL $name: not a measurement, so nothing was compared: got [$got] want [$want]" return 1 fi check "$name" "$got" "$want" @@ -1121,30 +1214,20 @@ check_ratio() { # $1 label, $2 a, $3 b, $4 max local name="$1" a="$2" b="$3" max="$4" ratio if ! pgc_is_number "$a" || ! pgc_is_number "$b" || ! pgc_is_number "$max"; then - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) - echo "FAIL $name: not a measurement, so no ratio was formed:" \ - "a=[$a] b=[$b] max=[$max]" + pgc_record FAIL "$name" \ + "FAIL $name: not a measurement, so no ratio was formed: a=[$a] b=[$b] max=[$max]" return 1 fi if [ "$(awk -v x="$a" -v y="$b" 'BEGIN { print (x + 0 == 0 || y + 0 == 0) ? "yes" : "no" }')" = yes ]; then - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) - echo "FAIL $name: a side of the ratio is zero, so nothing was measured:" \ - "a=[$a] b=[$b]" + pgc_record FAIL "$name" \ + "FAIL $name: a side of the ratio is zero, so nothing was measured: a=[$a] b=[$b]" return 1 fi ratio="$(awk -v a="$a" -v b="$b" 'BEGIN { printf "%.2f", a / b }')" - PGC_CHECKS=$((PGC_CHECKS + 1)) if [ "$(awk -v r="$ratio" -v m="$max" 'BEGIN { print (r <= m) ? "yes" : "no" }')" = yes ]; then - PGC_PASSED=$((PGC_PASSED + 1)) - echo "PASS $name (${ratio}x, bound ${max}x, from a=$a b=$b)" + pgc_record PASS "$name" "PASS $name (${ratio}x, bound ${max}x, from a=$a b=$b)" else - echo "FAIL $name: ${ratio}x exceeds the ${max}x bound (a=$a b=$b)" - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) + pgc_record FAIL "$name" "FAIL $name: ${ratio}x exceeds the ${max}x bound (a=$a b=$b)" fi } @@ -1157,10 +1240,8 @@ pgc_require_tools() { command -v "$t" >/dev/null 2>&1 || missing="$missing $t" done if [ -n "$missing" ]; then - echo "FAIL the tools this suite measures with are missing:$missing" - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) + pgc_record FAIL "the tools this suite measures with are missing" \ + "FAIL the tools this suite measures with are missing:$missing" return 1 fi return 0 @@ -1181,11 +1262,36 @@ pgc_require_tools() { # So the ratio is skipped and the rest of the suite runs. A skip is announced # rather than silent, and it is not counted as a pass, because a count that # includes checks nobody ran is the thing this project keeps having to unlearn. +# A named check that could not run HERE, for a reason the suite knows. +# +# WHY THIS EXISTS. `echo "SKIP ..."` printed a line a reader sees and left +# PGC_CHECKS alone, so the outcome existed for a human and for nobody else: no +# record, no count, and nothing for `pgc_reconcile_records` to reconcile. The +# tree's own comment at native_index_projection.sh said why that mattered -- "the +# skip must be visible: a check that reports nothing is indistinguishable from a +# check that passes" -- and that was TRUE while the human line WAS the record. It +# stopped being true when the RESULT stream became the machine-readable one, and +# 22 sites were left behind on the wrong side of the change. Named by +# @linuxhikerpm on #917; the owner asked for every site, not the three examples. +# +# DISPLAY IS PASSED WHOLE, exactly as pgc_record takes it, so every existing +# human line stays byte-identical. These messages are individually worded and a +# reader greps them; recomposing them here would change what people see for no +# gain, which is the same reason pgc_record does not compose PASS lines either. +check_skip() { # check_skip NAME DISPLAY [REASON] + pgc_record SKIP "$1" "$2" "${3:-}" +} + check_timing() { local name="$1" got="$2" want="$3" if [ "${PGC_SKIP_TIMING:-0}" = 1 ]; then - echo "SKIP $name (PGC_SKIP_TIMING: wall-clock measurement)" + # Counted and recorded, like every other outcome. It printed a line a + # reader sees; leaving PGC_CHECKS at zero made that outcome invisible to + # the count and to the records both (#917, found by @linuxhikerpm). + pgc_record SKIP "$name" \ + "SKIP $name (PGC_SKIP_TIMING: wall-clock measurement)" \ + "PGC_SKIP_TIMING" return 0 fi check "$name" "$got" "$want" @@ -1227,7 +1333,9 @@ check_timing() { # same-run ratio) and nothing said it here, where it is decided (#787). check_ratio_needs_quiet_machine() { # if [ "${PGC_SKIP_TIMING:-0}" = 1 ]; then - echo "SKIP $1 (PGC_SKIP_TIMING: wall-clock ratio)" + pgc_record SKIP "$1" \ + "SKIP $1 (PGC_SKIP_TIMING: wall-clock ratio)" \ + "PGC_SKIP_TIMING" return 0 fi check_ratio "$@" @@ -1476,13 +1584,15 @@ pgc_skip() { # pgc_skip cap="$(printf '%s' "$1" | tr '[:lower:]-' '[:upper:]_')" allow_one="PGC_ALLOW_MISSING_$cap" if [ "${PGC_ALLOW_MISSING:-0}" = 1 ] || [ "${!allow_one:-0}" = 1 ]; then - echo "SKIP $2 (waived by $allow_one or PGC_ALLOW_MISSING)" + # The unwaived branch below records a FAIL. This one printed and left + # PGC_CHECKS at zero, so waiving a dependency also erased the outcome -- + # the same asymmetry check_timing had, in the function whose whole + # subject is "a missing dependency is not a pass". + check_skip "$2" "SKIP $2 (waived by $allow_one or PGC_ALLOW_MISSING)" \ + "waived by $allow_one or PGC_ALLOW_MISSING" pgc_summary fi - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - PGC_FAILED=$((PGC_FAILED + 1)) - echo "FAIL $2" + pgc_record FAIL "$2" "FAIL $2" echo " A missing dependency is an environment defect, not a pass. Install" echo " it, or set $allow_one=1 to run knowingly without this coverage." pgc_summary @@ -1507,7 +1617,7 @@ pgc_skip() { # pgc_skip # did one level up for how many VERSIONS actually ran. pgc_summary() { local _failed=$PGC_FAILED - local _sum=$((PGC_PASSED + PGC_FAILED + PGC_UNRUN)) + local _sum=$((PGC_PASSED + PGC_FAILED + PGC_UNRUN + PGC_SKIPPED)) echo echo "checks run: $PGC_CHECKS" echo "checks unrunnable: $PGC_UNRUN" @@ -1515,7 +1625,7 @@ pgc_summary() { # go missing, and this harness has 3,762 check sites -- far past what anyone # notices by reading. If this line does not add up the harness is lying about # its own arithmetic, so it is a failure rather than a note. - echo "accounting: $PGC_PASSED passed + $_failed failed + $PGC_UNRUN unrunnable = $PGC_CHECKS" + echo "accounting: $PGC_PASSED passed + $_failed failed + $PGC_UNRUN unrunnable + $PGC_SKIPPED skipped = $PGC_CHECKS" # A MEASUREMENT, not an identity. The failed count is its own counter rather # than CHECKS - PASSED - UNRUN, because a derived third term makes # P + (N-P-U) + U = N true for ANY values: a helper that counts a check and @@ -1525,9 +1635,9 @@ pgc_summary() { # line could not see it. Three counters maintained independently, reconciled # against a fourth, is the only version of it that can fail. if [ "$_sum" != "$PGC_CHECKS" ]; then - echo "FAIL the summary does not reconcile: $PGC_PASSED passed + $PGC_FAILED failed + $PGC_UNRUN unrunnable = $_sum, but $PGC_CHECKS checks ran" + echo "FAIL the summary does not reconcile: $PGC_PASSED passed + $PGC_FAILED failed + $PGC_UNRUN unrunnable + $PGC_SKIPPED skipped = $_sum, but $PGC_CHECKS checks ran" echo " A check was counted whose outcome nothing recorded. Find the helper" - echo " that bumps PGC_CHECKS without touching PGC_PASSED, PGC_FAILED or PGC_UNRUN." + echo " that bumps PGC_CHECKS without touching PGC_PASSED, PGC_FAILED, PGC_UNRUN or PGC_SKIPPED." PGC_FAIL=1 fi if [ "$PGC_FAIL" != "0" ]; then @@ -1563,7 +1673,11 @@ pgc_summary() { fi exit 1 fi - if [ "$PGC_CHECKS" = "0" ]; then + # EVALUATED nothing, not ENCOUNTERED nothing. Before the fourth counter a + # skipped check left PGC_CHECKS at zero, so this branch caught the all-skipped + # suite by accident; now it is counted, and the suite would report PASSED with + # nothing behind it. The condition is what it always meant. + if [ "$((PGC_PASSED + PGC_FAILED + PGC_UNRUN))" = "0" ]; then echo "$(basename "$0"): SKIPPED (ran no checks)" exit $PGC_EXIT_SKIPPED fi diff --git a/test/logical_subscriber.sh b/test/logical_subscriber.sh index 045f93c9..40b35088 100755 --- a/test/logical_subscriber.sh +++ b/test/logical_subscriber.sh @@ -50,7 +50,7 @@ pick_port() { } SUB_PORT="$(pick_port)" if [ "$SUB_PORT" = 0 ]; then - echo "SKIP no free port for the subscriber cluster" + check_skip "the logical subscriber round trip" "SKIP no free port for the subscriber cluster" "no free port for the subscriber cluster" pgc_summary; exit 0 fi diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 6d39ecec..9cae1f8e 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -263,7 +263,7 @@ if [ "$(q "SELECT 1 FROM pg_collation WHERE collname = 'ci'")" = "1" ]; then "$(q "SELECT count(*) FROM (SELECT k FROM t_ci GROUP BY k) s")" \ "$(q "SELECT count(DISTINCT lower(k)) FROM t_ci")" else - echo "SKIP non-deterministic collation (ICU unavailable)" + check_skip "the non-deterministic collation case" "SKIP non-deterministic collation (ICU unavailable)" "ICU unavailable" fi # an output expression built on a group key (not a bare key) -> falls back diff --git a/test/native_groupagg_batch.sh b/test/native_groupagg_batch.sh index bc4642e8..dfa2ebe9 100755 --- a/test/native_groupagg_batch.sh +++ b/test/native_groupagg_batch.sh @@ -93,8 +93,7 @@ agree_in() { # agree_in TABLE LABEL "SELECT ... FROM %T ..." # md5 of empty input is a fixed string, so require the columnar arm produced # rows at all before trusting the comparison. if [ -z "$(q "${tmpl//%T/$tbl}" | head -1)" ]; then - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)); PGC_FAIL=1 - echo "FAIL $label: the columnar arm returned no rows, so nothing was compared" + pgc_fail "$label" "the columnar arm returned no rows, so nothing was compared" return 1 fi check_text "$label" "$col" "$heap" @@ -177,7 +176,7 @@ for pair in "text key:$Q_TEXTKEY" "expression key:$Q_EXPRKEY" \ else # The node itself declined the shape; there is no fold line to read and # nothing for this suite to gate. Say so rather than assert a missing line. - echo "SKIP $label: the grouped node is not planned for this shape" + check_skip "$label" "SKIP $label: the grouped node is not planned for this shape" "the grouped node is not planned for this shape" fi agree "$label: answers match the heap mirror" "$tmpl" done diff --git a/test/native_index_projection.sh b/test/native_index_projection.sh index 77a16640..4fd1a79e 100755 --- a/test/native_index_projection.sh +++ b/test/native_index_projection.sh @@ -185,16 +185,33 @@ check "parallel-built index returns each row once" \ # amcheck where available. The skip must be visible: a check that reports nothing is # indistinguishable from a check that passes, which is what this file is about. +# ONE LIST, BOTH BRANCHES. The names the else branch declines have to be the +# names the then branch checks, or two logs from two boxes differ by ten checks +# and nobody reading them can say why. It also stops the two drifting when an +# index is added to one branch and not the other. +_ixs="w_k w_k12 w_c18 w_expr w_part w_len w_par w_par2 w_pare w_parp w_ser" if psql_run "CREATE EXTENSION IF NOT EXISTS amcheck;" >/dev/null 2>&1 && [ "$(q "SELECT count(*) FROM pg_proc WHERE proname='bt_index_check'")" != "0" ]; then - for ix in w_k w_k12 w_c18 w_expr w_part w_len w_par w_par2 w_pare w_parp w_ser; do + for ix in $_ixs; do out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -c "SELECT bt_index_check('$ix'::regclass)" 2>&1) check "bt_index_check($ix)" \ "$(grep -qE 'ERROR' <<<"$out" && echo bad || echo ok)" "ok" done else - echo "SKIP amcheck is not installed on this build; the seq-scan oracle above still ran" + # ELEVEN LINES, NOT ONE. The first version printed the old single summary + # line and recorded eleven, which selftest 400 refused and was right to: a + # line reading `SKIP ...` that no record backs is exactly the shape this + # branch of the change exists to remove, and a reader cannot tell it from a + # recorded one. The then branch prints eleven PASS lines; this prints eleven + # SKIP lines, and the reason travels on each. + echo " amcheck is not installed on this build; the seq-scan oracle above still ran" + for ix in $_ixs; do + check_skip "bt_index_check($ix)" \ + "SKIP bt_index_check($ix): amcheck is not installed on this build" \ + "amcheck is not installed on this build" + done fi +unset _ixs pgc_summary diff --git a/test/native_parquet_flba.sh b/test/native_parquet_flba.sh index 2db4d09b..3e5719d3 100755 --- a/test/native_parquet_flba.sh +++ b/test/native_parquet_flba.sh @@ -208,7 +208,7 @@ if phys != {"i32": "INT32", "i64": "INT64"}: sys.exit("unexpected physical types: %s" % phys) PYINT if [ $? -ne 0 ]; then - echo "SKIP this pyarrow does not store decimals as integers as expected" + check_skip "the integer-backed decimal case" "SKIP this pyarrow does not store decimals as integers as expected" "this pyarrow does not store decimals as integers" else check "INT32-backed DECIMAL reads" \ "$(q "SELECT string_agg(d::text, ',' ORDER BY d) FROM pgcolumnar.read_parquet('$W/dec_i32.parquet') AS t(d numeric);")" \ @@ -231,7 +231,7 @@ PYINT "-3500000,0,1250000" fi else - echo "SKIP pyarrow not available; foreign-producer FLBA cases skipped" + check_skip "the foreign-producer FLBA cases" "SKIP pyarrow not available; foreign-producer FLBA cases skipped" "pyarrow not available" fi pgc_summary diff --git a/test/native_parquet_multifile.sh b/test/native_parquet_multifile.sh index 038918c6..4bad5fb2 100755 --- a/test/native_parquet_multifile.sh +++ b/test/native_parquet_multifile.sh @@ -172,7 +172,7 @@ if mkfifo "$DIR/pipe.parquet" 2>/dev/null; then check "FIFO named *.parquet is skipped, does not block" "$fifo_out" "3000" rm -f "$DIR/pipe.parquet" else - echo "SKIP mkfifo unavailable; FIFO case not exercised" + check_skip "the FIFO case" "SKIP mkfifo unavailable; FIFO case not exercised" "mkfifo unavailable" fi # ---- recursive walk -------------------------------------------------------- diff --git a/test/native_parquet_pushdown.sh b/test/native_parquet_pushdown.sh index 72e7403f..7ab4ab15 100755 --- a/test/native_parquet_pushdown.sh +++ b/test/native_parquet_pushdown.sh @@ -211,7 +211,7 @@ if f.metadata.num_row_groups != 4: sys.exit("expected 4 row groups") PYDEC if [ $? -ne 0 ]; then - echo "SKIP could not build the integer-DECIMAL pushdown file" + check_skip "the integer-DECIMAL pushdown case" "SKIP could not build the integer-DECIMAL pushdown file" "could not build the fixture file" else psql_run "CREATE FOREIGN TABLE ftdec (d numeric) SERVER pq OPTIONS (path '$PGC_WORKDIR/dec_push.parquet');" @@ -225,7 +225,7 @@ PYDEC "$(skipped_for_t ftdec 'd >= 0')" "0" fi else - echo "SKIP pyarrow not available; integer-DECIMAL pushdown case skipped" + check_skip "the integer-DECIMAL pushdown case" "SKIP pyarrow not available; integer-DECIMAL pushdown case skipped" "pyarrow not available" fi pgc_summary diff --git a/test/native_parquet_schema.sh b/test/native_parquet_schema.sh index d49ede6e..d552273c 100755 --- a/test/native_parquet_schema.sh +++ b/test/native_parquet_schema.sh @@ -133,7 +133,7 @@ PY "$(q "SELECT count(*) FILTER (WHERE field_id IS NULL) || '/' || count(*) FROM pgcolumnar.parquet_schema('$REQ');")" \ "2/2" else - echo "SKIP pyarrow not available; REQUIRED-column and field-id checks skipped" + check_skip "the REQUIRED-column and field-id checks" "SKIP pyarrow not available; REQUIRED-column and field-id checks skipped" "pyarrow not available" fi pgc_summary diff --git a/test/native_repack.sh b/test/native_repack.sh index b8ef2bd0..7ea82479 100755 --- a/test/native_repack.sh +++ b/test/native_repack.sh @@ -54,13 +54,11 @@ srv="$(q 'SHOW server_version_num')" # on an older major depends on it being 0. Asserting the premise must not destroy # the skip it guards. if ! pgc_is_number "$srv"; then - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 - echo "FAIL the server did not answer 'SHOW server_version_num': got [$srv]" + pgc_fail "the server did not answer 'SHOW server_version_num'" "got [$srv]" pgc_summary fi if [ "$srv" -lt 190000 ]; then - echo "SKIP REPACK requires PostgreSQL 19 (server_version_num=$srv)" + check_skip "REPACK" "SKIP REPACK requires PostgreSQL 19 (server_version_num=$srv)" "requires PostgreSQL 19" pgc_summary fi diff --git a/test/objstore_module.sh b/test/objstore_module.sh index 7fb139ac..4f03b08b 100755 --- a/test/objstore_module.sh +++ b/test/objstore_module.sh @@ -70,10 +70,8 @@ for stash in "$MOD.away" "$MOD.probe"; do # every check below would run against a broken installation and report the # confusing half of the truth, so stop here and say which file to look at. if ! stash_is_debris; then - echo "FAIL restored $stash to $MOD, but that is not a module either." - echo " This installation needs 'make install' before the suite can run." - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 + pgc_fail "restored $stash to $MOD, but that is not a module either" \ + "this installation needs 'make install' before the suite can run" pgc_summary exit 1 fi @@ -174,7 +172,7 @@ objstore_teardown() { restore_module; pgc_teardown; } trap objstore_teardown EXIT INT TERM if ! mv "$MOD" "$MOD.probe" 2>/dev/null; then - echo "SKIP cannot move $MOD, so the absent and broken paths are untested here" + check_skip "the absent and broken module paths" "SKIP cannot move $MOD, so the absent and broken paths are untested here" "cannot move $MOD" pgc_summary exit 0 fi diff --git a/test/objstore_stash_recovery.sh b/test/objstore_stash_recovery.sh index e0ef1906..1def6b88 100755 --- a/test/objstore_stash_recovery.sh +++ b/test/objstore_stash_recovery.sh @@ -52,13 +52,13 @@ echo "PG_CONFIG=$PG_CONFIG" if [ -z "${PGC_SKIP_BUILD:-}" ]; then echo "-- building" make -C "$SRCDIR" PG_CONFIG="$PG_CONFIG" >/dev/null || { - echo "FAIL build failed, so nothing below measures the guard" - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)); PGC_FAIL=1; pgc_summary + pgc_fail "build failed, so nothing below measures the guard" + pgc_summary } echo "-- installing" make -C "$SRCDIR" install PG_CONFIG="$PG_CONFIG" >/dev/null || { - echo "FAIL install failed, so nothing below measures the guard" - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)); PGC_FAIL=1; pgc_summary + pgc_fail "install failed, so nothing below measures the guard" + pgc_summary } fi @@ -89,7 +89,7 @@ cleanup() { trap cleanup EXIT INT TERM if ! mod_is_valid "$MOD"; then - echo "SKIP no valid module installed at $MOD, so there is no state to arrange" + check_skip "stash recovery" "SKIP no valid module installed at $MOD, so there is no state to arrange" "no valid module installed at $MOD" pgc_summary fi SAFE="$(mktemp /tmp/pgc-objstore-safe.XXXXXX)" diff --git a/test/parquet_export_stats.sh b/test/parquet_export_stats.sh index db83952b..f650d188 100755 --- a/test/parquet_export_stats.sh +++ b/test/parquet_export_stats.sh @@ -88,8 +88,7 @@ psql_run "SELECT pgcolumnar.export_parquet('es_c', '$PARQ');" if ! python3 "$STATS_PY" "$PARQ" > "$S" 2>"$PGC_WORKDIR/stats.err"; then echo "FAIL the footer parser could not read the exported file:" sed 's/^/ /' "$PGC_WORKDIR/stats.err" - PGC_FAIL=1 - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) + pgc_fail "the statistics export failed" "see the indented output above" pgc_summary fi @@ -229,10 +228,7 @@ check_num "every bound is its physical width" \ check_float() { # check_float NAME GOT WANT local name="$1" got="$2" want="$3" if ! pgc_is_number "$got" || ! pgc_is_number "$want"; then - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 - echo "FAIL $name: not a measurement, so nothing was compared:" \ - "got [$got] want [$want]" + pgc_fail "$name" "not a measurement, so nothing was compared: got [$got] want [$want]" return 1 fi check "$name" \ @@ -370,9 +366,8 @@ if python3 "$STATS_PY" "$NANQ" > "$NS" 2>&1; then check "a zero minimum is written as -0.0" \ "$(nf "$N_Z9" minhex)" "0000000000000080" else - echo "FAIL the footer parser could not read the NaN fixture:" + pgc_fail "the footer parser could not read the NaN fixture" "see the indented output below" sed 's/^/ /' "$NS" - PGC_FAIL=1; PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) fi # ---- (f) column_orders, without which the bounds have no defined meaning ---- diff --git a/test/parquet_nested_import.sh b/test/parquet_nested_import.sh index 3be75c41..6ab49426 100755 --- a/test/parquet_nested_import.sh +++ b/test/parquet_nested_import.sh @@ -86,7 +86,7 @@ then else rc=$? if [ "$rc" = 3 ]; then - echo "SKIP: pyarrow not available for the reference-writer case" + check_skip "the reference-writer case" "SKIP: pyarrow not available for the reference-writer case" "pyarrow not available" else echo "FAIL: pyarrow nested file generation errored (rc=$rc)" PGC_FAIL=1 diff --git a/test/pg19_vacuum_options.sh b/test/pg19_vacuum_options.sh index 6cafd0aa..1e281f25 100755 --- a/test/pg19_vacuum_options.sh +++ b/test/pg19_vacuum_options.sh @@ -40,13 +40,11 @@ srv="$(q 'SHOW server_version_num')" # on an older major depends on it being 0. Asserting the premise must not destroy # the skip it guards. if ! pgc_is_number "$srv"; then - PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAILED=$((PGC_FAILED + 1)) - PGC_FAIL=1 - echo "FAIL the server did not answer 'SHOW server_version_num': got [$srv]" + pgc_fail "the server did not answer 'SHOW server_version_num'" "got [$srv]" pgc_summary fi if [ "$srv" -lt 190000 ]; then - echo "SKIP parallel autovacuum requires PostgreSQL 19 (server_version_num=$srv)" + check_skip "parallel autovacuum" "SKIP parallel autovacuum requires PostgreSQL 19 (server_version_num=$srv)" "requires PostgreSQL 19" pgc_summary fi diff --git a/test/pgc_ledger.py b/test/pgc_ledger.py new file mode 100755 index 00000000..949dedc1 --- /dev/null +++ b/test/pgc_ledger.py @@ -0,0 +1,622 @@ +#!/usr/bin/env python3 +"""The mutation ledger: which checks have ever been seen red, and under what. + +Nothing recorded whether a check had ever been red. That is the gap that let 39 +checks across 35 suites ship unable to fail, three of them inside the suite whose +whole purpose is to stop exactly that. The gate answered "did anything print FAIL" +and had never answered "could anything print FAIL". + +WHAT THIS RECORDS, AND WHAT IT DOES NOT +--------------------------------------- +It records that a named check WAS OBSERVED RED in a recorded run. It does NOT +claim the check is proven able to fail: that needs a named mutation applied +deliberately, and conflating the two would put a claim in the ledger that nothing +measured -- the `defeated: 0` shape from VACUITY_MODES section 1. + +THE TWO NUMBERS ARE DIFFERENT KINDS OF THING, and the first design got this wrong +in a way that deadlocked. +------------------------------------------------------------------------------- +`checks_never_observed_red` is a CENSUS. It cannot be a ceiling: every new check +enters the ledger as `never`, so bounding it means every added check breaks the +gate, and the only way to land one is to raise a number the design says may only +fall. That is a deadlock, not a budget. It is reported, and it falls as checks are +attacked. + +`suites_not_covered` IS a ceiling, and a monotone one, because adding a check to a +covered suite does not move it. It falls as suites are seeded, and it may only +fall: the gate compares the working value against the previously committed one and +refuses an increase, so widening the debt is an edit a reviewer sees AND a gate +refuses, rather than either alone. + +WHAT THE GATE REFUSES +--------------------- +A check the committed ledger has never seen. That is the allowlist the issue asks +for -- existing checks are grandfathered, a new one is named and refused until the +ledger is regenerated, which is a reviewable one-line diff and the INTENDED action +rather than a forbidden one. + +WHAT FEEDS IT +------------- +`run_all_versions.sh` merges every suite's log before it removes the build +directory, so every matrix run feeds a ledger -- locally and in CI. The committed +ledger is updated deliberately, by running `merge` against a real run and +committing the diff. CI verifies; humans commit. A ledger that CI rewrote by +itself would be a file nobody reads changing under everybody. + +It is not only mutation runs. Every real CI red fills it, every flake, every +bisect. A mutation run is the deliberate accelerator. + +FAIL CLOSED +----------- +An unreadable file, an empty one, or a record with too few fields is an ERROR. +Silently skipping them made every integrity failure indistinguishable from a clean +run: a gate over a nonexistent log returned success. + +FORMAT +------ +Tab separated, five columns, keyed on the first three: + + suite part check name last observed red mutations + +`last observed red` is a date or `never`. `mutations` is `-`, or a `;`-separated +SET of the mutations that have reddened this check -- accumulated, not overwritten, +because a column that keeps only the last one records the most recent attack +rather than the catalogue it exists to become. +""" + +import argparse +import os +import pathlib +import re +import subprocess +import sys + +NEVER = "never" +UNKNOWN = "unknown" + +# ISO 8601 date, and nothing else. A free-form string was accepted verbatim, so a +# typo became an observation date the ledger then treated as authoritative -- +# measured: `--date not-a-date` stored `not-a-date`. +_DATE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") + + +def _newer(current, incoming): + """The later of two observations, with `unknown` and `never` below every date. + + LAST RED MAY ONLY MOVE FORWARD. It was a plain assignment, so merging an older + log rewrote a recent observation with an older one, and merging an undated log + replaced a real date with `unknown` -- both measured on #918, both silently. + A ledger whose whole subject is "when was this last seen red" cannot let the + answer regress because of the order somebody merged in. + """ + rank = {NEVER: 0, UNKNOWN: 1} + if rank.get(incoming, 2) < rank.get(current, 2): + return current + if rank.get(incoming, 2) > rank.get(current, 2): + return incoming + return max(current, incoming) if incoming not in rank else current +NONE = "-" +FIELDS = 5 + + +class LedgerError(Exception): + """An integrity failure. Never silently skipped.""" + + +# The verdicts pgc_record can emit, and the only ones a record may carry. Kept +# here as the reader's own list rather than derived from lib.sh: a python tool +# reading a shell file to learn its vocabulary is the coupling CONTEXT.md +# forbids, and the drift is caught by the arm that plants each verdict instead. +VERDICTS = ("PASS", "FAIL", "UNRUN", "SKIP") + +# RESULT plus suite, part, name, verdict, reason. +RECORD_FIELDS = 6 + + +def read_records(paths, *, require_nonempty=True): + """[(suite, part, name, verdict)] for every RESULT line in the given logs. + + Fails closed, and VALIDATES rather than merely counting. `len(f) < 5` accepted + a record missing its reason, a verdict outside the emitter's vocabulary, an + empty check name, and one record against `checks run: 2` -- all measured + returning 0 while the ledger absorbed them as evidence. Evidence that does not + parse is not evidence, and the ledger's whole subject is which checks have + been observed red: a malformed log is how an observation gets attributed to a + check that never ran. Reported by @linuxhikerpm on #918. + + The reconciliation against `checks run:` is here as well as in the runner + because the two answer different questions. The runner asks whether the suite + it just ran was internally consistent; this asks whether a log handed to the + ledger, possibly from another machine or another day, can be trusted at all. + """ + out = [] + for p in paths: + path = pathlib.Path(p) + try: + text = path.read_text(errors="replace") + except OSError as e: + raise LedgerError(f"cannot read {p}: {e}") from e + found = 0 + stated = None + for n, line in enumerate(text.splitlines(), 1): + m = re.match(r"^checks run: ([0-9]+)$", line) + if m: + stated = int(m.group(1)) + continue + if not line.startswith("RESULT\t"): + continue + f = line.split("\t") + if len(f) != RECORD_FIELDS: + raise LedgerError( + f"{p}:{n}: a record has {RECORD_FIELDS - 1} fields -- suite, part, " + f"name, verdict, reason; got {len(f) - 1}") + if not f[1] or not f[2] or not f[3]: + raise LedgerError( + f"{p}:{n}: a record with an empty suite, part or name names no check") + if f[4] not in VERDICTS: + raise LedgerError( + f"{p}:{n}: verdict {f[4]!r} is not one of {', '.join(VERDICTS)}, " + f"so this log was not written by pgc_record") + out.append((f[1], f[2], f[3], f[4])) + found += 1 + if require_nonempty and found == 0: + raise LedgerError(f"{p}: no RESULT records, so there is nothing to reconcile") + if found and stated is None: + raise LedgerError( + f"{p}: {found} record(s) but no `checks run:` line, so the log never " + f"reached its summary and cannot be reconciled") + if stated is not None and found != stated: + raise LedgerError( + f"{p}: {found} record(s) against `checks run: {stated}` -- the log does " + f"not reconcile with itself, so it is not evidence about either number") + return out + + +def read_ledger(path): + """{(suite, part, name): [last_red, {mutations}]}. + + Keyed on the part as well as the name: harness_selftest sources 40-odd parts + into one shell and phrases its premises to be COPIED, so a name-only key is a + key of check NAMES rather than of checks. + """ + rows = {} + p = pathlib.Path(path) + if not p.exists(): + return rows + for n, line in enumerate(p.read_text(errors="replace").splitlines(), 1): + if not line.strip() or line.startswith("#"): + continue + f = line.split("\t") + if len(f) != FIELDS: + raise LedgerError(f"{path}:{n}: a ledger row needs {FIELDS} fields, got {len(f)}") + muts = set() if f[4] == NONE else {m for m in f[4].split(";") if m} + rows[(f[0], f[1], f[2])] = [f[3] or NEVER, muts] + return rows + + +def write_ledger(path, rows): + lines = [] + for (suite, part, name), (red, muts) in sorted(rows.items()): + # No trailing tab. An empty last field is trailing whitespace on every + # row, which `git diff --check` reports and which made 614 of them. + lines.append("\t".join((suite, part, name, red, + ";".join(sorted(muts)) if muts else NONE))) + pathlib.Path(path).write_text("\n".join(lines) + ("\n" if lines else "")) + + +def _by_run(paths): + """[(path, {(suite, part, name): [verdicts]})] -- one entry per LOG. + + Per log, because the same check appearing in two logs is two RUNS of it, while + twice in one log is a duplicate name sharing a ledger row. Merging the logs + first cannot tell those apart, and reported the first as the second. + """ + runs = [] + for p in paths: + seen = {} + for suite, part, name, verdict in read_records([p]): + seen.setdefault((suite, part, name), []).append(verdict) + runs.append((p, seen)) + return runs + + +def cmd_census(args): + for suite, part, name, verdict in read_records(args.logs): + print(f"{suite}\t{part}\t{name}\t{verdict}") + return 0 + + +def cmd_merge(args): + rows = read_ledger(args.ledger) + runs = _by_run(args.logs) + + if args.date != UNKNOWN and not _DATE.match(args.date): + raise LedgerError( + f"--date {args.date!r} is not an ISO date (YYYY-MM-DD). A ledger row's " + f"last-red is compared against other dates, so a free-form string is not " + f"an observation") + + if args.mutation and len(runs) > 1: + raise LedgerError( + "--mutation names one deliberate change, so it cannot be attributed across " + f"{len(runs)} logs at once: merge them one run at a time") + + # A MUTATION NAMES ONE CHECK, and a run that mutates one thing can redden + # several: the target, plus whatever depended on it. Attributing the mutation + # to every failure records collateral damage as evidence that the mutation + # kills that check, which is the opposite of what this column is for. + # Measured on #918: a two-FAIL log merged with --mutation MUTATION_A recorded + # it against both. Reported by @linuxhikerpm. + if args.mutation: + failed = sorted({key for _p, seen in runs for key, vs in seen.items() if "FAIL" in vs}) + if len(failed) > 1: + listed = "\n".join(f" {s}\t{p}\t{n}" for s, p, n in failed[:6]) + more = "" if len(failed) <= 6 else f"\n ... and {len(failed) - 6} more" + raise LedgerError( + f"--mutation names one check, but {len(failed)} checks failed in this " + f"run:\n{listed}{more}\n Attributing it to all of them would record " + f"collateral damage as evidence. Merge without --mutation, or narrow the " + f"run to the check the mutation targets.") + + for path, seen in runs: + for key, verdicts in sorted(seen.items()): + if key not in rows: + # A check this ledger has never seen enters as DEBT. A green run + # has observed nothing go red, so merging one must never record a + # red observation. + rows[key] = [NEVER, set()] + if "FAIL" in verdicts: + rows[key][0] = _newer(rows[key][0], args.date) + if args.mutation: + # A SET. Keeping only the last one records the most recent + # attack rather than the catalogue this column exists to + # become. + rows[key][1].add(args.mutation) + for key, verdicts in sorted(seen.items()): + if len(verdicts) > 1: + print(f" duplicate check name in one run, so one ledger row covers " + f"{len(verdicts)}: {key[0]}\t{key[1]}\t{key[2]}") + + write_ledger(args.ledger, rows) + seen_all = {k for _, s in runs for k in s} + red = sum(1 for v in rows.values() if v[0] != NEVER) + print(f" ledger: rows={len(rows)} | runs={len(runs)}, distinct checks this merge={len(seen_all)}, " + f"observed red ever={red}, never={len(rows) - red}") + return 0 + + +def cmd_rename_scan(args): + """A name that appeared while another disappeared, WITHIN ONE PART, is a rename. + + Keyed by the display string, a rename loses history and reads exactly like a + brand-new check that has never been red -- the one state this ledger exists to + distinguish. Detected and named rather than silently reset. + + Grouped by (suite, part) before pairing. A global positional zip misses a real + rename whenever unrelated movement in another part shifts the ordering. + + Scanned against ONE run. Given a before-log and an after-log together, the + vanished name is present in the union and nothing appears to have gone. + """ + runs = _by_run(args.logs) + if len(runs) > 1: + raise LedgerError( + f"rename-scan compares ONE run against the ledger, but got {len(runs)} logs: " + "the union of a before-log and an after-log hides the disappearance") + rows = read_ledger(args.ledger) + now = set(runs[0][1]) + + parts = {(s, p) for s, p, _ in now} + known = {k for k in rows if (k[0], k[1]) in parts} + + rc = 0 + n_app = n_van = 0 + for part in sorted(parts): + app = sorted(k[2] for k in now - known if (k[0], k[1]) == part) + van = sorted(k[2] for k in known - now if (k[0], k[1]) == part) + n_app += len(app) + n_van += len(van) + for new, old in zip(app, van): + was = rows.get((part[0], part[1], old), [NEVER, set()])[0] + print(f" possible rename: {old} -> {new} " + f"(in {part[0]}/{part[1]}, history: last red {was})") + rc = 1 + print(f" rename scan: appeared={n_app}, vanished={n_van}") + return rc + + +def read_budget(path): + out = {} + text = pathlib.Path(path).read_text() + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + out[parts[0]] = int(parts[1]) + return out + + +def _resolve_against(path, spec): + """Which ref carries the prior ceiling. Fails closed rather than guessing. + + NOT a hardcoded remote name. `origin` is per-clone: in a contributor's setup it + is their fork, and OffgridwithJD measured theirs 446 commits behind upstream. + Comparing against a stale main makes this check WEAKER, never falsely red -- + the ceiling may only fall, so an older main carries a higher one, and a raise + passes whenever the stale prior is high enough. It fails open while printing a + line that reads like the enforcement happened, which is the same shape as the + absolute-path bug one level down: compared against the wrong thing, rather than + could not compare. + + So: + + GITHUB_BASE_REF in CI this names the PR's target branch, which IS the prior + by definition. Its remote-tracking ref must exist -- if the + checkout did not fetch it, that is an error, not a fallback. + main@{upstream} outside CI, ask git rather than a convention. The configured + upstream of the local main is the answer to "which main is + mine", per clone. + + Anything else is an error. A gate that quietly enforces less than it claims is + the thing this whole change exists to refuse, and a fallback that says so is + still a gate enforcing less. + """ + if spec != "auto": + return spec + repo = pathlib.Path(path).resolve().parent + + def _rev(ref): + r = subprocess.run(["git", "-C", str(repo), "rev-parse", "--verify", "-q", ref], + capture_output=True, text=True) + return ref if r.returncode == 0 else None + + base = os.environ.get("GITHUB_BASE_REF", "").strip() + if base: + for cand in (f"refs/remotes/origin/{base}", base): + if _rev(cand): + return cand + raise LedgerError( + f"GITHUB_BASE_REF is {base!r} but no ref for it resolves here, so the prior " + "ceiling cannot be read. The checkout needs to fetch the base branch") + + r = subprocess.run(["git", "-C", str(repo), "rev-parse", "--abbrev-ref", "main@{upstream}"], + capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip(): + return r.stdout.strip() + raise LedgerError( + "no trustworthy prior ceiling: GITHUB_BASE_REF is unset and the local main has no " + "configured upstream. Naming a remote would compare against whatever `origin` " + "happens to be in this clone, which is how a fork 446 commits stale gets treated " + "as the prior") + + +def _behind(path, ref): + """" (N commits behind HEAD)" when the prior lags, "" when it does not. + + A reader can see WHICH ref was compared against. Without this they cannot see + that it is 446 commits stale, which is the difference between knowing the + comparison happened and knowing what it was worth. + """ + repo = pathlib.Path(path).resolve().parent + r = subprocess.run(["git", "-C", str(repo), "rev-list", "--count", f"{ref}..HEAD"], + capture_output=True, text=True) + if r.returncode != 0 or not r.stdout.strip().isdigit(): + return " (distance from HEAD unknown)" + n = int(r.stdout.strip()) + return "" if n == 0 else f" ({n} commit{'s' if n != 1 else ''} behind HEAD)" + + +def _committed_budget(path, ref): + """The budget as of `ref`, or None when the file does not exist there. + + None means "this change introduces the file", which is not a raise. Every + OTHER failure -- an unresolvable ref, a path outside its repository, a budget + naming no ceiling -- raises, because asked to compare and unable is a different + thing from nothing to compare. + + `git show REF:PATH` needs a REPO-RELATIVE path. The production caller passes an + absolute one inside a copied build directory, so the first version returned + None and printed "no prior ceiling to compare" -- a message that reads like a + pass while the ceiling it was asked to enforce went unchecked. That is the + fail-open shape this whole change is about, in the code that closes it. + Reported by OffgridwithJD. + + So the path is resolved here rather than demanded of the caller, and every + failure to resolve it is an ERROR. Asked to compare, unable to compare, is not + the same as nothing to compare. + """ + abspath = pathlib.Path(path).resolve() + try: + top = subprocess.run(["git", "-C", str(abspath.parent), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=True).stdout.strip() + except (subprocess.CalledProcessError, OSError) as e: + raise LedgerError( + f"--against {ref} was given, but {path} is not inside a git repository, " + "so the prior ceiling cannot be read") from e + try: + rel = abspath.relative_to(pathlib.Path(top).resolve()) + except ValueError as e: + raise LedgerError(f"{path} resolves outside its own repository at {top}") from e + + # DOES THE REF EXIST, asked before anything is read from it. This check used to + # live inside the `auto` resolver, so it covered the production call site and + # nothing else: an EXPLICIT ref that did not resolve fell through to the + # file-absent branch and was reported as the bootstrap case -- rc=0, with a + # message asserting "this change introduces it" about a ref that does not + # exist, one clause after saying the distance from HEAD was unknown. The code + # knew it could not resolve the ref and contradicted itself in one sentence. + # + # Resolving belongs here, where every caller passes through, so that the + # file-absent branch below describes only what it claims: a file missing at a + # ref that IS there. Reported by OffgridwithJD, who scoped it precisely -- + # unreachable from the runner, which always passes `auto`, and a trap for the + # harness arms and anyone driving the tool by hand. + if subprocess.run(["git", "-C", top, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"], + capture_output=True, text=True).returncode != 0: + raise LedgerError( + f"--against {ref} was given, but that ref does not resolve here, so the prior " + "ceiling cannot be read") + # A FILE THAT DOES NOT EXIST AT THE PRIOR HAS NO CEILING TO VIOLATE. This + # returns None and the caller notes it, rather than erroring, and the + # distinction is the whole of it: introducing the budget is not raising it. + # + # The gate caught its own bootstrap the first time it ran in CI -- #925's base + # is #923's branch, where check_ledger_budget.txt does not exist because this + # PR adds it, so `auto` resolved the base correctly, found no prior, failed + # closed, and reddened the matrix. Correct behaviour for a rule that could not + # be satisfied: a PR introducing the file could never pass its own gate. + # + # It is not a hole. Deleting the budget on a branch and re-adding it with a + # higher ceiling does not reach here, because the file still exists at the + # prior and the comparison happens. Only a genuinely new file gets the note, + # and a genuinely new budget file is reviewed as a new file. + # + # An unresolvable REF stays an error above, and a malformed budget stays one + # below. Asked to compare and unable is different from nothing to compare. + try: + blob = subprocess.run(["git", "-C", top, "show", f"{ref}:{rel.as_posix()}"], + capture_output=True, text=True, check=True).stdout + except (subprocess.CalledProcessError, OSError): + return None + out = {} + for line in blob.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + f = line.split() + if len(f) == 2 and f[1].isdigit(): + out[f[0]] = int(f[1]) + if "suites_not_covered" not in out: + raise LedgerError(f"the budget at {ref} names no suites_not_covered") + return out + + +def cmd_gate(args): + rows = read_ledger(args.ledger) + budget = read_budget(args.budget) + seen = {k for k, _, _ in ((k, None, None) for k in + {(s, p, n) for s, p, n, _ in read_records(args.logs)})} + + rc = 0 + + # THE REFUSAL: a check the committed ledger has never seen, IN A SUITE THE + # LEDGER COVERS. + # + # The suite restriction is not a softening, it is the meaning of + # suites_not_covered: the gate cannot refuse a new check in a suite it has + # never seen, because it has no idea which of that suite's checks are new. + # Without it the gate refuses every check of all 250 uncovered suites and + # reddens the whole matrix on the first run -- which is a gate somebody turns + # off, the failure mode this issue family exists to prevent. + # + # It tightens on its own as suites are seeded, and the ceiling is what forces + # that direction. + covered_suites = {k[0] for k in rows} + unknown = sorted(k for k in seen - set(rows) if k[0] in covered_suites) + for suite, part, name in unknown: + print(f" not in the ledger: {suite}\t{part}\t{name}") + if unknown: + print(f" {len(unknown)} check(s) the ledger has never seen. Regenerate it with:") + print(f" python3 test/pgc_ledger.py merge --ledger {args.ledger} --date ") + rc = 1 + + # A CENSUS, not a ceiling. Bounding it deadlocks: every new check enters as + # `never`, so the only way to land one would be to raise a number the design + # says may only fall. + never = sum(1 for v in rows.values() if v[0] == NEVER) + print(f" ledger census: rows={len(rows)} | never observed red={never}, " + f"ever red={len(rows) - never}, new this run={len(unknown)}") + + if not args.registered: + raise LedgerError( + "--registered is required: without the registered suite list the coverage " + "claim cannot be made, and skipping it silently is how a gate reports success " + "for a question it never asked") + registered = {w for w in pathlib.Path(args.registered).read_text().split() if w} + if not registered: + raise LedgerError(f"{args.registered}: no registered suites listed") + uncovered = sorted(registered - {k[0] for k in rows}) + want = budget.get("suites_not_covered") + print(f" ledger coverage: registered={len(registered)} | covered={len(registered) - len(uncovered)}, " + f"not covered={len(uncovered)}, ceiling={want if want is not None else 'unset'}") + if want is None: + print(" the budget names no suites_not_covered, so nothing bounds the coverage debt") + return 1 + if len(uncovered) > want: + print(f" suites_not_covered: {len(uncovered)} exceeds the ceiling of {want}") + rc = 1 + + # MONOTONE, mechanically. The tracked file says the ceiling may only fall; + # without this that sentence is prose and raising the number passes. + if args.against: + ref = _resolve_against(args.budget, args.against) + # HOW FAR BEHIND THE PRIOR IS, printed beside it. + # + # `main@{upstream}` is the per-clone answer to "which main is mine", and + # in a contributor's setup it resolves to their FORK -- OffgridwithJD's is + # 446 commits behind upstream. Naming the ref told a reader WHICH prior + # was used; it did not tell them what the comparison was worth. The + # direction still fails open: an older main carries a higher ceiling, so a + # raise passes whenever the stale prior is high enough. + # + # There is no better ref to pick that does not guess, so the weakness is + # made visible instead. It costs nothing when the number is 0. + shown = f"{ref}{_behind(args.budget, ref)}" + prior = _committed_budget(args.budget, ref) + if prior is None: + print(f" no budget at {shown}: this change introduces it, so there is no " + f"prior ceiling it could have raised") + return rc + p_want = prior["suites_not_covered"] + if want > p_want: + print(f" suites_not_covered was raised from {p_want} to {want} " + f"(against {shown}): the ceiling may only fall") + rc = 1 + else: + print(f" ceiling against {shown}: {p_want} -> {want}, which does not rise") + return rc + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + + c = sub.add_parser("census", help="print suite/part/name/verdict for each record") + c.add_argument("logs", nargs="+") + c.set_defaults(fn=cmd_census) + + m = sub.add_parser("merge", help="merge a run's records into the ledger") + m.add_argument("--ledger", required=True) + m.add_argument("--date", default="unknown") + m.add_argument("--mutation", default="") + m.add_argument("logs", nargs="+") + m.set_defaults(fn=cmd_merge) + + r = sub.add_parser("rename-scan", help="report names that look renamed") + r.add_argument("--ledger", required=True) + r.add_argument("logs", nargs="+") + r.set_defaults(fn=cmd_rename_scan) + + g = sub.add_parser("gate", help="refuse a check the ledger has never seen") + g.add_argument("--ledger", required=True) + g.add_argument("--budget", required=True) + g.add_argument("--registered", default="", + help="file listing every registered suite (required)") + g.add_argument("--against", default="", + help="'auto' to resolve the prior from GITHUB_BASE_REF or main@{upstream}, " + "or an explicit git ref. Fails closed when no trustworthy prior exists") + g.add_argument("logs", nargs="+") + g.set_defaults(fn=cmd_gate) + + args = ap.parse_args(argv) + try: + return args.fn(args) + except LedgerError as e: + print(f" ledger integrity failure: {e}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 55dd7aeb..b044b049 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -68,6 +68,7 @@ behaviour, the source of that number is named. - [20. test_raises_sqlstate.py: which error, and which statement](#20-test_raises_sqlstatepy-which-error-and-which-statement) - [21. test_failed_query_sentinel.py: a failed query is not a comparison](#21-test_failed_query_sentinelpy-a-failed-query-is-not-a-comparison) - [22. test_writes_wrote_rows.py: a write that wrote nothing](#22-test_writes_wrote_rowspy-a-write-that-wrote-nothing) +- [23. test_mutation_ledger.py: which checks have ever been red](#23-test_mutation_ledgerpy-which-checks-have-ever-been-red) ## 1. How to read a test in here @@ -1866,3 +1867,111 @@ each pattern to one `E` line, and pytest wraps a long traceback line. Matching like "the guard did not fire". The refusal now leads with the mode's own kebab-case id, which is one token and cannot be split, and each arm matches one token per call. + +## 23. test_mutation_ledger.py: which checks have ever been red + +Nothing recorded whether a check had ever been red. That is the gap that let **39 checks +across 35 suites** ship unable to fail, three of them inside the suite whose whole +purpose is to stop exactly that. + +It records that a named check **was observed red in a recorded run**. Not that it is +proven able to fail: that needs a named mutation applied deliberately, and conflating +the two would put a claim in the ledger nothing measured. + +### The first design deadlocked, and the fix is the distinction + +Bounding `checks_never_observed_red` means **every added check breaks the gate**, because +a new check enters as `never` — so the only way to land one was to raise a number the +design said may only fall. It shipped at 614 rows, 614 `never`, ceiling 614. + +| number | kind | why | +| --- | --- | --- | +| `suites_not_covered` | **ceiling**, monotone | adding a check to a covered suite does not move it | +| `checks_never_observed_red` | **census**, asserted | every new check enters as `never`, so bounding it deadlocks | + +What the gate refuses is a check the committed ledger has never seen, **in a suite the +ledger covers**. Regenerating the ledger is the intended fix and a reviewable diff. + +The format is five tab-separated columns keyed on the first three: +`suite`, `part`, `check name`, `last observed red`, `mutations` — the last a `-` or a +`;`-separated **set**, accumulated rather than overwritten. + +`run_all_versions.sh` invokes the gate before it removes the build directory, which is +the only place a matrix run can reach every suite's log. + +### `test_bad_input_is_an_integrity_failure_not_a_clean_run` + +A nonexistent log, an empty one and a record missing its verdict all returned **rc=0**. +An integrity failure that reads as a clean run is worse than no gate, because it +certifies. They now return 2, distinguishable from a real refusal at 1, and +`--registered` is required rather than silently skipped. + +### `test_a_green_run_records_debt_and_never_a_red_observation` + +A green run has observed nothing go red, so merging one must never record a red +observation — otherwise an ordinary CI run retires the debt the ledger exists to count. + +### `test_the_mutation_column_accumulates_rather_than_overwriting` + +Last-write-wins records the most recent attack rather than the catalogue the column +exists to become. One `--mutation` copied across several logs attributes a deliberate +change to failures it had nothing to do with, and is refused. + +### `test_a_log_that_does_not_parse_is_not_evidence` + +`read_records` accepted `len(f) >= 5`, so a record missing its reason, a verdict +outside `pgc_record`'s vocabulary, an empty check name, one record against +`checks run: 2`, and a log with no count at all all merged at rc=0. The ledger +absorbed as evidence a log that does not parse, which is how an observation gets +attributed to a check that never ran. Five refusals and a control, because five +arms all reporting rc=2 prove nothing if the tool has started refusing everything. + +### `test_last_red_may_only_move_forward` + +The date was a plain assignment, so the answer depended on merge order: an older +log rewrote a recent observation, and an undated merge replaced a real date with +`unknown`. A free-form `--date` was stored verbatim, so a typo became an +observation date the ledger treated as authoritative. + +### `test_a_mutation_names_one_check_not_every_casualty` + +One deliberate change can redden the target and whatever depended on it. +Attributing `--mutation` to every failure records collateral damage as evidence +that the mutation kills that check. A run with more than one failing check is +refused with the count, and a single failure still carries the mutation on the +check that reddened. + +### `test_two_runs_of_a_check_are_not_a_duplicate_of_it` + +Merging logs first cannot tell *the same check in two runs* from *the same name twice in +one run*, and reported the first as the second. + +### `test_renames_are_grouped_by_part_and_scanned_against_one_run` + +A global positional pairing misses a real rename whenever unrelated movement in another +part shifts the ordering. Given a before-log and an after-log together the vanished name +is present in the union, so the scan **refuses** rather than silently finding nothing. + +### `test_the_gate_refuses_a_new_check_only_in_a_suite_it_covers` + +The suite restriction is the *meaning* of `suites_not_covered`, not a softening: without +it the gate refuses every check of all 250 uncovered suites and reddens the whole matrix +on its first run. It tightens on its own as suites are seeded, and the deadlock that +shipped is pinned as its own arm — regenerating the ledger lets a new check through. + +### `test_the_ceiling_may_only_fall_and_that_is_enforced` + +The tracked file says the ceiling may only fall. Without a mechanism that is prose, and +raising the number passed. The gate compares against the previously committed value. + +### `test_the_runner_invokes_the_gate_before_it_removes_the_logs` + +A gate nothing runs is a comment. Nothing in the repository called this tool: zero +references in `.github/`, zero in the runner. + +### `test_the_committed_ledger_and_budget_agree` + +If they disagree, one was edited by hand. `suites_not_covered` is 250 of 251, so the +gate cannot refuse a new check in 250 suites — a real limit, counted rather than hidden, +which falls as suites are seeded. + diff --git a/test/pytest/test_harness_deps.py b/test/pytest/test_harness_deps.py index da263f08..110aa9b8 100644 --- a/test/pytest/test_harness_deps.py +++ b/test/pytest/test_harness_deps.py @@ -81,6 +81,13 @@ # declaration has to say so. The third time this arm has caught a merge-order # consequence rather than a mistake. "test_failed_query_sentinel.py", + # This branch's file, named by the same arm the moment #921's classifier + # arrived on the base. It drives test/pgc_ledger.py, which is a python tool + # rather than the shell harness, so it needs neither a cluster nor psycopg: + # measured at 9 passed in a venv with no driver, and driver_dependent() + # agrees. The fourth time this arm has caught a merge-order consequence + # rather than a mistake, which is the argument for it. + "test_mutation_ledger.py", ] diff --git a/test/pytest/test_mutation_ledger.py b/test/pytest/test_mutation_ledger.py new file mode 100644 index 00000000..3ba7ff5c --- /dev/null +++ b/test/pytest/test_mutation_ledger.py @@ -0,0 +1,391 @@ +"""The mutation ledger: which checks have ever been seen red, and under what. + +Nothing recorded whether a check had ever been red. That is the gap that let 39 +checks across 35 suites ship unable to fail, three of them inside the suite whose +whole purpose is to stop exactly that. + +**What this records.** That a named check *was observed red in a recorded run*. Not +that it is proven able to fail: that needs a named mutation applied deliberately, and +conflating them would put a claim in the ledger nothing measured. + +**The first design deadlocked.** Bounding `checks_never_observed_red` means every +added check breaks the gate, because a new check enters as `never` -- so the only way +to land one was to raise a number the design said may only fall. It shipped at 614 +rows, 614 never, ceiling 614. It is now a CENSUS, asserted to match the ledger. The +CEILING is `suites_not_covered`, which adding a check does not move, and which the +gate refuses to see raised. + +**What the gate refuses** is a check the committed ledger has never seen, in a suite +the ledger covers. Regenerating the ledger is the intended fix and a reviewable diff. + +These tests drive the real tool, for the same reason the other files drive the real +shell: a Python twin of a Python tool would agree with itself. +""" + +import pathlib +import subprocess + +REPO = pathlib.Path(__file__).resolve().parents[2] +TOOL = REPO / "test" / "pgc_ledger.py" +RUNNER = REPO / "test" / "run_all_versions.sh" + +GREEN = ("RESULT\tdemo\tpart1\tfirst check\tPASS\t\n" + "RESULT\tdemo\tpart1\tsecond check\tPASS\t\nchecks run: 2\n") +RED = ("RESULT\tdemo\tpart1\tfirst check\tFAIL\t\n" + "RESULT\tdemo\tpart1\tsecond check\tPASS\t\nchecks run: 2\n") + + +def _run(*args, cwd=None): + r = subprocess.run(["python3", str(TOOL), *args], + capture_output=True, text=True, cwd=cwd) + return r.stdout + r.stderr, r.returncode + + +def _w(tmp_path, name, text): + p = tmp_path / name + p.write_text(text) + return str(p) + + +def _rows(path): + return [l.split("\t") for l in pathlib.Path(path).read_text().splitlines() if l] + + +def test_bad_input_is_an_integrity_failure_not_a_clean_run(tmp_path, expect): + """Every one of these returned rc=0 before. + + `read_records` ignored unreadable files, empty ones and short records, so a gate + over a NONEXISTENT log reported success. An integrity failure that reads as a + clean run is worse than no gate, because it certifies. Reported by @linuxhikerpm. + """ + ledger = _w(tmp_path, "l.tsv", "") + budget = _w(tmp_path, "b.txt", "suites_not_covered 0\n") + reg = _w(tmp_path, "reg", "demo\n") + empty = _w(tmp_path, "empty.log", "") + short = _w(tmp_path, "short.log", "RESULT\tdemo\tpart1\tname\n") + + for label, log in (("nonexistent", str(tmp_path / "nope.log")), + ("empty", empty), ("malformed", short)): + out, rc = _run("gate", "--ledger", ledger, "--budget", budget, + "--registered", reg, log) + expect.num(rc, 2, f"a {label} log is an integrity failure") + expect.num(out.count("ledger integrity failure"), 1, + f"and the {label} log says what was wrong with it") + + # It must stay distinguishable from a real refusal, or fail-closed just renames + # every outcome to the same thing. + good = _w(tmp_path, "g.log", GREEN) + expect.num(_run("gate", "--ledger", ledger, "--budget", budget, + "--registered", reg, good)[1], 1, + "a real refusal is a different status from an integrity failure") + + # --registered is required: skipping it silently is how a gate reports success + # for a question it never asked. + expect.num(_run("gate", "--ledger", ledger, "--budget", budget, good)[1], 2, + "the gate refuses to run without the registered suite list") + + +def test_a_green_run_records_debt_and_never_a_red_observation(tmp_path, expect): + """A green run has seen nothing go red, so merging one must never record a red + observation -- otherwise an ordinary CI run retires the debt it exists to count.""" + ledger = _w(tmp_path, "l.tsv", "") + _run("merge", "--ledger", ledger, _w(tmp_path, "g.log", GREEN)) + rows = _rows(ledger) + expect.num(len(rows), 2, "merging a green run records both checks") + expect.text(",".join(sorted({r[3] for r in rows})), "never", + "and records neither as ever having been red") + expect.num(len([r for r in rows if len(r) != 5]), 0, "every row has five fields") + expect.num(pathlib.Path(ledger).read_text().count("\t\n"), 0, + "and no row ends in a tab, which was 614 of them") + + +def test_the_mutation_column_accumulates_rather_than_overwriting(tmp_path, expect): + """Last-write-wins records the most recent attack rather than the catalogue the + column exists to become, which defeats its purpose rather than limiting it. + + And one `--mutation` copied across several logs attributes a deliberate change to + failures it had nothing to do with. Both reported by @linuxhikerpm. + """ + ledger = _w(tmp_path, "l.tsv", "") + red = _w(tmp_path, "r.log", RED) + _run("merge", "--ledger", ledger, "--date", "2026-09-10", "--mutation", "SAOP 128 -> 0", red) + by = {r[2]: r[4] for r in _rows(ledger)} + expect.text(by["first check"], "SAOP 128 -> 0", + "a named mutation is recorded against the check that reddened") + expect.text(by["second check"], "-", "and not against one that stayed green") + + _run("merge", "--ledger", ledger, "--date", "2026-09-10", "--mutation", "bloom neutered", red) + expect.text({r[2]: r[4] for r in _rows(ledger)}["first check"], + "SAOP 128 -> 0;bloom neutered", + "a second mutation accumulates rather than replacing the first") + + expect.num(_run("merge", "--ledger", ledger, "--date", "2026-09-10", "--mutation", "X", + red, _w(tmp_path, "g.log", GREEN))[1], 2, + "one mutation cannot be attributed across several runs at once") + + +def test_two_runs_of_a_check_are_not_a_duplicate_of_it(tmp_path, expect): + """Merging the logs first cannot tell "the same check in two runs" from "the same + name twice in one run", and reported the first as the second.""" + ledger = _w(tmp_path, "l.tsv", "") + g = _w(tmp_path, "g.log", GREEN) + out, _ = _run("merge", "--ledger", ledger, "--date", "2026-09-10", g, g) + expect.num(out.count("duplicate"), 0, + "the same check in two logs is two runs, not a duplicate") + + twice = _w(tmp_path, "twice.log", + "RESULT\tdemo\tpart1\tsame\tPASS\t\n" + "RESULT\tdemo\tpart1\tsame\tFAIL\t\nchecks run: 2\n") + out, _ = _run("merge", "--ledger", _w(tmp_path, "l2.tsv", ""), "--date", "2026-09-10", twice) + expect.num(out.count("duplicate check name in one run, so one ledger row covers 2: " + "demo\tpart1\tsame"), 1, + "the same name twice in ONE log is a duplicate, and is named") + + +def test_renames_are_grouped_by_part_and_scanned_against_one_run(tmp_path, expect): + """A global positional pairing misses a real rename whenever unrelated movement in + another part shifts the ordering. + + And given a before-log and an after-log together, the vanished name is present in + the union and nothing appears to have gone -- a scan that silently finds nothing is + worse than one that refuses. + """ + ledger = _w(tmp_path, "l.tsv", "") + before = _w(tmp_path, "b.log", + "RESULT\tdemo\tpartA\told A\tFAIL\t\n" + "RESULT\tdemo\tpartB\tstable B\tPASS\t\nchecks run: 2\n") + after = _w(tmp_path, "a.log", + "RESULT\tdemo\tpartA\tnew A\tPASS\t\n" + "RESULT\tdemo\tpartB\tstable B\tPASS\t\n" + "RESULT\tdemo\tpartB\tadded B\tPASS\t\nchecks run: 3\n") + _run("merge", "--ledger", ledger, "--date", "2026-09-01", before) + + out, rc = _run("rename-scan", "--ledger", ledger, after) + expect.num(out.count("possible rename: old A -> new A"), 1, + "a rename in one part survives an addition in another") + expect.num(out.count("last red 2026-09-01"), 1, + "and the history it is about to lose travels with it") + expect.num(out.count("added B"), 0, + "the addition in the other part is not called a rename") + expect.num(rc, 1, "and a detected rename is reported as a nonzero status") + + expect.num(_run("rename-scan", "--ledger", ledger, before, after)[1], 2, + "a before-log and an after-log together are refused, not silently empty") + + +def test_the_gate_refuses_a_new_check_only_in_a_suite_it_covers(tmp_path, expect): + """The suite restriction is the MEANING of `suites_not_covered`, not a softening. + + Without it the gate refuses every check of all 250 uncovered suites and reddens the + whole matrix on its first run -- a gate somebody turns off within the week, which is + the failure this issue family exists to prevent. It tightens on its own as suites + are seeded. + """ + ledger = _w(tmp_path, "l.tsv", "") + reg = _w(tmp_path, "reg", "demo\nother\n") + _run("merge", "--ledger", ledger, "--date", "2026-09-10", _w(tmp_path, "g.log", GREEN)) + + other = _w(tmp_path, "o.log", "RESULT\tother\tpartX\tsomething\tPASS\t\nchecks run: 1\n") + b1 = _w(tmp_path, "b1.txt", "suites_not_covered 1\n") + out, rc = _run("gate", "--ledger", ledger, "--budget", b1, "--registered", reg, other) + expect.num(rc, 0, "a check in an uncovered suite is not refused") + expect.num(out.count("not covered=1"), 1, "but that suite is counted as debt") + + _run("merge", "--ledger", ledger, "--date", "2026-09-10", other) + other2 = _w(tmp_path, "o2.log", + "RESULT\tother\tpartX\tsomething\tPASS\t\n" + "RESULT\tother\tpartX\tnewly added\tPASS\t\nchecks run: 2\n") + b0 = _w(tmp_path, "b0.txt", "suites_not_covered 0\n") + out, rc = _run("gate", "--ledger", ledger, "--budget", b0, "--registered", reg, other2) + expect.num(rc, 1, "once the suite is covered, a new check in it IS refused") + expect.num(out.count("not in the ledger: other\tpartX\tnewly added"), 1, + "and it is the new one that is named") + expect.num(out.count("Regenerate it with"), 1, + "and the message says how to fix it, because that is the intended action") + + # THE DEADLOCK THAT SHIPPED, as its own arm: adding a check must not require an + # edit the design forbids. + _run("merge", "--ledger", ledger, "--date", "2026-09-10", other2) + expect.num(_run("gate", "--ledger", ledger, "--budget", b0, + "--registered", reg, other2)[1], 0, + "regenerating the ledger lets the new check through") + expect.text({r[2]: r[3] for r in _rows(ledger)}["newly added"], "never", + "and it entered as debt, not as an observation nothing made") + + +def test_the_ceiling_may_only_fall_and_that_is_enforced(tmp_path, expect): + """The tracked file says the ceiling may only fall. Without a mechanism that + sentence is prose, and raising the number passed.""" + repo = tmp_path / "repo" + repo.mkdir() + for cmd in (["git", "init", "-q", "."], ["git", "config", "user.email", "t@t"], + ["git", "config", "user.name", "t"]): + subprocess.run(cmd, cwd=repo, capture_output=True) + (repo / "b.txt").write_text("suites_not_covered 5\n") + subprocess.run(["git", "add", "b.txt"], cwd=repo, capture_output=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=repo, capture_output=True) + + prior = subprocess.run(["git", "show", "HEAD:b.txt"], cwd=repo, + capture_output=True, text=True).stdout + expect.num(prior.count("suites_not_covered 5"), 1, + "premise: the scratch repo has a prior ceiling committed") + + ledger = _w(tmp_path, "l.tsv", "") + reg = _w(tmp_path, "reg", "demo\n") + _run("merge", "--ledger", ledger, "--date", "2026-09-10", _w(tmp_path, "g.log", GREEN)) + log = _w(tmp_path, "g2.log", GREEN) + + (repo / "b.txt").write_text("suites_not_covered 9\n") + out, rc = _run("gate", "--ledger", ledger, "--budget", "b.txt", + "--registered", reg, "--against", "HEAD", log, cwd=repo) + expect.num(rc, 1, "raising the ceiling above its committed value is refused") + expect.num(out.count("was raised from 5 to 9"), 1, "and the refusal names both values") + + (repo / "b.txt").write_text("suites_not_covered 3\n") + expect.num(_run("gate", "--ledger", ledger, "--budget", "b.txt", + "--registered", reg, "--against", "HEAD", log, cwd=repo)[1], 0, + "lowering it is allowed, which is the direction the burn-down goes") + + +def test_the_runner_invokes_the_gate_before_it_removes_the_logs(expect): + """A gate nothing runs is a comment -- selftest 350's phrasing about its own + subject. Nothing in the repository called this tool: zero references in `.github/`, + zero in the runner. Reported by @linuxhikerpm and by OffgridwithJD independently. + """ + text = RUNNER.read_text().splitlines() + call = [i for i, l in enumerate(text) if 'pgc_ledger.py" gate' in l] + teardown = [i for i, l in enumerate(text) if 'rm -rf "$builddir"' in l] + expect.num(len(call), 1, "the runner invokes the ledger gate exactly once") + expect.at_least(len(teardown), 1, "premise: the runner removes the build directory") + expect.text("before" if call[0] < teardown[-1] else "after", "before", + "and it runs before the logs are removed, the only place it can") + # The block is extracted, not a fixed-size window: a window's size is a fact + # about formatting, and the first version measured 8 lines and broke the moment + # the call site gained a comment. + src = RUNNER.read_text() + block = src[src.index("\t\t_led_rc=$?"):] + block = block[:block.index("\t\tesac") + len("\t\tesac")] + # Comments stripped: the block's own explanation quotes the sentences counted + # below, so an unstripped extraction counts the documentation as an occurrence. + block = "\n".join(l for l in block.splitlines() if not l.strip().startswith("#")) + expect.num(block.count("verfail=1"), 2, + "both failure arms fail the major") + expect.num(block.count("has a check the ledger has never seen"), 1, + "a refusal keeps the regenerate-the-ledger wording") + expect.num(block.count("could not run the ledger gate at all"), 1, + "and an integrity failure gets its own sentence, since regenerating " + "the ledger would not help") + + +def test_the_committed_ledger_and_budget_agree(expect): + """Both are tracked, so a change to either is a diff a reviewer sees. If they + disagree, one was edited by hand -- the failure this design refuses.""" + ledger = REPO / "test" / "check_ledger.tsv" + budget = REPO / "test" / "check_ledger_budget.txt" + expect.text("yes" if ledger.exists() else "no", "yes", "the ledger is in the tree") + expect.text("yes" if budget.exists() else "no", "yes", "the budget is in the tree") + + rows = [l.split("\t") for l in ledger.read_text().splitlines() if l] + never = [r for r in rows if r[3] == "never"] + red = [r for r in rows if r[3] != "never"] + print(f" ledger: inputs={len(rows)} | observed red={len(red)}, never={len(never)}") + expect.num(len(red) + len(never), len(rows), "the ledger partitions") + expect.num(len([r for r in rows if len(r) != 5]), 0, "every committed row has five fields") + expect.num(ledger.read_text().count("\t\n"), 0, "and none ends in a tab") + + nums = {} + for line in budget.read_text().splitlines(): + p = line.split() + if len(p) == 2 and p[1].isdigit() and not line.startswith("#"): + nums[p[0]] = int(p[1]) + expect.num(nums.get("checks_never_observed_red", -1), len(never), + "the committed census matches the committed ledger") + + listed = subprocess.run(["bash", str(RUNNER), "--list-suites"], + capture_output=True, text=True).stdout.split() + covered = {r[0] for r in rows} + expect.num(nums.get("suites_not_covered", -1), len(set(listed) - covered), + "and the ceiling matches the suites with no rows") + + +def test_a_log_that_does_not_parse_is_not_evidence(tmp_path, expect): + """`len(f) >= 5` accepted four shapes the emitter cannot produce. + + A record missing its reason, a verdict outside pgc_record's vocabulary, an empty + check name, and one record against `checks run: 2` all merged at rc=0 -- the + ledger absorbing as evidence a log that does not parse. The ledger's subject is + which checks have been observed red, so a malformed log is how an observation + gets attributed to a check that never ran. Reported by @linuxhikerpm on #918. + """ + led = _w(tmp_path, "l.tsv", "") + cases = { + "no reason field": "RESULT\tdemo\tpart1\ta name\tPASS\nchecks run: 1\n", + "a verdict the emitter cannot emit": "RESULT\tdemo\tpart1\ta name\tBOGUS\t\nchecks run: 1\n", + "an empty check name": "RESULT\tdemo\tpart1\t\tPASS\t\nchecks run: 1\n", + "a count that disagrees with the records": "RESULT\tdemo\tpart1\ta name\tPASS\t\nchecks run: 2\n", + "no count at all": "RESULT\tdemo\tpart1\ta name\tPASS\t\n", + } + for label, text in cases.items(): + log = _w(tmp_path, "bad.log", text) + expect.num(_run("merge", "--ledger", led, "--date", "2026-09-10", log)[1], 2, + f"{label} is an integrity failure, not a merge") + + # THE CONTROL. Five arms all reporting 2 prove nothing if the tool has simply + # started refusing every log. + good = _w(tmp_path, "good.log", GREEN) + expect.num(_run("merge", "--ledger", led, "--date", "2026-09-10", good)[1], 0, + "control: a well-formed log still merges") + + +def test_last_red_may_only_move_forward(tmp_path, expect): + """It was a plain assignment, so the answer depended on merge order. + + Merging an older log rewrote a recent observation with an older one, and an + undated merge replaced a real date with `unknown`. A ledger whose whole subject + is when a check was last seen red cannot let that regress. + """ + led = _w(tmp_path, "l.tsv", "") + red = _w(tmp_path, "red.log", RED) + + def stored(): + for line in pathlib.Path(led).read_text().splitlines(): + f = line.split("\t") + if len(f) > 3 and f[2] == "first check": + return f[3] + return None + + _run("merge", "--ledger", led, "--date", "2026-09-10", red) + _run("merge", "--ledger", led, "--date", "2026-09-01", red) + expect.text(stored(), "2026-09-10", "an older observation does not overwrite a newer one") + _run("merge", "--ledger", led, "--date", "2026-09-20", red) + expect.text(stored(), "2026-09-20", "and a newer one does") + _run("merge", "--ledger", led, red) + expect.text(stored(), "2026-09-20", "and an undated merge does not erase a known date") + expect.num(_run("merge", "--ledger", led, "--date", "not-a-date", red)[1], 2, + "a date that is not a date is refused rather than stored") + + +def test_a_mutation_names_one_check_not_every_casualty(tmp_path, expect): + """One deliberate change can redden the target and whatever depended on it. + + Attributing `--mutation` to every failure records collateral damage as evidence + that the mutation kills that check, which is the opposite of what the column is + for. Measured on #918: a two-FAIL log recorded it against both. + """ + led = _w(tmp_path, "l.tsv", "") + two = _w(tmp_path, "two.log", + "RESULT\tdemo\tpart1\tthe target\tFAIL\t\n" + "RESULT\tdemo\tpart1\tcollateral\tFAIL\t\nchecks run: 2\n") + out, rc = _run("merge", "--ledger", led, "--date", "2026-09-10", "--mutation", "M", two) + expect.num(rc, 2, "--mutation across two failing checks in one run is refused") + expect.num(out.count("2 checks failed"), 1, + "and the refusal counts them, so the author can narrow the run") + expect.num(_run("merge", "--ledger", led, "--date", "2026-09-10", two)[1], 0, + "control: the same log merges without --mutation") + + one = _w(tmp_path, "one.log", RED) + _run("merge", "--ledger", led, "--date", "2026-09-10", "--mutation", "M", one) + rows = [l.split("\t") for l in pathlib.Path(led).read_text().splitlines() if l.strip()] + tagged = [r[2] for r in rows if len(r) > 4 and "M" in r[4].split(";")] + expect.rows([[n] for n in sorted(tagged)], [["first check"]], + "and a single failure still carries it, on the check that reddened") diff --git a/test/pytest/test_suite_accounting.py b/test/pytest/test_suite_accounting.py index 764e5d63..8b75119c 100644 --- a/test/pytest/test_suite_accounting.py +++ b/test/pytest/test_suite_accounting.py @@ -111,10 +111,10 @@ def test_the_accounting_line_is_read_on_every_exit_path(tmp_path, expect): for PASSED: a suite that failed still reached its summary and still accounted. """ shapes = { - "pass": "accounting: 3 passed + 0 failed + 0 unrunnable = 3\nx.sh: PASSED\n", - "fail": "accounting: 1 passed + 2 failed + 0 unrunnable = 3\nx.sh: FAILED\n", - "skip": "accounting: 0 passed + 0 failed + 0 unrunnable = 0\nx.sh: SKIPPED (ran no checks)\n", - "inc": "accounting: 2 passed + 0 failed + 1 unrunnable = 3\nx.sh: INCOMPLETE\n", + "pass": "accounting: 3 passed + 0 failed + 0 unrunnable + 0 skipped = 3\nx.sh: PASSED\n", + "fail": "accounting: 1 passed + 2 failed + 0 unrunnable + 0 skipped = 3\nx.sh: FAILED\n", + "skip": "accounting: 0 passed + 0 failed + 0 unrunnable + 0 skipped = 0\nx.sh: SKIPPED (ran no checks)\n", + "inc": "accounting: 2 passed + 0 failed + 1 unrunnable + 0 skipped = 3\nx.sh: INCOMPLETE\n", } for shape, text in shapes.items(): log = _write(tmp_path, f"{shape}.log", text) @@ -135,9 +135,9 @@ def test_the_accounting_line_is_read_on_every_exit_path(tmp_path, expect): # arm here green. Reported by OffgridwithJD. The distinguishing input is a # well-formed accounting line that does not start its line. indented = _write(tmp_path, "indented.log", - " accounting: 3 passed + 0 failed + 0 unrunnable = 3\nx.sh: PASSED\n") + " accounting: 3 passed + 0 failed + 0 unrunnable + 0 skipped = 3\nx.sh: PASSED\n") expect.num(pathlib.Path(indented).read_text() - .count("accounting: 3 passed + 0 failed + 0 unrunnable = 3"), 1, + .count("accounting: 3 passed + 0 failed + 0 unrunnable + 0 skipped = 3"), 1, "premise: the fixture carries a well-formed line, just indented") expect.text(_call("pgc_log_shows_accounting", indented)[0].strip(), "no", "an accounting line that does not start its line is refused") @@ -368,7 +368,7 @@ def test_the_accounted_reader_takes_either_runtime_mechanism(tmp_path, expect): that adopts either leaves the debt bucket on its own. """ lib = _write(tmp_path, "lib.log", - "accounting: 1 passed + 0 failed + 0 unrunnable = 1\nx.sh: PASSED\n") + "accounting: 1 passed + 0 failed + 0 unrunnable + 0 skipped = 1\nx.sh: PASSED\n") own = _write(tmp_path, "own.log", "checks run: 9\ndocs_style.sh: PASSED\n") neither = _write(tmp_path, "none.log", "some output\nPASSED\n") expect.text(_call("pgc_log_shows_any_accounting", lib)[0].strip(), "yes", diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 4af868b9..8edb93e2 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -933,13 +933,87 @@ pgc_log_shows_accounting() { # pgc_log_shows_accounting LOGFILE -> yes|no # of thing that reads as an oversight later. local _log="$1" [ -f "$_log" ] || { echo no; return 0; } - if grep -qE '^accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable = [0-9]+$' "$_log"; then + if grep -qE '^accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable \+ [0-9]+ skipped = [0-9]+$' "$_log"; then echo yes else echo no fi } +pgc_reconcile_records() { # pgc_reconcile_records LOGFILE -> 0 ok, 1 mismatch + # A suite's log states `checks run: N` and carries N record lines. They are + # the same increment seen twice -- pgc_record does both -- so this cannot + # fail by drifting. It CAN fail, which is why it is asserted: a suite killed + # mid-way, a truncated log, or a helper that prints an outcome without + # recording it all separate the two. + # + # A log with no `checks run:` line at all never reached its summary. That is a + # different fault from a miscount, and it must not read as a clean + # reconciliation just because there is nothing to compare against. + local _log="$1" _records _stated + if [ ! -f "$_log" ]; then + echo " no log to reconcile records against: $_log" + return 1 + fi + _records="$(grep -c '^RESULT ' "$_log" || true)" + + # THE COUNT IS NOT THE SCHEMA, and counting alone let six malformed shapes + # reconcile cleanly: a record missing two fields, one carrying extra fields, + # a verdict outside the vocabulary, an empty check name, and every field + # empty. All measured returning 0 before this arm, against a well-formed + # control that also returned 0 -- so the function could not tell them apart. + # Reported by @linuxhikerpm on #917. + # + # ONE awk PASS, not a loop with a fork per record: a full matrix run carries + # thousands of these, and the emitter next door already paid for that lesson + # at 331x. The verdict list is the emitter's own, so the two cannot drift + # without this going red. + local _bad + _bad="$(awk -F'\t' ' + /^RESULT / { + n++ + if (NF != 6) { why[n] = "has " NF-1 " fields, want 5"; bad++; next } + if ($2 == "" || $3 == "" || $4 == "") { why[n] = "has an empty suite, part or name"; bad++; next } + if ($5 != "PASS" && $5 != "FAIL" && $5 != "UNRUN" && $5 != "SKIP") { + why[n] = "has verdict \"" $5 "\", which pgc_record cannot emit"; bad++; next + } + } + END { + if (bad) { for (i = 1; i <= n; i++) if (i in why) print " record " i " " why[i] } + exit 0 + }' "$_log")" + if [ -n "$_bad" ]; then + echo " $_records record(s) present but at least one does not parse:" + printf '%s\n' "$_bad" | head -5 + return 1 + fi + + _stated="$(sed -n 's/^checks run: \([0-9][0-9]*\)$/\1/p' "$_log" | tail -1)" + if [ -z "$_stated" ]; then + echo " records=$_records but the log never stated a count, so it did not reach its summary" + return 1 + fi + if [ "$_records" != "$_stated" ]; then + echo " records=$_records but the log states checks run: $_stated" + # NAME THE CAUSE, not just the arithmetic. The two directions have + # different causes and a reader who has not met either has no route from + # a pair of numbers to the defect. Raised by OffgridwithJD. + if [ "$_records" -gt "$_stated" ]; then + echo " $((_records - _stated)) check(s) reported an outcome the count never saw:" + echo " a check ran in a subshell, so its counter bump died with it while its" + echo " outcome and record still reached the log. The usual shape is a check" + # The example is ASSEMBLED, not written out: spelling the shape here + # made the sweep in selftest 400 flag this very line. + printf ' inside a piped loop -- `cmd %s while read x; do check ...; done`.\n' '|' + else + echo " $((_stated - _records)) check(s) were counted without emitting a record:" + echo " something bumped PGC_CHECKS without going through pgc_record." + fi + return 1 + fi + return 0 +} + pgc_log_shows_any_accounting() { # pgc_log_shows_any_accounting LOGFILE -> yes|no # Did this suite count its checks AT RUNTIME, by any mechanism the log shows? # @@ -951,9 +1025,20 @@ pgc_log_shows_any_accounting() { # pgc_log_shows_any_accounting LOGFILE -> yes|n # Both are runtime-observable and derived rather than declared, so a suite # that adopts either mechanism leaves the debt bucket on its own -- which is # the property that keeps the debt file from becoming a permission slip. + # A HALF-MERGE WOULD BE CONFUSING RATHER THAN LOUD, and it is worth knowing + # which way. The `checks run:` alternative below answers YES to an accounting + # line of ANY shape, so it MASKS a change to that line: if the producer ever + # moved without this file, the population reconciliation would stay green + # while pgc_log_shows_accounting broke and the accounting reconciliation + # reddened. Two checks disagreeing about the same log is a worse signal than + # either failing. + # + # It cannot happen inside one tree -- producer and both readers move in the + # same commit -- so this is a note about what to look for, not a defect. + # Raised by OffgridwithJD while verifying the four-term shape change. local _log="$1" [ -f "$_log" ] || { echo no; return 0; } - if [ "$(grep -cE '^accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable = [0-9]+$' "$_log" || true)" -ne 0 ] \ + if [ "$(grep -cE '^accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable \+ [0-9]+ skipped = [0-9]+$' "$_log" || true)" -ne 0 ] \ || [ "$(grep -cE '^checks run: [0-9]+$' "$_log" || true)" -ne 0 ]; then echo yes else @@ -1192,11 +1277,25 @@ pgc_tally_suite() { # pgc_tally_suite NAME VERDICT LOGFILE # PG16 would report PG15's incomplete suites in its own summary line and # still print PASS, because verfail is per major and this count was not. suites_incomplete=0 + _rec_bad=0 for s in "${SUITES[@]}"; do _rc="$(cat "$builddir/${s}.rc" 2>/dev/null)" _verdict="$(pgc_classify_suite_rc "$_rc" "$builddir/${s}.log")" pgc_tally_suite "$s" "$_verdict" "$builddir/${s}.log" + # Only a suite that reached its summary has a count to reconcile against + # (#917). One that was never dispatched, or that does not use lib.sh's + # accounting at all, has nothing to compare and is not a mismatch. + if [ "$(pgc_log_shows_accounting "$builddir/${s}.log")" = yes ]; then + if ! pgc_reconcile_records "$builddir/${s}.log"; then + echo " in $s" + _rec_bad=$((_rec_bad + 1)) + fi + fi done + if [ "$_rec_bad" != 0 ]; then + echo " $_rec_bad suite(s) on PG$major state a check count their records do not match" + verfail=1 + fi # How many suites actually asserted something, said out loud (#447). # @@ -1262,6 +1361,71 @@ pgc_tally_suite() { # pgc_tally_suite NAME VERDICT LOGFILE verfail=1 fi + # THE LEDGER GATE (#918). Every suite's log is here and the build directory is + # about to be removed, so this is the only place a matrix run can feed it. + # + # The gate runs against the COMMITTED ledger: a check it has never seen is + # named and refused, which is the allowlist the issue asks for. Regenerating + # the ledger is the intended fix and a reviewable diff, so this cannot + # deadlock the way a ceiling on `never` rows did. + # + # CI verifies; humans commit. A ledger that CI rewrote by itself would be a + # file nobody reads changing under everybody. + # + # Only logs that CARRY records are passed. The twelve suites outside lib.sh's + # accounting produce none, and the tool fails closed on an empty input -- + # correctly, since a caller asking it to reconcile nothing is a caller with a + # bug. + _led_logs="" + for s in "${SUITES[@]}"; do + [ -s "$builddir/${s}.log" ] || continue + [ "$(grep -c '^RESULT ' "$builddir/${s}.log" || true)" -ne 0 ] \ + && _led_logs="$_led_logs $builddir/${s}.log" + done + if [ -z "$_led_logs" ]; then + echo " no suite emitted a check record on PG$major, so the ledger has nothing to gate" + verfail=1 + elif [ ! -f "$builddir/test/check_ledger.tsv" ]; then + echo " the ledger is missing from the tree under test, which is not a pass" + verfail=1 + else + # WHICH REF CARRIES THE PRIOR CEILING is resolved by the tool, from + # GITHUB_BASE_REF in CI or the local main's configured upstream outside + # it, and it FAILS CLOSED when neither gives a trustworthy answer. + # + # It used to be chosen here, preferring origin/main with a printed + # fallback to HEAD. Both halves were wrong. `origin` is per-clone -- in a + # contributor's setup it is their fork, measured 446 commits stale -- and + # comparing against an older main makes the check WEAKER rather than + # falsely red, because the ceiling may only fall. And the fallback to HEAD + # compares a committed file against itself, so it caught nothing for any + # change under review while printing that it had compared. A gate that + # quietly enforces less than it claims is what this whole change refuses. + # shellcheck disable=SC2086 + python3 "$builddir/test/pgc_ledger.py" gate \ + --ledger "$builddir/test/check_ledger.tsv" \ + --budget "$builddir/test/check_ledger_budget.txt" \ + --registered "$_acc_registered" \ + --against auto \ + $_led_logs + _led_rc=$? + # BRANCH ON THE STATUS THE TOOL WENT TO THE TROUBLE OF DISTINGUISHING. + # rc=1 is a real refusal and the fix is to regenerate the ledger; rc=2 is + # the gate unable to do its job at all, where regenerating helps nothing. + # Collapsing them printed "has a check the ledger has never seen" three + # lines below the gate's own "new this run=0", which contradicts it and + # sends the reader at the wrong repair. Reported by OffgridwithJD. + case "$_led_rc" in + 0) ;; + 1) echo " PG$major has a check the ledger has never seen, which is not a pass" + verfail=1 ;; + *) echo " PG$major could not run the ledger gate at all, which is not a pass:" + echo " the input or the prior ceiling was unusable, and regenerating the" + echo " ledger will not help. The failure above says which." + verfail=1 ;; + esac + fi + # How many of the suites counted as having RUN actually accounted for their # checks (#916). Ten registered suites exit 0 having never called pgc_summary; # counting them among the suites that ran is the overcount #447 added this @@ -1285,6 +1449,24 @@ pgc_tally_suite() { # pgc_tally_suite NAME VERDICT LOGFILE SUMMARY+=("FAIL PG$major ($suites_ran ran, $suites_skipped skipped, $suites_incomplete incomplete) ${results}") overall=1 fi + # KEEP THE LOGS, THEN DELETE THE BUILD DIRECTORY. The gate runs above and the + # rm runs here, and CI's "Collect logs on failure" step globs + # /tmp/pgcolumnar-matrix-*/*.log AFTER this loop has finished -- so it searched + # a directory this line had already removed and collected nothing. Run + # 34503924812 is the measurement. That mattered beyond diagnosis: the ledger is + # fed by merging real logs, and a CI red is exactly the run whose logs record a + # check going red for the first time. Deleting them meant CI could never feed + # the thing it gates. Reported by @linuxhikerpm. + # + # Copied rather than left in place, because the build directory is large and + # the logs are not, and because a retained path that does not move is what a + # workflow step can name. + _logkeep="${PGC_LOG_KEEP:-/tmp/pgcolumnar-logs}" + mkdir -p "$_logkeep" + for _l in "$builddir"/*.log; do + [ -e "$_l" ] || continue + cp -p "$_l" "$_logkeep/pg${major}-${_l##*/}" + done rm -rf "$builddir" done diff --git a/test/selftest/320-a-check-that-could-not-run.sh b/test/selftest/320-a-check-that-could-not-run.sh index 439d919f..f7dc7e7b 100644 --- a/test/selftest/320-a-check-that-could-not-run.sh +++ b/test/selftest/320-a-check-that-could-not-run.sh @@ -87,8 +87,8 @@ check "one unrunnable check makes the suite INCOMPLETE, not passed" \ # the feature from its absence. Assert the accounting line instead, which only a # suite that recorded BOTH states can print. check "a failure outranks an unrunnable check, and both are still counted" \ - "$(_cur_out failunrun | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable')" \ - "accounting: 0 passed + 1 failed + 1 unrunnable" + "$(_cur_out failunrun | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable \+ [0-9]+ skipped')" \ + "accounting: 0 passed + 1 failed + 1 unrunnable + 0 skipped" # The distinction 66 cannot carry: this suite RAN a check. Reporting it as # "ran no checks" would merge "inert suite" with "could not evaluate one thing". @@ -125,12 +125,12 @@ check "an unrunnable reason outside the enum fails rather than being accepted" \ # Every state is in a total, or it is a state that can go missing. 3,762 check # sites is well past what anyone notices by reading. check "the summary reconciles the three states against the total" \ - "$(_cur_out passunrun | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable = [0-9]+')" \ - "accounting: 1 passed + 0 failed + 1 unrunnable = 2" + "$(_cur_out passunrun | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable \+ [0-9]+ skipped = [0-9]+')" \ + "accounting: 1 passed + 0 failed + 1 unrunnable + 0 skipped = 2" check "and a suite with no unrunnable checks reconciles too" \ - "$(_cur_out onlypass | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable = [0-9]+')" \ - "accounting: 1 passed + 0 failed + 0 unrunnable = 1" + "$(_cur_out onlypass | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable \+ [0-9]+ skipped = [0-9]+')" \ + "accounting: 1 passed + 0 failed + 0 unrunnable + 0 skipped = 1" # ---- the accounting must be a MEASUREMENT, not an identity ------------------ # @@ -155,8 +155,8 @@ _cur_make ratio 'check "a" ok ok check_ratio "a ratio well inside its bound" 10 100 1.0' check "a passing ratio check is counted as a pass, not a failure" \ - "$(_cur_out ratio | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable = [0-9]+')" \ - "accounting: 2 passed + 0 failed + 0 unrunnable = 2" + "$(_cur_out ratio | grep -oE 'accounting: [0-9]+ passed \+ [0-9]+ failed \+ [0-9]+ unrunnable \+ [0-9]+ skipped = [0-9]+')" \ + "accounting: 2 passed + 0 failed + 0 unrunnable + 0 skipped = 2" check "and the suite that holds it still passes" \ "$(_cur_run ratio)" "0 PASSED" @@ -201,8 +201,15 @@ while IFS= read -r _cnt_l; do done < <(grep -rn 'PGC_CHECKS=\$((PGC_CHECKS' "$_cnt_dir"/*.sh "$_cnt_dir"/selftest/*.sh 2>/dev/null \ | grep -v '/lib\.sh:' | sort) -check "premise: the sweep finds the direct writes it is meant to police" \ - "$([ "${#_cnt_sites[@]}" -ge 5 ] && echo enough || echo "${#_cnt_sites[@]}")" "enough" +# The premise used to require FIVE direct writes to exist, which was a premise +# about the corpus rather than about the sweep -- and #917 converted the thirteen +# that used lib.sh's accounting, so it went red for the reason the change is FOR. +# A premise that fails when the thing it guards is fixed is the wrong premise. +# +# It now asserts the sweep read something, and the classifier is proven on a +# FIXTURE below rather than on whatever the corpus happens to contain. +check "premise: the sweep read the corpus and found sites to classify" \ + "$([ "${#_cnt_sites[@]}" -ge 1 ] && echo yes || echo "no (${#_cnt_sites[@]})")" "yes" # A file that keeps its OWN counters and never calls pgc_summary is not bound by # this invariant, because nothing reconciles it. Asserted rather than assumed: @@ -223,6 +230,83 @@ done check "every direct write to PGC_CHECKS records an outcome too" \ "$(_tsm_fmt_cnt "$_cnt_n" "$_cnt_bad")" "[]" +# AND, SINCE #917, THERE MUST BE NONE AT ALL among files that use lib.sh's +# accounting. That is a stronger rule than the one above and it was not +# satisfiable before: counting a check and RECORDING it are now one operation, +# pgc_record, so a direct write to PGC_CHECKS counts a check that emits no +# RESULT line -- and pgc_reconcile_records then reports a records/checks mismatch +# on top of whatever the suite was actually failing for. +# +# The two rules disagreed until the thirteen sites were converted, which +# OffgridwithJD found: selftest 320 blessed a direct write with a nearby outcome, +# while the reconciler required a record. pgc_pass and pgc_fail exist precisely so +# a suite-local helper need not touch the counters, and lib.sh's own header +# already calls them "the only supported way to add a check from outside this +# file". This arm makes that a rule rather than a description. +# +# A file that keeps its OWN counters and never calls pgc_summary is exempt for the +# same reason as above, measured from the file rather than named: bench_guards +# reuses the variable name privately and never sources lib.sh. +_cnt_lib=0; _cnt_lib_bad="" +for _cnt_l in "${_cnt_sites[@]}"; do + _cnt_f="${_cnt_l%%:*}" + _cnt_ln="$(printf '%s' "$_cnt_l" | cut -d: -f2)" + [ "$(grep -c 'pgc_summary' "$_cnt_f" || true)" -ne 0 ] || continue + _cnt_lib=$((_cnt_lib + 1)) + [ "$_cnt_lib" -le 5 ] && _cnt_lib_bad="$_cnt_lib_bad ${_cnt_f##*/}:$_cnt_ln" +done +check "no suite that uses lib.sh's accounting writes PGC_CHECKS directly" \ + "$(_tsm_fmt_cnt "$_cnt_lib" "$_cnt_lib_bad")" "[]" + +# BOTH RULES, PROVEN ON A FIXTURE. With the corpus clean, two arms reading "[]" +# are satisfied by a sweep that classifies nothing, so the classifier is driven +# against files built to trip it. +_cnt_fx="$PGC_WORKDIR/cntfx"; mkdir -p "$_cnt_fx" + +# Uses lib.sh's accounting, bumps the counter, records no outcome nearby: both +# rules must see it. +# The bump is ASSEMBLED from the variable name rather than written out, so these +# generator lines do not themselves carry the shape the sweep looks for. Writing +# it literally made this file flag its own three fixtures -- the same mistake +# selftest 080's control avoids by living in a quoted heredoc. +_cnt_bump="$(printf '%s=$((%s + 1))' PGC_CHECKS PGC_CHECKS)" +_cnt_out="$(printf '%s=$((%s + 1))' PGC_FAILED PGC_FAILED)" +printf 'pgc_summary\n%s\necho hi\n' "$_cnt_bump" > "$_cnt_fx/bare.sh" +# Uses lib.sh's accounting, bumps the counter, DOES record an outcome: the old +# rule lets it through, the new one does not. That difference is the change. +printf 'pgc_summary\n%s\n%s\n' "$_cnt_bump" "$_cnt_out" > "$_cnt_fx/outcome.sh" +# Keeps its own counter and never calls pgc_summary: exempt from both. +printf '%s\necho "checks run: 1"\n' "$_cnt_bump" > "$_cnt_fx/private.sh" + +_cnt_fx_old() { # the ORIGINAL rule, applied to one file + local _f="$1" _l + [ "$(grep -c 'pgc_summary' "$_f" || true)" -ne 0 ] || { echo exempt; return; } + _l="$(grep -n 'PGC_CHECKS=\$((PGC_CHECKS' "$_f" | head -1 | cut -d: -f1)" + [ -n "$_l" ] || { echo none; return; } + if [ "$(sed -n "$((_l > 3 ? _l - 3 : 1)),$((_l + 6))p" "$_f" \ + | grep -cE 'PGC_PASSED=|PGC_FAILED=|PGC_UNRUN=' || true)" = 0 ]; then + echo flagged + else + echo allowed + fi +} +_cnt_fx_new() { # the STRONGER rule, applied to one file + local _f="$1" + [ "$(grep -c 'pgc_summary' "$_f" || true)" -ne 0 ] || { echo exempt; return; } + [ "$(grep -c 'PGC_CHECKS=\$((PGC_CHECKS' "$_f" || true)" -ne 0 ] && echo flagged || echo none +} + +check "premise: the fixtures carry the shapes these rules are about" \ + "$(grep -lc 'PGC_CHECKS=\$((PGC_CHECKS' "$_cnt_fx"/*.sh 2>/dev/null | grep -c . || true)" "3" +check "the original rule flags a bump that records no outcome" \ + "$(_cnt_fx_old "$_cnt_fx/bare.sh")" "flagged" +check "and allows one that does, which is what it was written to allow" \ + "$(_cnt_fx_old "$_cnt_fx/outcome.sh")" "allowed" +check "the stronger rule flags that same allowed bump, which is the change" \ + "$(_cnt_fx_new "$_cnt_fx/outcome.sh")" "flagged" +check "and both exempt a file that keeps its own counter without lib.sh" \ + "$(_cnt_fx_old "$_cnt_fx/private.sh")/$(_cnt_fx_new "$_cnt_fx/private.sh")" "exempt/exempt" + # ---- and the RUNNER must not report an INCOMPLETE suite as a pass ------------ # # lib.sh exiting 67 is only half the state. The runner decides what a status diff --git a/test/selftest/330-the-incomplete-path-must-run-whole.sh b/test/selftest/330-the-incomplete-path-must-run-whole.sh index ea51103f..6e2075c1 100644 --- a/test/selftest/330-the-incomplete-path-must-run-whole.sh +++ b/test/selftest/330-the-incomplete-path-must-run-whole.sh @@ -92,26 +92,40 @@ check "control fixture: a suite whose checks all ran exits 0" \ _e2e_txt_classify="$(sed -n '/^pgc_classify_suite_rc()/,/^}/p' "$_e2e_rv")" _e2e_txt_fails="$(sed -n '/^pgc_verdict_fails_major()/,/^}/p' "$_e2e_rv")" _e2e_txt_tally="$(sed -n '/^pgc_tally_suite()/,/^}/p' "$_e2e_rv")" +# TWO MORE, because the collect loop grew two callers and this part did not follow. +# +# #917 added a records-versus-count reconciliation inside the loop, guarded by +# pgc_log_shows_accounting. Neither was evalled here, and the guard is a COMMAND +# SUBSTITUTION: an undefined function yields "", the branch never runs, and the +# first absence hides the second. A real run printed `command not found` three +# times and still reported PASSED, 608 checks, 0 failures. +# +# So the part claiming to run "the runner's OWN collect loop" was running a loop +# whose reconciliation could not execute. Reported by OffgridwithJD. +_e2e_txt_anyacct="$(sed -n '/^pgc_log_shows_accounting()/,/^}/p' "$_e2e_rv")" +_e2e_txt_records="$(sed -n '/^pgc_reconcile_records()/,/^}/p' "$_e2e_rv")" -check "premise: all three runner functions were extracted, not empty ranges" \ - "$([ -n "$_e2e_txt_classify" ] && echo y || echo n)$([ -n "$_e2e_txt_fails" ] && echo y || echo n)$([ -n "$_e2e_txt_tally" ] && echo y || echo n)" \ - "yyy" +check "premise: all five runner functions were extracted, not empty ranges" \ + "$([ -n "$_e2e_txt_classify" ] && echo y || echo n)$([ -n "$_e2e_txt_fails" ] && echo y || echo n)$([ -n "$_e2e_txt_tally" ] && echo y || echo n)$([ -n "$_e2e_txt_anyacct" ] && echo y || echo n)$([ -n "$_e2e_txt_records" ] && echo y || echo n)" \ + "yyyyy" # A truncated extraction evals to a syntax error, not to nothing, so the closing # brace is asserted rather than assumed: the sed range stops at the first line # beginning with `}`, and a body containing one would yield a fragment. check "premise: and each extraction ends at its own closing brace" \ - "$(printf '%s\n%s\n%s\n' "$_e2e_txt_classify" "$_e2e_txt_fails" "$_e2e_txt_tally" | grep -c '^}$')" \ - "3" + "$(printf '%s\n%s\n%s\n%s\n%s\n' "$_e2e_txt_classify" "$_e2e_txt_fails" "$_e2e_txt_tally" \ + "$_e2e_txt_anyacct" "$_e2e_txt_records" | grep -c '^}$')" \ + "5" eval "$_e2e_txt_classify" eval "$_e2e_txt_fails" eval "$_e2e_txt_tally" +eval "$_e2e_txt_anyacct" +eval "$_e2e_txt_records" -check "premise: and all three are callable" \ - "$(type -t pgc_classify_suite_rc)/$(type -t pgc_verdict_fails_major)/$(type -t pgc_tally_suite)" \ - "function/function/function" - +check "premise: and all five are callable" \ + "$(type -t pgc_classify_suite_rc)/$(type -t pgc_verdict_fails_major)/$(type -t pgc_tally_suite)/$(type -t pgc_log_shows_accounting)/$(type -t pgc_reconcile_records)" \ + "function/function/function/function/function" check "the runner classifies the file that suite actually produced" \ "$(pgc_classify_suite_rc "$(cat "$_e2e_dir/e2e_incomplete.rc")" "$_e2e_dir/e2e_incomplete.log")" \ "INCOMPLETE" @@ -165,6 +179,22 @@ check "and reprints the suite's own UNRUN line beneath it" \ _e2e_txt_loop="$(awk '/^\tsuites_incomplete=/{f=1} f{print} f&&/^\tdone$/{exit}' "$_e2e_rv")" +# AND NOTHING THE LOOP CALLS MAY BE MISSING. The premise above names five because +# five is what the loop calls today; a sixth caller added later would reproduce +# exactly the failure this fixes, silently. So the loop's own text is swept for +# the functions it invokes, and every one must be defined here. +_e2e_loop_calls="$(printf '%s\n' "$_e2e_txt_loop" | grep -oE '(^|[^_[:alnum:]])pgc_[a-z_]+' \ + | grep -oE 'pgc_[a-z_]+' | sort -u)" +_e2e_undef=0 +while IFS= read -r _e2e_fn; do + [ -n "$_e2e_fn" ] || continue + [ "$(type -t "$_e2e_fn" 2>/dev/null)" = function ] || { + _e2e_undef=$((_e2e_undef + 1)); echo " the collect loop calls $_e2e_fn, which this part never evalled"; } +done <<<"$_e2e_loop_calls" +check "premise: the sweep found the loop's callers to check" \ + "$([ "$(printf '%s' "$_e2e_loop_calls" | grep -c .)" -ge 3 ] && echo yes || echo no)" "yes" +check "every function the collect loop calls is defined here" "$_e2e_undef" "0" + check "premise: the runner's collect loop was extracted, not an empty range" \ "$([ -n "$_e2e_txt_loop" ] && echo yes || echo no)" "yes" diff --git a/test/selftest/340-the-binary-must-be-built-from.sh b/test/selftest/340-the-binary-must-be-built-from.sh index 78797aba..5a2bdee7 100644 --- a/test/selftest/340-the-binary-must-be-built-from.sh +++ b/test/selftest/340-the-binary-must-be-built-from.sh @@ -265,7 +265,7 @@ printf 'static int x=1;\n' > "$_fs_rp/src/a.c" printf 'static int x=2;\n' > "$_fs_rp/src/b.c" _fs_before="$(pgc_source_fingerprint "$_fs_rp")" -check "premise: the fixture fingerprints at all" \ +check "premise: the partition fixture fingerprints at all" \ "$([ -n "$_fs_before" ] && echo yes || echo no)" "yes" # The same bytes, a different partition: a.c gains b.c's line and b.c is emptied. @@ -557,7 +557,7 @@ _fp_as() { # _fp_as EXPR -> stdout } if [ -z "$_fp_user" ]; then - echo "SKIP no non-root user to read as; root ignores chmod 000" + check_skip "the unreadable-source refusal" "SKIP no non-root user to read as; root ignores chmod 000" "no non-root user to read as" else _fp_base="$(_fp_as "pgc_source_fingerprint \"$_fp/tree\"")" check "premise: the tree fingerprints to something when it is readable" \ @@ -891,7 +891,7 @@ if [ "$_lc_count" -ge 2 ]; then for _lc_l in $_lc_have; do _lc_vals="$_lc_vals $(LC_ALL="$_lc_l" LANG="$_lc_l" pgc_source_fingerprint "$_lc/tree")" done - check "premise: the fixture fingerprints at all" \ + check "premise: every locale produced a fingerprint" \ "$([ -n "$(printf '%s' $_lc_vals)" ] && echo yes || echo empty)" "yes" check "one tree, one fingerprint, whatever the locale" \ "$(printf '%s\n' $_lc_vals | sort -u | grep -c .)" "1" diff --git a/test/selftest/390-a-registered-suite-must-account.sh b/test/selftest/390-a-registered-suite-must-account.sh index 42321d4a..90f35a9b 100644 --- a/test/selftest/390-a-registered-suite-must-account.sh +++ b/test/selftest/390-a-registered-suite-must-account.sh @@ -164,18 +164,18 @@ check "and the reader answers no on it, which is the wrong answer the arm catche # on a pass, a failure, a skip and an incomplete alike. That is what makes it the # runtime twin of the declaration rather than a synonym for PASSED. -printf 'checks run: 3\nchecks unrunnable: 0\naccounting: 3 passed + 0 failed + 0 unrunnable = 3\nx.sh: PASSED\n' > "$_acc/pass.log" +printf 'checks run: 3\nchecks unrunnable: 0\naccounting: 3 passed + 0 failed + 0 unrunnable + 0 skipped = 3\nx.sh: PASSED\n' > "$_acc/pass.log" check "a passing log shows accounting" "$(pgc_log_shows_accounting "$_acc/pass.log")" "yes" -printf 'accounting: 1 passed + 2 failed + 0 unrunnable = 3\nx.sh: FAILED\n' > "$_acc/fail.log" +printf 'accounting: 1 passed + 2 failed + 0 unrunnable + 0 skipped = 3\nx.sh: FAILED\n' > "$_acc/fail.log" check "and so does a failing one, which is the point" \ "$(pgc_log_shows_accounting "$_acc/fail.log")" "yes" -printf 'accounting: 0 passed + 0 failed + 0 unrunnable = 0\nx.sh: SKIPPED (ran no checks)\n' > "$_acc/skip.log" +printf 'accounting: 0 passed + 0 failed + 0 unrunnable + 0 skipped = 0\nx.sh: SKIPPED (ran no checks)\n' > "$_acc/skip.log" check "and a skip, which reached the summary and counted zero" \ "$(pgc_log_shows_accounting "$_acc/skip.log")" "yes" -printf 'accounting: 2 passed + 0 failed + 1 unrunnable = 3\nx.sh: INCOMPLETE\n' > "$_acc/inc.log" +printf 'accounting: 2 passed + 0 failed + 1 unrunnable + 0 skipped = 3\nx.sh: INCOMPLETE\n' > "$_acc/inc.log" check "and an incomplete" "$(pgc_log_shows_accounting "$_acc/inc.log")" "yes" printf 'x.sh: PASSED\n' > "$_acc/bare.log" @@ -193,10 +193,10 @@ check "and prose containing the word does not count as the line" \ # suite run produces. Inert on real data today (0 non-line-start occurrences # across 246 PG17 logs and 244 PG18), so this closes a coverage gap rather than a # live defect. -printf ' accounting: 3 passed + 0 failed + 0 unrunnable = 3\nx.sh: PASSED\n' \ +printf ' accounting: 3 passed + 0 failed + 0 unrunnable + 0 skipped = 3\nx.sh: PASSED\n' \ > "$_acc/indented_acc.log" check "premise: the fixture carries a well-formed accounting line, just indented" \ - "$(grep -c 'accounting: 3 passed + 0 failed + 0 unrunnable = 3' "$_acc/indented_acc.log")" "1" + "$(grep -c 'accounting: 3 passed + 0 failed + 0 unrunnable + 0 skipped = 3' "$_acc/indented_acc.log")" "1" check "an accounting line that does not start its line is refused" \ "$(pgc_log_shows_accounting "$_acc/indented_acc.log")" "no" @@ -532,7 +532,7 @@ check "premise: the population reconciliation is callable" \ # ---- the accounted reader takes EITHER mechanism ---------------------------- -printf 'accounting: 1 passed + 0 failed + 0 unrunnable = 1\nx.sh: PASSED\n' > "$_acc/lib.log" +printf 'accounting: 1 passed + 0 failed + 0 unrunnable + 0 skipped = 1\nx.sh: PASSED\n' > "$_acc/lib.log" check "a log carrying lib.sh's accounting line is accounted" \ "$(pgc_log_shows_any_accounting "$_acc/lib.log")" "yes" diff --git a/test/selftest/400-a-check-result-must-be-machine.sh b/test/selftest/400-a-check-result-must-be-machine.sh new file mode 100644 index 00000000..15c1789a --- /dev/null +++ b/test/selftest/400-a-check-result-must-be-machine.sh @@ -0,0 +1,526 @@ +# ---- a check result must be machine-readable, from ONE counter -------------- +# +# Check results are prose. `check`, `check_num` and `check_text` print PASS or +# FAIL and nothing else, so proving that a mutation reddened one NAMED check +# means grepping text. Every mutation proof in this repository is currently a +# person reading `FAIL ` out of a log and retyping it into a comment. +# +# That is how a reverted guard once reported plain green while the check count +# fell from 190 to 186: the suite passed, and the only evidence anything had +# changed was a number nobody was comparing. +# +# The fix is NOT a second emitter beside the counters. A second source of truth +# for how many checks ran is the defect this issue family exists to close, and +# lib.sh had ELEVEN places that bumped PGC_CHECKS -- eleven chances to add the +# twelfth and forget the line beside it. +# +# So counting a check and recording it are ONE operation, pgc_record, and every +# helper routes through it. The arms below hold that shape rather than the +# behaviour of any one helper, because the shape is what stops the next +# expect_fail from being written. +# --------------------------------------------------------------------------- + +_libsh="$PGC_TESTDIR/lib.sh" + +check "premise: lib.sh is where the check helpers live" \ + "$(grep -c '^check() {' "$_libsh")" "1" + +# THE STRUCTURAL ARM. One counter site, not eleven. +check "lib.sh bumps PGC_CHECKS in exactly one place" \ + "$(grep -c 'PGC_CHECKS=\$((PGC_CHECKS' "$_libsh")" "1" +check "and that place is pgc_record" \ + "$(sed -n '/^pgc_record()/,/^}/p' "$_libsh" | grep -c 'PGC_CHECKS=\$((PGC_CHECKS')" "1" + +# ---- the record line itself ------------------------------------------------- +# +# Tab separated, so a name containing spaces survives. Five columns after the +# RESULT marker: suite, part, name, verdict, reason. The reason carries phase 1's +# REASON_CODE, which is what makes this more than a reformat: an unrunnable check +# is distinguishable from a passing one without parsing prose. The verdict is one +# of PASS, FAIL, UNRUN or SKIP. +# +# No mutation column: that is the LEDGER's (#918). A record is one observation, +# not a history. + +_rec() { # _rec HELPER ARGS... -> the RESULT lines that helper emitted + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0 + "$@" 2>/dev/null | grep '^RESULT' ) +} +_human() { # _human HELPER ARGS... -> the human lines that helper emitted + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0 + "$@" 2>/dev/null | grep -v '^RESULT' ) +} + +check "a passing check emits exactly one record" \ + "$(_rec check "a name" x x | wc -l)" "1" +check "and its verdict field says PASS" \ + "$(_rec check "a name" x x | cut -f5)" "PASS" +check "and its name field is the check's name, spaces intact" \ + "$(_rec check "a name" x x | cut -f4)" "a name" + +# ---- and WHICH PART asked it ------------------------------------------------ +# +# The suite is not enough. harness_selftest sources 40-odd parts into one shell, +# and its premises are phrased to be COPIED: "premise: the pytest layer is where +# THIS PART thinks it is" says "this part" so the same sentence works in any of +# them. main carries two copies of that one and two of another, and the count +# grows with every part anyone adds -- OffgridwithJD found all six of their own +# branches adding more. +# +# So (suite, name) is not a key of checks, it is a key of check NAMES. The part +# makes it a key of the thing it identifies, and it is derived from BASH_SOURCE +# rather than from a convention, so the next part written the same way is keyed +# correctly without anyone remembering. +check "the record names the part the check was asked from" \ + "$(_rec check "a name" x x | cut -f3)" "$(basename "${BASH_SOURCE[0]}" .sh)" +check "premise: and that is this fragment, not the suite" \ + "$([ "$(_rec check "n" x x | cut -f3)" != "$(_rec check "n" x x | cut -f2)" ] \ + && echo different || echo same)" "different" + +check "a failing check emits exactly one record" \ + "$(_rec check "a name" x y | wc -l)" "1" +check "and its verdict field says FAIL" \ + "$(_rec check "a name" x y | cut -f5)" "FAIL" + +check "an unrunnable check emits exactly one record" \ + "$(_rec check_unrunnable "a name" MISSING_DEPENDENCY "no jq" | wc -l)" "1" +check "and its verdict field says UNRUN, which is neither of the other two" \ + "$(_rec check_unrunnable "a name" MISSING_DEPENDENCY "no jq" | cut -f5)" "UNRUN" +check "and the REASON_CODE travels in the reason field, not in prose" \ + "$(_rec check_unrunnable "a name" MISSING_DEPENDENCY "no jq" | cut -f6)" "MISSING_DEPENDENCY" + +# A reason code the enum does not contain is already a FAIL. It must record that +# verdict, not the one it was asked for. +check "a bogus reason code records FAIL, not UNRUN" \ + "$(_rec check_unrunnable "a name" NOT_A_REASON "x" | cut -f5)" "FAIL" + +# ---- every helper, not just the two that were easy -------------------------- +# +# check_text, check_num, check_ratio and pgc_require_tools each had their own +# counter bump and their own outcome line. Each is one place the pair could come +# apart, which is why the arm is over ALL of them rather than a sample. + +check "check_text on an empty side emits one record" \ + "$(_rec check_text "n" "" "x" | wc -l)" "1" +check "and records FAIL, because nothing was compared" \ + "$(_rec check_text "n" "" "x" | cut -f5)" "FAIL" +check "check_num on a non-number emits one record" \ + "$(_rec check_num "n" "abc" "1" | wc -l)" "1" +check "and records FAIL" "$(_rec check_num "n" "abc" "1" | cut -f5)" "FAIL" +check "check_ratio on a non-number emits one record" \ + "$(_rec check_ratio "n" "abc" "1" "2" | wc -l)" "1" +check "check_ratio with a zero side emits one record" \ + "$(_rec check_ratio "n" "0" "1" "2" | wc -l)" "1" +check "check_ratio that forms a ratio emits one record" \ + "$(_rec check_ratio "n" "1" "1" "2" | wc -l)" "1" +check "and records PASS when the ratio is inside the bound" \ + "$(_rec check_ratio "n" "1" "1" "2" | cut -f5)" "PASS" +check "pgc_pass emits one record" "$(_rec pgc_pass "n" | wc -l)" "1" +check "pgc_fail emits one record" "$(_rec pgc_fail "n" "d" | wc -l)" "1" + +# ---- the human lines must not have changed ---------------------------------- +# +# 3,762 check sites, and suites, selftests and CI all grep `^FAIL` and `^PASS`. +# Adding a record beside them is only safe if the prose is byte-identical, so the +# arms pin the exact strings rather than trusting that a refactor was careful. + +check "a passing check still prints its old line" \ + "$(_human check "a name" x x)" "PASS a name" +check "a failing check still prints its old line" \ + "$(_human check "a name" x y)" "FAIL a name: got [x] want [y]" +check "an unrunnable check still prints its old line" \ + "$(_human check_unrunnable "a name" MISSING_DEPENDENCY "no jq")" "UNRUN a name: MISSING_DEPENDENCY: no jq" +check "check_text's empty-side line is unchanged" \ + "$(_human check_text "n" "" "x")" "FAIL n: a side is empty, so nothing was compared: got [] want [x]" +check "check_num's non-measurement line is unchanged" \ + "$(_human check_num "n" "abc" "1")" "FAIL n: not a measurement, so nothing was compared: got [abc] want [1]" + +# ---- the count and the records cannot come apart ---------------------------- +# +# They are one operation, so this cannot fail by drifting. It CAN fail if a +# helper is added that prints an outcome without recording it, which is exactly +# the expect_fail shape, so it is asserted rather than argued. + +_n_calls=7 +_recorded="$( ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0 + check a x x; check b x y; check_text c "" x; check_num d abc 1 + check_ratio e 1 1 2; pgc_pass f; check_unrunnable g MISSING_DEPENDENCY h + echo "COUNTED $PGC_CHECKS" ) )" +check "premise: the probe ran every helper shape once" \ + "$(printf '%s\n' "$_recorded" | grep -c '^RESULT')" "$_n_calls" +check "the record count equals the counter the summary reports" \ + "$(printf '%s\n' "$_recorded" | grep -c '^RESULT')" \ + "$(printf '%s\n' "$_recorded" | sed -n 's/^COUNTED //p')" + +# ---- and the RUNNER must reconcile them ------------------------------------- +# +# A suite's log states `checks run: N` and carries N record lines. Those are two +# artifacts of the same run and they can genuinely disagree: a suite killed +# mid-way, a truncated log, a helper that prints an outcome without recording it. + +_rv="$PGC_TESTDIR/run_all_versions.sh" +check "the runner defines the record reconciliation" \ + "$(grep -c '^pgc_reconcile_records()' "$_rv")" "1" + +eval "$(sed -n '/^pgc_reconcile_records()/,/^}/p' "$_rv")" +check "premise: it is callable" "$(type -t pgc_reconcile_records)" "function" + +_rl="$PGC_WORKDIR/rec.log" +printf 'RESULT\ts\tp\ta\tPASS\t\nRESULT\ts\tp\tb\tPASS\t\nchecks run: 2\n' > "$_rl" +check "a log whose records match its stated count reconciles" \ + "$(pgc_reconcile_records "$_rl" >/dev/null 2>&1 && echo ok || echo mismatch)" "ok" + +printf 'RESULT\ts\tp\ta\tPASS\t\nchecks run: 2\n' > "$_rl" +check "a log with fewer records than it claims is caught" \ + "$(pgc_reconcile_records "$_rl" >/dev/null 2>&1 && echo ok || echo mismatch)" "mismatch" +check "and the two numbers are named, not just the verdict" \ + "$(pgc_reconcile_records "$_rl" 2>&1 | grep -c 'records=1 .*checks run: 2')" "1" + +printf 'RESULT\ts\tp\ta\tPASS\t\nRESULT\ts\tp\tb\tPASS\t\nRESULT\ts\tp\tc\tPASS\t\nchecks run: 2\n' > "$_rl" +check "a log with more records than it claims is caught too" \ + "$(pgc_reconcile_records "$_rl" >/dev/null 2>&1 && echo ok || echo mismatch)" "mismatch" + +# A log with no `checks run:` line at all did not reach its summary. That is a +# different fault from a miscount and must not read as a clean reconciliation. +printf 'RESULT\ts\tp\ta\tPASS\t\n' > "$_rl" +check "a log that never stated a count is not silently accepted" \ + "$(pgc_reconcile_records "$_rl" >/dev/null 2>&1 && echo ok || echo mismatch)" "mismatch" + +check "the runner calls the record reconciliation, not merely defines it" \ + "$(grep -c '[^_[:alnum:]]pgc_reconcile_records "' "$_rv")" "1" + +# THE COUNT IS NOT THE SCHEMA. Counting `^RESULT` lines and comparing to +# `checks run:` accepted every malformed record below, each against a well-formed +# control that reconciled the same way -- so the function returned 0 whether the +# record parsed or not. Named by @linuxhikerpm on #917. +# +# The control comes FIRST and is asserted, because five arms that all say +# "mismatch" prove nothing if the function has simply started refusing +# everything. That is the shape this suite exists to catch. +printf 'RESULT\ts\tp\ta\tPASS\t\nchecks run: 1\n' > "$_rl" +check "control: a well-formed record still reconciles" \ + "$(pgc_reconcile_records "$_rl" >/dev/null 2>&1 && echo ok || echo mismatch)" "ok" + +_rq_bad() { # _rq_bad RECORD -> ok|mismatch, with the count always matching + printf '%s\nchecks run: 1\n' "$1" > "$_rl" + pgc_reconcile_records "$_rl" >/dev/null 2>&1 && echo ok || echo mismatch +} + +check "a record missing fields does not reconcile" \ + "$(_rq_bad "$(printf 'RESULT\ts\tp\ta')")" "mismatch" + +check "a record carrying extra fields does not reconcile" \ + "$(_rq_bad "$(printf 'RESULT\ts\tp\tn\tPASS\tr\textra\tmore')")" "mismatch" + +check "a verdict pgc_record cannot emit does not reconcile" \ + "$(_rq_bad "$(printf 'RESULT\ts\tp\tn\tBOGUS\t')")" "mismatch" + +check "an empty check name does not reconcile" \ + "$(_rq_bad "$(printf 'RESULT\ts\tp\t\tPASS\t')")" "mismatch" + +# The four verdicts are the emitter's own list. If pgc_record grows a fifth and +# this one does not, this arm goes red rather than the vocabulary drifting. +for _rq_v in PASS FAIL UNRUN SKIP; do + check "the reconciliation accepts the verdict $_rq_v, which pgc_record emits" \ + "$(_rq_bad "$(printf 'RESULT\ts\tp\tn\t%s\t' "$_rq_v")")" "ok" +done +unset -f _rq_bad +unset _rq_v + +# ---- a named SKIP is an outcome, so it is counted and recorded -------------- +# +# `echo "SKIP ..."` printed a line a reader sees and left PGC_CHECKS alone, so +# 22 sites across 20 files reported an outcome that no count and no record ever +# saw. The tree's own comment said why that mattered -- "the skip must be +# visible: a check that reports nothing is indistinguishable from a check that +# passes" -- and it was true while the human line WAS the record. +# +# THE EXEMPTION IS DERIVED, NOT A FILENAME LIST. A file that calls `check` is a +# suite or a part, and its skips are check outcomes. run_all_versions.sh calls +# `check` zero times because it is the runner: its one `echo "SKIP"` declines a +# whole PostgreSQL major before any suite exists, so there is no PGC_CHECKS for +# it to belong to. That distinction is read off the files rather than written +# here, so a new runner or a new suite is classified without editing this arm. +_sk_offenders="" +for _sk_f in "$PGC_TESTDIR"/*.sh "$PGC_TESTDIR"/selftest/*.sh; do + [ -e "$_sk_f" ] || continue + # A comment is not a statement, so the pattern anchors to the line start + # after whitespace only. Measured: this arm's own prose above says + # echo "SKIP" and is not matched, which a looser pattern would flag. + [ "$(grep -cE '^[[:space:]]*check(_[a-z_]+)? ' "$_sk_f")" -gt 0 ] || continue + if [ "$(grep -cE '^[[:space:]]*echo "SKIP' "$_sk_f")" -gt 0 ]; then + _sk_offenders="$_sk_offenders ${_sk_f##*/}" + fi +done +check "no file that calls check prints a SKIP outcome the count cannot see" \ + "$(printf '%s' "$_sk_offenders" | wc -w | tr -d ' ')" "0" +[ -z "$_sk_offenders" ] || printf ' %s\n' $_sk_offenders + +# The sweep has to be looking at something, and it has to be able to find one. +_sk_seen=0 +for _sk_f in "$PGC_TESTDIR"/*.sh "$PGC_TESTDIR"/selftest/*.sh; do + [ -e "$_sk_f" ] || continue + [ "$(grep -cE '^[[:space:]]*check(_[a-z_]+)? ' "$_sk_f")" -gt 0 ] && _sk_seen=$((_sk_seen + 1)) +done +check "premise: the sweep classified a corpus of check-calling files" \ + "$([ "$_sk_seen" -ge 50 ] && echo yes || echo "no ($_sk_seen)")" "yes" + +_sk_fix="$PGC_WORKDIR/skipsweep"; rm -rf "$_sk_fix"; mkdir -p "$_sk_fix" +printf 'check "x" a a\necho "SKIP a bare skip"\n' > "$_sk_fix/offender.sh" +check "premise: and it would name a file that calls check and echoes a SKIP" \ + "$([ "$(grep -cE '^[[:space:]]*check(_[a-z_]+)? ' "$_sk_fix/offender.sh")" -gt 0 ] \ + && [ "$(grep -cE '^[[:space:]]*echo "SKIP' "$_sk_fix/offender.sh")" -gt 0 ] \ + && echo caught || echo missed)" "caught" + +printf 'echo "SKIP a bare skip"\n' > "$_sk_fix/runner.sh" +check "premise: while a file that calls no check is not its business" \ + "$([ "$(grep -cE '^[[:space:]]*check(_[a-z_]+)? ' "$_sk_fix/runner.sh")" -gt 0 ] \ + && echo caught || echo "not a suite")" "not a suite" + +unset _sk_offenders _sk_f _sk_seen _sk_fix + +# ---- a check_skip must not read a name its own file never assigns ---------- +# +# I SHIPPED THIS DEFECT AND THIS ARM IS WHY IT CANNOT COME BACK. Converting the +# skips, I wrote `check_skip "the amcheck oracle for $idx"` into +# native_index_projection.sh. The loop variable is `ix`, in the OTHER branch, and +# `$idx` appeared nowhere else in the file. Every suite runs under `set -u`, so on +# any box without amcheck the else branch died with "idx: unbound variable" at +# rc=1, before pgc_summary and before any `checks run:` line. CI never saw it: the +# PGDG packages carry amcheck, so CI always takes the then branch. Found by +# @OffgridwithJD on a container without it. +# +# `bash -n` cannot see this -- the syntax is fine -- and neither can any arm that +# only runs the branch CI happens to take. So the check is static and reads the +# text: every name a check_skip line expands must be assigned somewhere in that +# same file, or be one of the harness globals lib.sh exports. +_us_globals=" PGC_MAJOR PGC_SUITE PGC_DB PGC_PORT PGC_BINDIR PGC_TESTDIR PGC_WORKDIR PGC_SRCDIR PGC_ALLOW_MISSING " +_us_unbound() { # _us_unbound FILE -> lines naming a variable the file never assigns + local _f="$1" _line _v _n + grep -nE '^[[:space:]]*check_skip ' "$_f" 2>/dev/null | while IFS= read -r _line; do + _n="${_line%%:*}" + for _v in $(printf '%s' "${_line#*:}" | grep -oE '\$\{?[A-Za-z_][A-Za-z0-9_]*' | tr -d '${'); do + case "$_us_globals" in *" $_v "*) continue ;; esac + grep -qE "(^|[[:space:]]|;)$_v=" "$_f" && continue + # BRACES, because `$_v[` reads as an array expansion: shellcheck + # SC1087 at -S error, which is the CI gate for this harness. It is + # lint-only -- bash cannot take `[` as part of a name, so both + # spellings expand identically and this arm worked before the fix -- + # but it is the tidiest example of what the arm is for: the guard + # against a variable-expansion mistake carried one, in the very + # regex that looks for the for-loop assigning the name. Noticed by + # @OffgridwithJD, who also measured that the two expansions match. + grep -qE "for[[:space:]]+${_v}[[:space:]]+in[[:space:]]" "$_f" && continue + grep -qE "local[[:space:]][^#]*\b$_v\b" "$_f" && continue + printf '%s:%s:%s\n' "${_f##*/}" "$_n" "$_v" + done + done +} + +_us_bad="" +for _us_f in "$PGC_TESTDIR"/*.sh "$PGC_TESTDIR"/selftest/*.sh; do + [ -e "$_us_f" ] || continue + _us_bad="$_us_bad$(_us_unbound "$_us_f")" +done +check "no check_skip reads a name its own file never assigns" \ + "$(printf '%s' "$_us_bad" | grep -c . || true)" "0" +[ -z "$_us_bad" ] || printf ' %s\n' $_us_bad + +# The sweep must be able to find one, and must not flag a name that IS assigned. +_us_fix="$PGC_WORKDIR/unbound"; rm -rf "$_us_fix"; mkdir -p "$_us_fix" +printf 'check_skip "the oracle for $idx" "SKIP x" "y"\n' > "$_us_fix/bad.sh" +check "premise: it names a check_skip reading an unassigned variable" \ + "$(_us_unbound "$_us_fix/bad.sh" | grep -c . || true)" "1" + +printf 'for ix in a b; do\ncheck_skip "the oracle for $ix" "SKIP x" "y"\ndone\n' > "$_us_fix/good.sh" +check "and leaves one whose variable is the loop it sits in" \ + "$(_us_unbound "$_us_fix/good.sh" | grep -c . || true)" "0" + +printf 'idx=w_k\ncheck_skip "the oracle for $idx" "SKIP x" "y"\n' > "$_us_fix/assigned.sh" +check "and leaves one whose variable is assigned earlier" \ + "$(_us_unbound "$_us_fix/assigned.sh" | grep -c . || true)" "0" + +unset -f _us_unbound +unset _us_bad _us_f _us_fix _us_globals + +# ---- the timing helpers reported an outcome that nothing counted ------------- +# +# check_timing and check_ratio_needs_quiet_machine, under PGC_SKIP_TIMING=1, +# printed a human SKIP line and returned. Driven before the fix: +# +# SKIP a timing check (PGC_SKIP_TIMING: wall-clock measurement) +# SKIP a ratio check (PGC_SKIP_TIMING: wall-clock ratio) +# -> PGC_CHECKS=0 PGC_PASSED=0 PGC_FAILED=0 PGC_UNRUN=0 +# +# Two outcomes a reader sees, nothing counted, no record. That is the hole this +# part's whole argument cannot have, and NOTHING here reached those branches: +# removing both emitters left every arm above green. Found by @linuxhikerpm. +# +# SKIP IS A FOURTH OUTCOME, counted like the other three. `checks run: N` now +# reports the checks a suite ENCOUNTERED rather than the ones it managed to +# evaluate, and pgc_summary reconciles four counters against it instead of three +# -- the same shape, one term wider. +# +# It is NOT check_unrunnable. That third state exists for a check whose INPUT was +# absent and it exits the suite INCOMPLETE, which would turn every CI run red the +# moment PGC_SKIP_TIMING is set -- and CI sets it on every run. A wall-clock check +# on a shared runner is deliberately not asked, which is a different thing from a +# check that could not be answered. + +_tm() { # _tm SKIPFLAG HELPER ARGS... -> records emitted + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0; PGC_SKIPPED=0 + PGC_SKIP_TIMING="$1"; shift; "$@" 2>/dev/null | grep '^RESULT' ) +} +_tmh() { # _tmh SKIPFLAG HELPER ARGS... -> the human lines + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0; PGC_SKIPPED=0 + PGC_SKIP_TIMING="$1"; shift; "$@" 2>/dev/null | grep -v '^RESULT' ) +} +_tmc() { # _tmc SKIPFLAG HELPER ARGS... -> "CHECKS/PASSED/SKIPPED" + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0; PGC_SKIPPED=0 + PGC_SKIP_TIMING="$1"; shift; "$@" >/dev/null 2>&1 + echo "$PGC_CHECKS/$PGC_PASSED/$PGC_SKIPPED" ) +} + +check "a skipped timing check emits exactly one record" \ + "$(_tm 1 check_timing "a timing check" 1 1 | wc -l)" "1" +check "and its verdict is SKIP" \ + "$(_tm 1 check_timing "a timing check" 1 1 | cut -f5)" "SKIP" +check "and it is counted, so checks run: reports it" \ + "$(_tmc 1 check_timing "a timing check" 1 1)" "1/0/1" +check "and its human line is unchanged" \ + "$(_tmh 1 check_timing "a timing check" 1 1)" \ + "SKIP a timing check (PGC_SKIP_TIMING: wall-clock measurement)" + +check "the same timing check, ENABLED, emits one record and passes" \ + "$(_tm 0 check_timing "a timing check" 1 1 | cut -f5)" "PASS" +check "and is counted as a pass, not a skip" \ + "$(_tmc 0 check_timing "a timing check" 1 1)" "1/1/0" + +check "a skipped ratio check emits exactly one record" \ + "$(_tm 1 check_ratio_needs_quiet_machine "a ratio check" 1 1 2 | wc -l)" "1" +check "and its verdict is SKIP" \ + "$(_tm 1 check_ratio_needs_quiet_machine "a ratio check" 1 1 2 | cut -f5)" "SKIP" +check "and it is counted" \ + "$(_tmc 1 check_ratio_needs_quiet_machine "a ratio check" 1 1 2)" "1/0/1" +check "and its human line is unchanged" \ + "$(_tmh 1 check_ratio_needs_quiet_machine "a ratio check" 1 1 2)" \ + "SKIP a ratio check (PGC_SKIP_TIMING: wall-clock ratio)" + +check "the same ratio check, ENABLED, emits one record and passes" \ + "$(_tm 0 check_ratio_needs_quiet_machine "a ratio check" 1 1 2 | cut -f5)" "PASS" +check "and is counted as a pass, not a skip" \ + "$(_tmc 0 check_ratio_needs_quiet_machine "a ratio check" 1 1 2)" "1/1/0" + +# ---- and the accounting line carries the fourth term ------------------------ + +_acct_line() { # _acct_line SKIPFLAG -> pgc_summary's accounting line + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0; PGC_SKIPPED=0 + PGC_SKIP_TIMING="$1" + check "an ordinary check" x x + check_timing "a timing check" 1 1 + pgc_summary ) 2>/dev/null | sed -n 's/^\(accounting: .*\)$/\1/p' +} +check "the accounting line reconciles four outcomes against the count" \ + "$(_acct_line 1)" "accounting: 1 passed + 0 failed + 0 unrunnable + 1 skipped = 2" +check "and with timing enabled the skipped term is zero, not absent" \ + "$(_acct_line 0)" "accounting: 2 passed + 0 failed + 0 unrunnable + 0 skipped = 2" + +# A suite whose every check was skipped has evaluated nothing, so it must not +# report PASSED. Before the fourth counter it could not reach this state at all, +# because a skipped check left PGC_CHECKS at zero. +_allskip() { + ( PGC_FAIL=0; PGC_CHECKS=0; PGC_PASSED=0; PGC_FAILED=0; PGC_UNRUN=0; PGC_SKIPPED=0 + PGC_SKIP_TIMING=1 + check_timing "a timing check" 1 1 + pgc_summary ) 2>/dev/null | grep -oE ': (PASSED|FAILED|SKIPPED \(ran no checks\)|INCOMPLETE)$' +} +check "a suite that skipped every check did not pass" \ + "$(_allskip)" ": SKIPPED (ran no checks)" + +# ---- a check inside a pipeline loses its count, and the message must say so -- +# +# `printf ... | while read n; do check "$n" a a; done` runs the loop body in a +# SUBSHELL, so the counter bump dies with it while the outcome and the record are +# both printed to the parent's stdout. Driven: +# +# four checks print PASS, four RESULT lines appear, PGC_CHECKS=2 +# +# pgc_reconcile_records catches it -- that is what it is for -- but it reported +# `records=3 but the log states checks run: 1`, which is the BOOKKEEPING rather +# than the cause. A reader who has not met this before has no way from that line +# to the pipeline. Raised by OffgridwithJD. +# +# Latent today: four piped loops in the tree, none with a check inside. So the +# sweep below reports zero, and a fixture proves it can fire -- a rule whose only +# evidence is that the corpus is currently clean is not a rule. + +eval "$(sed -n '/^pgc_reconcile_records()/,/^}/p' "$_rv")" +_pl="$PGC_WORKDIR/piped.log" +printf 'RESULT\ts\tp\ta\tPASS\t\nRESULT\ts\tp\tb\tPASS\t\nRESULT\ts\tp\tc\tPASS\t\nchecks run: 1\n' > "$_pl" +check "more records than counted checks names the cause, not just the arithmetic" \ + "$(pgc_reconcile_records "$_pl" 2>&1 | grep -c 'a check ran in a subshell')" "1" +check "and still reports the two numbers" \ + "$(pgc_reconcile_records "$_pl" 2>&1 | grep -c 'records=3 .*checks run: 1')" "1" + +# Fewer records than checks is the OPPOSITE fault -- a counted check that emitted +# no record -- and must not be described as a subshell. +printf 'RESULT\ts\tp\ta\tPASS\t\nchecks run: 3\n' > "$_pl" +check "fewer records than counted checks is not described as a subshell" \ + "$(pgc_reconcile_records "$_pl" 2>&1 | grep -c 'a check ran in a subshell')" "0" +check "and names its own cause instead" \ + "$(pgc_reconcile_records "$_pl" 2>&1 | grep -c 'counted without emitting a record')" "1" + +# ---- and the shape is swept, the way selftest 080 sweeps its cousin ---------- + +_pipeloop_sites() { # _pipeloop_sites FILE... -> file:line of a check inside a piped loop + # Two refinements, both from measuring rather than reading. Requiring the + # closing `done` to be alone on its line left the scanner inside a loop for + # the rest of any file whose loop ended `done)"` -- 27 hits against a true + # zero. And a loop written entirely on ONE line, inside a command + # substitution, opened a block that never closed, flagging every later check. + # + # So: open only on a line that opens the loop and does NOT close it, and close + # on a `done` token wherever it sits. Comments are stripped first, because the + # rule's own explanation necessarily spells the shape out. + awk ' + FNR == 1 { inloop = 0 } + { line = $0; sub(/^[[:space:]]*#.*/, "", line) } + line ~ /\|[[:space:]]*(while|for)[[:space:]]/ && line ~ /(^|[[:space:];])do([[:space:]]|$)/ \ + && line !~ /(^|[[:space:]();])done([[:space:]();]|$)/ { inloop = 1; next } + inloop && line ~ /(^|[[:space:]();])done([[:space:]();]|$)/ { inloop = 0; next } + inloop && line ~ /(^|[^_[:alnum:]])(check|check_num|check_text|check_ratio|check_unrunnable|pgc_pass|pgc_fail)[[:space:]]/ { + print FILENAME ":" FNR + } + ' "$@" +} + +_pl_fx="$PGC_WORKDIR/plfx"; mkdir -p "$_pl_fx" +_pl_call="$(printf '%s "$n" a a' check)" +{ printf 'printf "a\\nb\\n" | while IFS= read -r n; do\n'; printf '\t%s\n' "$_pl_call"; printf 'done\n'; } \ + > "$_pl_fx/bad.sh" +{ printf 'while IFS= read -r n; do\n'; printf '\t%s\n' "$_pl_call"; printf 'done < <(printf "a\\nb\\n")\n'; } \ + > "$_pl_fx/good.sh" +{ printf 'printf "a\\nb\\n" | while IFS= read -r n; do\n'; printf '\techo "$n"\n'; printf 'done\n'; } \ + > "$_pl_fx/nocheck.sh" + +check "premise: the fixtures carry the shapes this sweep is about" \ + "$(grep -lc 'while IFS= read' "$_pl_fx"/*.sh | grep -c .)" "3" +check "the sweep finds a check inside a PIPED loop" \ + "$(_pipeloop_sites "$_pl_fx/bad.sh" | grep -c .)" "1" +check "and not one inside a process-substitution loop, which keeps its shell" \ + "$(_pipeloop_sites "$_pl_fx/good.sh" | grep -c .)" "0" +check "and not a piped loop with no check in it" \ + "$(_pipeloop_sites "$_pl_fx/nocheck.sh" | grep -c .)" "0" + +# A loop written entirely on one line inside a command substitution opens and +# closes in the same place. The first version of this sweep opened a block there +# and never closed it, flagging every check after it -- which is how twelve of +# its twenty-seven false hits were in this very file. +{ printf 'x="$(printf "a\\n" | while IFS= read -r n; do echo "$n"; done)"\n' + printf '%s\n' "$_pl_call"; } > "$_pl_fx/oneline.sh" +check "and not a check after a one-line piped loop that already closed" \ + "$(_pipeloop_sites "$_pl_fx/oneline.sh" | grep -c .)" "0" + +_pl_hits="$(_pipeloop_sites "$PGC_TESTDIR"/*.sh "$PGC_TESTDIR"/selftest/*.sh 2>/dev/null | grep -c . || true)" +[ "${_pl_hits:-0}" = 0 ] || _pipeloop_sites "$PGC_TESTDIR"/*.sh "$PGC_TESTDIR"/selftest/*.sh | sed 's/^/ /' +check "no suite calls a check inside a piped loop" "${_pl_hits:-0}" "0" diff --git a/test/selftest/410-a-check-must-have-been-red.sh b/test/selftest/410-a-check-must-have-been-red.sh new file mode 100644 index 00000000..f5e3fd27 --- /dev/null +++ b/test/selftest/410-a-check-must-have-been-red.sh @@ -0,0 +1,650 @@ +# ---- a check must have been seen red, or be counted as debt ----------------- +# +# Nothing recorded whether a check had ever been red. That is the gap that let 39 +# checks across 35 suites ship unable to fail, three of them inside this very +# suite. The gate answered "did anything print FAIL" and had never answered +# "could anything print FAIL". +# +# WHAT THIS RECORDS, AND WHAT IT DOES NOT. It records that a named check WAS +# OBSERVED RED in a recorded run. It does NOT claim the check is proven able to +# fail: that needs a named mutation applied deliberately, and conflating the two +# would put a claim in the ledger that nothing measured. +# +# THE FIRST DESIGN DEADLOCKED AND THE SECOND DOES NOT. Bounding +# `checks_never_observed_red` means every added check breaks the gate, because a +# new check enters as `never` -- so the only way to land one was to raise a number +# the design said may only fall. It shipped at 614 rows, 614 never, ceiling 614. +# It is now a CENSUS, asserted to match the ledger; the CEILING is +# `suites_not_covered`, which adding a check does not move. +# +# WHAT THE GATE REFUSES is a check the committed ledger has never seen. Existing +# checks are grandfathered; a new one is named, and regenerating the ledger is the +# INTENDED fix rather than a forbidden edit. +# --------------------------------------------------------------------------- + +_led="$PGC_TESTDIR/pgc_ledger.py" +_ledger="$PGC_TESTDIR/check_ledger.tsv" +_budget="$PGC_TESTDIR/check_ledger_budget.txt" + +check "premise: the ledger tool exists" "$([ -f "$_led" ] && echo yes || echo no)" "yes" +check "premise: the ledger itself is a tracked file, not a variable" \ + "$([ -f "$_ledger" ] && echo yes || echo no)" "yes" +check "premise: the budget is a tracked file too" \ + "$([ -f "$_budget" ] && echo yes || echo no)" "yes" + +_lw="$PGC_WORKDIR/ledger"; mkdir -p "$_lw" +_led_run() { python3 "$_led" "$@" 2>&1; } +_led_rc() { python3 "$_led" "$@" >/dev/null 2>&1; echo $?; } + +printf 'RESULT\tdemo\tpart1\tfirst check\tPASS\t\nRESULT\tdemo\tpart1\tsecond check\tPASS\t\nchecks run: 2\n' > "$_lw/green.log" +printf 'RESULT\tdemo\tpart1\tfirst check\tFAIL\t\nRESULT\tdemo\tpart1\tsecond check\tPASS\t\nchecks run: 2\n' > "$_lw/red.log" +printf 'demo\n' > "$_lw/registered" +printf 'suites_not_covered 0\n' > "$_lw/budget.txt" + +# ---- fail closed. Every one of these returned rc=0 before ------------------- +# +# read_records ignored unreadable files, empty ones and short records, so a gate +# over a NONEXISTENT log reported success. An integrity failure that reads as a +# clean run is worse than no gate, because it certifies. Reported by @linuxhikerpm. + +: > "$_lw/empty.log" +printf 'RESULT\tdemo\tpart1\tname\n' > "$_lw/short.log" +: > "$_lw/l.tsv" +check "a gate over a nonexistent log is an integrity failure, not a pass" \ + "$(_led_rc gate --ledger "$_lw/l.tsv" --budget "$_lw/budget.txt" --registered "$_lw/registered" "$_lw/nope.log")" "2" +check "an empty log is one too, because there is nothing to reconcile" \ + "$(_led_rc gate --ledger "$_lw/l.tsv" --budget "$_lw/budget.txt" --registered "$_lw/registered" "$_lw/empty.log")" "2" +check "and a record missing its verdict" \ + "$(_led_rc gate --ledger "$_lw/l.tsv" --budget "$_lw/budget.txt" --registered "$_lw/registered" "$_lw/short.log")" "2" +check "each says what was wrong with the input" \ + "$(_led_run gate --ledger "$_lw/l.tsv" --budget "$_lw/budget.txt" --registered "$_lw/registered" "$_lw/short.log" \ + | grep -c 'a record has 5 fields')" "1" + +# ---- the count is not the schema, on INGESTION as well as in the runner ------ +# +# read_records accepted `len(f) >= 5`, so a record missing its reason, a verdict +# outside pgc_record's vocabulary, an empty check name, and one record against +# `checks run: 2` all merged at rc=0 -- the ledger absorbing as evidence a log +# that does not parse. The runner reconciles the suite it just ran; this +# reconciles a log handed to the ledger, possibly from another machine. +# Reported by @linuxhikerpm on #918. +printf 'RESULT\tdemo\tpart1\ta name\tBOGUS\t\nchecks run: 1\n' > "$_lw/bogus.log" +printf 'RESULT\tdemo\tpart1\t\tPASS\t\nchecks run: 1\n' > "$_lw/noname.log" +printf 'RESULT\tdemo\tpart1\ta name\tPASS\t\nchecks run: 2\n' > "$_lw/miscount.log" + +check "a verdict pgc_record cannot emit is an integrity failure" \ + "$(_led_rc merge --ledger "$_lw/v.tsv" --date 2026-09-10 "$_lw/bogus.log")" "2" +check "and it names the verdict, so the author knows which record" \ + "$(_led_run merge --ledger "$_lw/v.tsv" --date 2026-09-10 "$_lw/bogus.log" | grep -c "'BOGUS'")" "1" +check "an empty check name is one too, because it names no check" \ + "$(_led_rc merge --ledger "$_lw/v.tsv" --date 2026-09-10 "$_lw/noname.log")" "2" +check "and a log that does not reconcile with its own checks run:" \ + "$(_led_rc merge --ledger "$_lw/v.tsv" --date 2026-09-10 "$_lw/miscount.log")" "2" + +# THE CONTROL, because four arms all saying 2 prove nothing if the tool has +# started refusing everything. +check "control: a well-formed log still merges" \ + "$(_led_rc merge --ledger "$_lw/v.tsv" --date 2026-09-10 "$_lw/green.log")" "0" + +# ---- last red may only move forward ----------------------------------------- +# +# It was a plain assignment: merging an older log rewrote a recent observation +# with an older one, and merging an undated log replaced a real date with +# `unknown`. A free-form --date was accepted verbatim, so a typo became an +# observation date the ledger treated as authoritative. +: > "$_lw/date.tsv" +_led_run merge --ledger "$_lw/date.tsv" --date 2026-09-10 "$_lw/red.log" >/dev/null +_led_run merge --ledger "$_lw/date.tsv" --date 2026-09-01 "$_lw/red.log" >/dev/null +check "an older observation does not overwrite a newer one" \ + "$(awk -F'\t' '$3=="first check"{print $4}' "$_lw/date.tsv")" "2026-09-10" +_led_run merge --ledger "$_lw/date.tsv" --date 2026-09-20 "$_lw/red.log" >/dev/null +check "and a newer one does" \ + "$(awk -F'\t' '$3=="first check"{print $4}' "$_lw/date.tsv")" "2026-09-20" +_led_run merge --ledger "$_lw/date.tsv" "$_lw/red.log" >/dev/null +check "and an undated merge does not erase a known date" \ + "$(awk -F'\t' '$3=="first check"{print $4}' "$_lw/date.tsv")" "2026-09-20" +check "a date that is not a date is refused rather than stored" \ + "$(_led_rc merge --ledger "$_lw/date.tsv" --date not-a-date "$_lw/red.log")" "2" + +# ---- a mutation names ONE check --------------------------------------------- +# +# A run that mutates one thing can redden several: the target, plus whatever +# depended on it. Attributing --mutation to every failure records collateral +# damage as evidence that the mutation kills that check. +printf 'RESULT\tdemo\tpart1\tthe target\tFAIL\t\nRESULT\tdemo\tpart1\tcollateral\tFAIL\t\nchecks run: 2\n' \ + > "$_lw/twofail.log" +check "--mutation across two failing checks in one run is refused" \ + "$(_led_rc merge --ledger "$_lw/m2.tsv" --date 2026-09-10 --mutation M "$_lw/twofail.log")" "2" +check "and the refusal names how many failed, so the author can narrow the run" \ + "$(_led_run merge --ledger "$_lw/m2.tsv" --date 2026-09-10 --mutation M "$_lw/twofail.log" \ + | grep -c '2 checks failed')" "1" +check "control: the same log merges without --mutation" \ + "$(_led_rc merge --ledger "$_lw/m2.tsv" --date 2026-09-10 "$_lw/twofail.log")" "0" + +# The three must be distinguishable from a REAL refusal, or fail-closed just +# renames every outcome. +check "a real refusal is a different status from an integrity failure" \ + "$(_led_rc gate --ledger "$_lw/l.tsv" --budget "$_lw/budget.txt" --registered "$_lw/registered" "$_lw/green.log")" "1" + +# --registered is required. Skipping it silently is how a gate reports success +# for a question it never asked. +check "the gate refuses to run without the registered suite list" \ + "$(_led_rc gate --ledger "$_lw/l.tsv" --budget "$_lw/budget.txt" "$_lw/green.log")" "2" + +# ---- the census: a green run records debt and never a red observation ------- + +: > "$_lw/ledger.tsv" +_led_run merge --ledger "$_lw/ledger.tsv" --date 2026-09-10 "$_lw/green.log" >/dev/null +check "merging a green run records both checks" "$(grep -c . "$_lw/ledger.tsv")" "2" +check "and records neither as ever having been red" \ + "$(cut -f4 "$_lw/ledger.tsv" | sort -u | tr '\n' ' ')" "never " +check "every row has five fields and no trailing tab" \ + "$(awk -F'\t' 'NF!=5' "$_lw/ledger.tsv" | grep -c . || true)" "0" +check "and an empty mutation is a placeholder, not an empty last field" \ + "$(grep -cP '\t$' "$_lw/ledger.tsv" || true)" "0" + +_led_run merge --ledger "$_lw/ledger.tsv" --date 2026-09-10 "$_lw/red.log" >/dev/null +check "a check observed red gains the date it was seen" \ + "$(awk -F'\t' '$3=="first check"{print $4}' "$_lw/ledger.tsv")" "2026-09-10" +check "and one that stayed green keeps its debt" \ + "$(awk -F'\t' '$3=="second check"{print $4}' "$_lw/ledger.tsv")" "never" +_led_run merge --ledger "$_lw/ledger.tsv" --date 2026-09-11 "$_lw/green.log" >/dev/null +check "a later green run does not erase an observation" \ + "$(awk -F'\t' '$3=="first check"{print $4}' "$_lw/ledger.tsv")" "2026-09-10" + +# ---- the mutation column ACCUMULATES ---------------------------------------- +# +# Last-write-wins records the most recent attack rather than the catalogue the +# column exists to become, which defeats its stated purpose rather than limiting +# it. And one --mutation value copied across several logs attributes a deliberate +# change to failures it had nothing to do with. Both reported by @linuxhikerpm. + +: > "$_lw/mut.tsv" +_led_run merge --ledger "$_lw/mut.tsv" --date 2026-09-10 --mutation 'SAOP limit 128 -> 0' "$_lw/red.log" >/dev/null +check "a named mutation is recorded against the check that reddened" \ + "$(awk -F'\t' '$3=="first check"{print $5}' "$_lw/mut.tsv")" "SAOP limit 128 -> 0" +check "and not against one that stayed green" \ + "$(awk -F'\t' '$3=="second check"{print $5}' "$_lw/mut.tsv")" "-" +_led_run merge --ledger "$_lw/mut.tsv" --date 2026-09-10 --mutation 'bloom neutered' "$_lw/red.log" >/dev/null +check "a second mutation ACCUMULATES rather than replacing the first" \ + "$(awk -F'\t' '$3=="first check"{print $5}' "$_lw/mut.tsv")" "SAOP limit 128 -> 0;bloom neutered" +check "one --mutation cannot be attributed across several runs at once" \ + "$(_led_rc merge --ledger "$_lw/mut.tsv" --date 2026-09-10 --mutation X "$_lw/red.log" "$_lw/green.log")" "2" + +# ---- two runs of a check are not a duplicate of it -------------------------- +# +# Merging the logs first cannot tell "the same check in two runs" from "the same +# name twice in one run", and reported the first as the second. + +: > "$_lw/dup.tsv" +check "the same check in two logs is two runs, not a duplicate" \ + "$(_led_run merge --ledger "$_lw/dup.tsv" --date 2026-09-10 "$_lw/green.log" "$_lw/green.log" | grep -c 'duplicate')" "0" +printf 'RESULT\tdemo\tpart1\tsame\tPASS\t\nRESULT\tdemo\tpart1\tsame\tFAIL\t\nchecks run: 2\n' > "$_lw/twice.log" +: > "$_lw/dup2.tsv" +check "the same name twice in ONE log is a duplicate, and is named" \ + "$(_led_run merge --ledger "$_lw/dup2.tsv" --date 2026-09-10 "$_lw/twice.log" \ + | grep -c 'duplicate check name in one run, so one ledger row covers 2: demo part1 same')" "1" + +# ---- renames, grouped by part and scanned against ONE run ------------------- +# +# A global positional pairing misses a real rename whenever unrelated movement in +# another part shifts the ordering. And given a before-log and an after-log +# together, the vanished name is present in the union and nothing appears to have +# gone -- a scan that silently finds nothing is worse than one that refuses. + +: > "$_lw/ren.tsv" +printf 'RESULT\tdemo\tpart1\tthe old name\tFAIL\t\nRESULT\tdemo\tpart1\ta stable check\tPASS\t\nchecks run: 2\n' > "$_lw/before.log" +printf 'RESULT\tdemo\tpart1\tthe new name\tPASS\t\nRESULT\tdemo\tpart1\ta stable check\tPASS\t\nchecks run: 2\n' > "$_lw/after.log" +_led_run merge --ledger "$_lw/ren.tsv" --date 2026-09-01 "$_lw/before.log" >/dev/null +check "premise: the check has history before the rename" \ + "$(awk -F'\t' '$3=="the old name"{print $4}' "$_lw/ren.tsv")" "2026-09-01" +check "a name that appeared while another disappeared is reported as a rename" \ + "$(_led_run rename-scan --ledger "$_lw/ren.tsv" "$_lw/after.log" \ + | grep -c 'possible rename: the old name -> the new name')" "1" +check "and the history it is about to lose travels with it" \ + "$(_led_run rename-scan --ledger "$_lw/ren.tsv" "$_lw/after.log" | grep -c 'last red 2026-09-01')" "1" +check "the stable check is not reported" \ + "$(_led_run rename-scan --ledger "$_lw/ren.tsv" "$_lw/after.log" | grep -c 'a stable check')" "0" +check "a before-log and an after-log together are refused, not silently empty" \ + "$(_led_rc rename-scan --ledger "$_lw/ren.tsv" "$_lw/before.log" "$_lw/after.log")" "2" + +# Movement in ANOTHER part must not consume this part's pairing. That is what a +# global positional zip gets wrong, and it fails silently. +: > "$_lw/ren2.tsv" +printf 'RESULT\tdemo\tpartA\told A\tPASS\t\nRESULT\tdemo\tpartB\tstable B\tPASS\t\nchecks run: 2\n' > "$_lw/b2.log" +printf 'RESULT\tdemo\tpartA\tnew A\tPASS\t\nRESULT\tdemo\tpartB\tstable B\tPASS\t\nRESULT\tdemo\tpartB\tadded B\tPASS\t\nchecks run: 3\n' > "$_lw/a2.log" +_led_run merge --ledger "$_lw/ren2.tsv" --date 2026-09-10 "$_lw/b2.log" >/dev/null +check "a rename in one part survives an addition in another" \ + "$(_led_run rename-scan --ledger "$_lw/ren2.tsv" "$_lw/a2.log" \ + | grep -c 'possible rename: old A -> new A')" "1" +check "and the addition in the other part is not called a rename" \ + "$(_led_run rename-scan --ledger "$_lw/ren2.tsv" "$_lw/a2.log" | grep -c 'added B')" "0" + +# A check merely added, or merely removed, is not a rename. +printf 'RESULT\tdemo\tpart1\tthe old name\tPASS\t\nRESULT\tdemo\tpart1\ta stable check\tPASS\t\nRESULT\tdemo\tpart1\tbrand new\tPASS\t\nchecks run: 3\n' > "$_lw/added.log" +check "a check merely added is not reported as a rename" \ + "$(_led_run rename-scan --ledger "$_lw/ren.tsv" "$_lw/added.log" | grep -c 'possible rename')" "0" +printf 'RESULT\tdemo\tpart1\ta stable check\tPASS\t\nchecks run: 1\n' > "$_lw/removed.log" +check "nor is one merely removed" \ + "$(_led_run rename-scan --ledger "$_lw/ren.tsv" "$_lw/removed.log" | grep -c 'possible rename')" "0" + +# ---- the gate refuses a check the ledger has never seen --------------------- + +: > "$_lw/g.tsv" +_led_run merge --ledger "$_lw/g.tsv" --date 2026-09-10 "$_lw/green.log" >/dev/null +printf 'suites_not_covered 0\n' > "$_lw/gb.txt" +check "a run whose checks are all ledgered passes the gate" \ + "$(_led_rc gate --ledger "$_lw/g.tsv" --budget "$_lw/gb.txt" --registered "$_lw/registered" "$_lw/green.log")" "0" +printf 'RESULT\tdemo\tpart1\tfirst check\tPASS\t\nRESULT\tdemo\tpart1\tsecond check\tPASS\t\nRESULT\tdemo\tpart1\tbrand new\tPASS\t\nchecks run: 3\n' > "$_lw/new.log" +check "a check the ledger has never seen is refused" \ + "$(_led_rc gate --ledger "$_lw/g.tsv" --budget "$_lw/gb.txt" --registered "$_lw/registered" "$_lw/new.log")" "1" +check "and it is named, so the author knows which one" \ + "$(_led_run gate --ledger "$_lw/g.tsv" --budget "$_lw/gb.txt" --registered "$_lw/registered" "$_lw/new.log" \ + | grep -c 'not in the ledger: demo part1 brand new')" "1" +check "and the message says how to fix it, because regenerating is the intended action" \ + "$(_led_run gate --ledger "$_lw/g.tsv" --budget "$_lw/gb.txt" --registered "$_lw/registered" "$_lw/new.log" \ + | grep -c 'Regenerate it with')" "1" + +# THE DEADLOCK THAT SHIPPED, as its own arm. Adding a check must not require an +# edit the design forbids. +_led_run merge --ledger "$_lw/g.tsv" --date 2026-09-10 "$_lw/new.log" >/dev/null +check "regenerating the ledger lets the new check through" \ + "$(_led_rc gate --ledger "$_lw/g.tsv" --budget "$_lw/gb.txt" --registered "$_lw/registered" "$_lw/new.log")" "0" +check "and it entered as debt, not as an observation nothing made" \ + "$(awk -F'\t' '$3=="brand new"{print $4}' "$_lw/g.tsv")" "never" + +# ---- the ceiling is monotone, mechanically ---------------------------------- +# +# The file says the ceiling may only fall. Without this the sentence is prose: +# raising the number passed. Measured against a prior value from git rather than +# taken on trust. + +check "the ceiling refuses being exceeded" \ + "$(printf 'suites_not_covered 0\n' > "$_lw/gb0.txt" + printf 'other\ndemo\n' > "$_lw/reg2" + _led_rc gate --ledger "$_lw/g.tsv" --budget "$_lw/gb0.txt" --registered "$_lw/reg2" "$_lw/new.log")" "1" + +_lg="$_lw/repo"; rm -rf "$_lg"; mkdir -p "$_lg" +( cd "$_lg" && git init -q . && git config user.email t@t && git config user.name t + printf 'suites_not_covered 5\n' > b.txt && git add b.txt && git commit -qm base ) >/dev/null 2>&1 +check "premise: the scratch repo has a prior ceiling committed" \ + "$(cd "$_lg" && git show HEAD:b.txt | grep -c 'suites_not_covered 5')" "1" +printf 'suites_not_covered 9\n' > "$_lg/b.txt" +check "raising the ceiling above its committed value is refused" \ + "$(cd "$_lg" && _led_rc gate --ledger "$_lw/g.tsv" --budget b.txt \ + --registered "$_lw/registered" --against HEAD "$_lw/new.log")" "1" +check "and the refusal names both values" \ + "$(cd "$_lg" && _led_run gate --ledger "$_lw/g.tsv" --budget b.txt \ + --registered "$_lw/registered" --against HEAD "$_lw/new.log" \ + | grep -c 'was raised from 5 to 9')" "1" +printf 'suites_not_covered 3\n' > "$_lg/b.txt" +check "lowering it is allowed, which is the direction the burn-down goes" \ + "$(cd "$_lg" && _led_rc gate --ledger "$_lw/g.tsv" --budget b.txt \ + --registered "$_lw/registered" --against HEAD "$_lw/new.log")" "0" + +# ---- and the RUNNER must invoke it ------------------------------------------ +# +# A gate nothing runs is a comment, which is selftest 350's phrasing about its own +# subject. Nothing in the repository called this tool: zero references in +# .github/, zero in the runner. Reported by @linuxhikerpm and by OffgridwithJD +# independently. + +check "the runner invokes the ledger gate" \ + "$(grep -c 'pgc_ledger.py" gate' "$_rv")" "1" +# ---- and the logs outlive the build directory ------------------------------- +# +# The gate runs, then `rm -rf "$builddir"` runs, and CI's collection step globs +# the build directory AFTER the loop has finished -- so it searched a path that +# had already been removed and collected nothing. The ledger is fed by merging +# real logs, and a CI red is exactly the run that first records a check going +# red, so deleting them meant CI could never feed the thing it gates. +# Reported by @linuxhikerpm on #918. +check "the runner keeps the logs somewhere that outlives the build directory" \ + "$(grep -c 'cp -p "\$_l" "\$_logkeep' "$_rv")" "1" +check "and it copies them BEFORE removing the build directory, which is the only order that works" \ + "$([ "$(grep -n 'cp -p "\$_l" "\$_logkeep' "$_rv" | cut -d: -f1)" -lt \ + "$(grep -n 'rm -rf "\$builddir"' "$_rv" | tail -1 | cut -d: -f1)" ] && echo before || echo after)" "before" +check "and CI collects from the retained path rather than the deleted one" \ + "$(grep -c '/tmp/pgcolumnar-logs/\*\.log' "$PGC_SRCDIR/.github/workflows/ci.yml")" "1" + +check "and it runs before the build directory is removed, which is the only place it can" \ + "$([ "$(grep -n 'pgc_ledger.py" gate' "$_rv" | cut -d: -f1)" -lt \ + "$(grep -n 'rm -rf "\$builddir"' "$_rv" | tail -1 | cut -d: -f1)" ] && echo before || echo after)" "before" + +# ---- the committed files agree ---------------------------------------------- + +_l_total="$(grep -c . "$_ledger" || true)" +_l_red="$(awk -F'\t' '$4!="never"' "$_ledger" | grep -c . || true)" +_l_never="$(awk -F'\t' '$4=="never"' "$_ledger" | grep -c . || true)" +echo " ledger: inputs=$_l_total | observed red=$_l_red, never=$_l_never | sum=$((_l_red + _l_never))" +check "the ledger partitions into observed and never" "$((_l_red + _l_never))" "$_l_total" +check "premise: the ledger is not empty, so the partition means something" \ + "$([ "$_l_total" -gt 0 ] && echo yes || echo no)" "yes" +check "every committed row has five fields" \ + "$(awk -F'\t' 'NF!=5' "$_ledger" | grep -c . || true)" "0" +check "and none of them ends in a tab" "$(grep -cP '\t$' "$_ledger" || true)" "0" +check "the committed census matches the committed ledger" \ + "$(sed -n 's/^checks_never_observed_red //p' "$_budget")" "$_l_never" +check "the budget names a ceiling and a census, and says which is which" \ + "$(grep -cE '^(suites_not_covered|checks_never_observed_red) [0-9]+$' "$_budget")" "2" + +# ---- the gate cannot refuse a check in a suite it has never seen ------------- +# +# The suite restriction is the MEANING of suites_not_covered, not a softening of +# the refusal. Without it the gate refuses every check of all 250 uncovered +# suites and reddens the whole matrix on its first run -- a gate somebody turns +# off within the week, which is the failure this issue family exists to prevent. +# +# It tightens on its own as suites are seeded, and the ceiling forces that +# direction. + +printf 'RESULT\tother\tpartX\tsomething\tPASS\t\nchecks run: 1\n' > "$_lw/othersuite.log" +printf 'demo\nother\n' > "$_lw/reg_both" +printf 'suites_not_covered 1\n' > "$_lw/gb1.txt" +check "a check in an UNCOVERED suite is not refused" \ + "$(_led_rc gate --ledger "$_lw/g.tsv" --budget "$_lw/gb1.txt" --registered "$_lw/reg_both" "$_lw/othersuite.log")" "0" +check "but that suite is counted as not covered, which is the debt" \ + "$(_led_run gate --ledger "$_lw/g.tsv" --budget "$_lw/gb1.txt" --registered "$_lw/reg_both" "$_lw/othersuite.log" \ + | grep -c 'not covered=1')" "1" + +# And once the suite IS covered, a new check in it is refused again -- the +# restriction tightens rather than exempting the suite forever. +_led_run merge --ledger "$_lw/g.tsv" --date 2026-09-10 "$_lw/othersuite.log" >/dev/null +printf 'RESULT\tother\tpartX\tsomething\tPASS\t\nRESULT\tother\tpartX\tnewly added\tPASS\t\nchecks run: 2\n' > "$_lw/other2.log" +printf 'suites_not_covered 0\n' > "$_lw/gb2.txt" +check "once the suite is covered, a new check in it IS refused" \ + "$(_led_rc gate --ledger "$_lw/g.tsv" --budget "$_lw/gb2.txt" --registered "$_lw/reg_both" "$_lw/other2.log")" "1" +check "and it is the new one that is named, not the one already ledgered" \ + "$(_led_run gate --ledger "$_lw/g.tsv" --budget "$_lw/gb2.txt" --registered "$_lw/reg_both" "$_lw/other2.log" \ + | grep -c 'not in the ledger: other partX newly added')" "1" + +# ---- the monotone check, in the REAL tree, with the REAL path --------------- +# +# The scratch-repo arms above prove the TOOL. They do not prove the WIRING, and +# the two came apart exactly the way the gate-nothing-invokes finding did one +# level down. OffgridwithJD measured it on the shipped form: +# +# the runner's exact invocation, no --against rc=0 the raise is not refused +# --against HEAD, absolute path rc=0 "no prior ceiling to compare" +# --against HEAD, repo-relative path rc=1 correctly refused +# +# The middle line is the dangerous one: asked to compare, unable to compare, and +# it printed a note that reads like a pass. `git show REF:PATH` needs a +# repo-relative path, and the runner passes an absolute one inside a copied build +# directory. +# +# So the tool resolves the path itself, and every failure to resolve it is an +# ERROR rather than a shrug. These arms use the REAL budget at its real path. + +_mono_budget="$PGC_TESTDIR/check_ledger_budget.txt" +_mono_reg="$_lw/mono_reg" +cut -f1 "$_ledger" | sort -u > "$_mono_reg" +_mono_log="$_lw/mono.log" +awk -F'\t' 'NR<=2 {printf "RESULT\t%s\t%s\t%s\tPASS\t\n", $1, $2, $3}' "$_ledger" > "$_mono_log" +# A log states its own count. read_records reconciles the two, so a fixture +# without this line is not a log the ledger will accept -- which is the point. +printf 'checks run: %s\n' "$(grep -c '^RESULT' "$_mono_log")" >> "$_mono_log" + +check "premise: the real budget is inside a git repository" \ + "$(git -C "$PGC_TESTDIR" rev-parse --show-toplevel >/dev/null 2>&1 && echo yes || echo no)" "yes" +check "premise: the fixture log names checks the real ledger already knows" \ + "$(_led_rc gate --ledger "$_ledger" --budget "$_mono_budget" --registered "$_mono_reg" "$_mono_log")" "0" + +# The absolute path the runner passes must WORK, not fail open. +check "an absolute budget path resolves against git rather than shrugging" \ + "$(_led_run gate --ledger "$_ledger" --budget "$_mono_budget" --registered "$_mono_reg" \ + --against HEAD "$_mono_log" | grep -c 'ceiling against HEAD')" "1" +check "and the comparison passes when the ceiling did not rise" \ + "$(_led_rc gate --ledger "$_ledger" --budget "$_mono_budget" --registered "$_mono_reg" \ + --against HEAD "$_mono_log")" "0" + +# A raise in the real tracked file, at its real path, must redden. +_mono_raised="$PGC_TESTDIR/check_ledger_budget.txt.raised" +sed 's/^suites_not_covered [0-9]*$/suites_not_covered 9999/' "$_mono_budget" > "$_mono_raised" +check "premise: the raised copy really does carry a higher ceiling" \ + "$(sed -n 's/^suites_not_covered //p' "$_mono_raised")" "9999" +check "premise: and it is a file git has never seen, which is the case that used to fail open" \ + "$(git -C "$PGC_TESTDIR" show "HEAD:test/check_ledger_budget.txt.raised" >/dev/null 2>&1 && echo tracked || echo untracked)" "untracked" +# A FILE THAT DOES NOT EXIST AT THE PRIOR HAS NO CEILING TO VIOLATE, so it is a +# NOTE rather than an error. Introducing the budget is not raising it. +# +# The first version made it an error and the gate caught its own bootstrap the +# first time it ran in CI: #925's base is #923's branch, where the budget does not +# exist because this change adds it, so `auto` resolved the base correctly, found +# no prior, failed closed, and reddened the matrix. A PR introducing the file could +# never pass its own gate. +# +# It is not a hole: deleting the budget on a branch and re-adding it higher does +# not reach here, because the file still exists at the prior and the comparison +# happens. Only a genuinely new file gets the note. +check "a budget that does not exist at the prior is a note, not a refusal" \ + "$(_led_rc gate --ledger "$_ledger" --budget "$_mono_raised" --registered "$_mono_reg" \ + --against HEAD "$_mono_log")" "0" +check "and it says the change introduces the file rather than raising anything" \ + "$(_led_run gate --ledger "$_ledger" --budget "$_mono_raised" --registered "$_mono_reg" \ + --against HEAD "$_mono_log" | grep -c 'this change introduces it')" "1" +rm -f "$_mono_raised" + +# And the raise itself, on the tracked path, by rewriting it in place and putting +# it back byte-exact. +_mono_orig="$_lw/budget.orig" +cp "$_mono_budget" "$_mono_orig" +sed -i 's/^suites_not_covered [0-9]*$/suites_not_covered 9999/' "$_mono_budget" +_mono_rc="$(_led_rc gate --ledger "$_ledger" --budget "$_mono_budget" --registered "$_mono_reg" \ + --against HEAD "$_mono_log")" +_mono_out="$(_led_run gate --ledger "$_ledger" --budget "$_mono_budget" --registered "$_mono_reg" \ + --against HEAD "$_mono_log" | grep -c 'was raised from')" +cp "$_mono_orig" "$_mono_budget" +check "raising the ceiling in the tracked file is refused" "$_mono_rc" "1" +check "and the refusal names the raise" "$_mono_out" "1" +check "premise: the budget was restored byte-exact" \ + "$(cmp -s "$_mono_orig" "$_mono_budget" && echo same || echo CHANGED)" "same" + +# ---- and the RUNNER must pass --against, or none of the above is wired ------- + +# The ref is a variable, chosen above the call and pinned by the three arms at +# the end of this part. What matters here is that the call site passes one at all: +# without --against the monotone block never runs, which is how the tool was right +# and the wiring was not. +check "the runner passes --against to the gate" \ + "$(grep -A5 'pgc_ledger.py" gate' "$_rv" | grep -c -- '--against')" "1" + + + +# ---- the prior is resolved, never named ------------------------------------ +# +# `origin` is not a fixed thing. In a contributor's clone it is their FORK, and +# OffgridwithJD measured theirs 446 commits behind upstream. Comparing the ceiling +# against a stale main makes this check WEAKER rather than falsely red -- the +# ceiling may only fall, so an older main carries a higher one, and a raise passes +# whenever the stale prior is high enough. +# +# It fails open while printing a line that reads like the enforcement happened. +# That is the same shape as the absolute-path bug this part already carries, with +# "compared against the wrong thing" in place of "could not compare". +# +# So the tool resolves it: GITHUB_BASE_REF in CI, which names the PR's target and +# IS the prior by definition; the local main's configured upstream outside CI, +# which is the per-clone answer to "which main is mine". Neither available is an +# ERROR, because a fallback that enforces less while saying so is still a gate +# enforcing less. + +check "the runner asks the tool to resolve the prior rather than naming one" \ + "$(grep -A5 'pgc_ledger.py" gate' "$_rv" | grep -c -- '--against auto')" "1" +check "and the runner names no remote at that call site" \ + "$(grep -A6 'pgc_ledger.py" gate' "$_rv" | grep -c 'origin/main')" "0" + +_res="$_lw/resolve"; rm -rf "$_res"; mkdir -p "$_res" +( cd "$_res" && git init -q . && git config user.email t@t && git config user.name t + printf 'suites_not_covered 7\n' > b.txt && git add b.txt && git commit -qm base ) >/dev/null 2>&1 +printf 'suites_not_covered 7\n' > "$_res/b.txt" + +check "premise: the scratch repo has a committed ceiling and no upstream" \ + "$(cd "$_res" && git rev-parse --abbrev-ref main@{upstream} 2>/dev/null || echo none)" "none" +check "auto with no base ref and no upstream is an integrity failure" \ + "$(cd "$_res" && env -u GITHUB_BASE_REF python3 "$_led" gate --ledger "$_lw/g.tsv" \ + --budget b.txt --registered "$_lw/registered" --against auto "$_lw/new.log" \ + >/dev/null 2>&1; echo $?)" "2" +check "and it says why, rather than falling back to something weaker" \ + "$(cd "$_res" && env -u GITHUB_BASE_REF python3 "$_led" gate --ledger "$_lw/g.tsv" \ + --budget b.txt --registered "$_lw/registered" --against auto "$_lw/new.log" 2>&1 \ + | grep -c 'no trustworthy prior ceiling')" "1" + +# GITHUB_BASE_REF names a branch whose ref must actually exist. A CI checkout that +# did not fetch the base is an error the workflow fixes, not one the author works +# around. +check "auto refuses when GITHUB_BASE_REF names a ref that is not here" \ + "$(cd "$_res" && GITHUB_BASE_REF=nosuchbranch python3 "$_led" gate --ledger "$_lw/g.tsv" \ + --budget b.txt --registered "$_lw/registered" --against auto "$_lw/new.log" 2>&1 \ + | grep -ci 'the checkout needs to fetch the base branch')" "1" + +# And when it IS here, that is the ref used -- named in the output, so a reader +# can see which prior the comparison actually made. +( cd "$_res" && git branch -q basebranch && git update-ref refs/remotes/origin/basebranch \ + "$(git rev-parse HEAD)" ) >/dev/null 2>&1 +check "auto uses the base ref when it resolves, and names it" \ + "$(cd "$_res" && GITHUB_BASE_REF=basebranch python3 "$_led" gate --ledger "$_lw/g.tsv" \ + --budget b.txt --registered "$_lw/registered" --against auto "$_lw/new.log" 2>&1 \ + | grep -c 'ceiling against refs/remotes/origin/basebranch')" "1" +printf 'suites_not_covered 99\n' > "$_res/b.txt" +check "and a raise against that base ref is refused" \ + "$(cd "$_res" && GITHUB_BASE_REF=basebranch python3 "$_led" gate --ledger "$_lw/g.tsv" \ + --budget b.txt --registered "$_lw/registered" --against auto "$_lw/new.log" 2>&1 \ + | grep -c 'was raised from 7 to 99')" "1" + +# ---- and CI must fetch that base, or the gate stops the run ---------------- + +_ci="$PGC_SRCDIR/.github/workflows/ci.yml" +check "premise: the workflow file is where this part thinks it is" \ + "$([ -f "$_ci" ] && echo yes || echo no)" "yes" +check "the suites job fetches the PR base for the ceiling comparison" \ + "$(grep -c 'fetch the PR base, for the ledger ceiling comparison' "$_ci")" "1" +check "and only when there is a base, so a push build does not fail on it" \ + "$(grep -A1 'fetch the PR base, for the ledger ceiling comparison' "$_ci" \ + | grep -c "github.base_ref != ''")" "1" + +# ---- and how far behind the prior is, printed beside it --------------------- +# +# `main@{upstream}` is the per-clone answer to "which main is mine", and in a +# contributor's setup it resolves to their FORK. OffgridwithJD measured theirs 446 +# commits behind upstream -- and `git push -u origin main` is what sets that +# config, so `auto` outside CI lands on exactly the ref the hardcoded version did. +# The route changed; the destination did not. +# +# There is no better ref to pick that does not guess, so the weakness is made +# VISIBLE rather than removed. Naming the ref told a reader WHICH prior was used; +# it did not tell them what the comparison was worth. The direction still fails +# open: an older main carries a higher ceiling, so a raise passes whenever the +# stale prior is high enough. +# +# Their suggestion, and it costs nothing when the number is 0. + +_dist="$_lw/dist"; rm -rf "$_dist"; mkdir -p "$_dist" +( cd "$_dist" && git init -q . && git config user.email t@t && git config user.name t + printf 'suites_not_covered 7\n' > b.txt && git add b.txt && git commit -qm base + git branch -q oldbase + for i in 1 2 3; do echo "x$i" > f; git add f; git commit -qm "c$i"; done ) >/dev/null 2>&1 +printf 's\tp\ta\tnever\t-\n' > "$_dist/led" +printf 'RESULT\ts\tp\ta\tPASS\t\nchecks run: 1\n' > "$_dist/log" +printf 's\n' > "$_dist/reg" + +check "premise: the scratch prior really is three commits behind" \ + "$(cd "$_dist" && git rev-list --count oldbase..HEAD)" "3" +check "a stale prior is named WITH its distance from HEAD" \ + "$(cd "$_dist" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against oldbase log 2>&1 | grep -c 'ceiling against oldbase (3 commits behind HEAD)')" "1" +check "and a level prior carries no distance, so zero is silent" \ + "$(cd "$_dist" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against HEAD log 2>&1 | grep -c 'ceiling against HEAD: ')" "1" +check "the distance travels with a refusal too, not only with a pass" \ + "$(cd "$_dist" && sed -i 's/^suites_not_covered 7$/suites_not_covered 99/' b.txt + python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against oldbase log 2>&1 | grep -c 'against oldbase (3 commits behind HEAD)')" "1" + +# ---- the runner must not collapse the two failure kinds --------------------- +# +# The gate distinguishes rc=1, a real refusal whose fix is to regenerate the +# ledger, from rc=2, the gate unable to do its job at all. The runner reported +# both as "has a check the ledger has never seen", which sends the reader at a +# repair that cannot help -- and printed it three lines below the gate's own +# "new this run=0", which says the opposite. Reported by OffgridwithJD from the +# CI log of this very branch. + +# The block is EXTRACTED and tested, not counted inside a -A window. A window's +# size is a fact about formatting: the first version of these arms measured 6, 8 +# and 12 lines, and every one of them went wrong the moment the call site gained +# a comment. Selftest 320 already takes this approach with the runner's classifier. +# Comments stripped. The block's own explanation quotes the sentences these arms +# count, so an unstripped extraction counts the documentation as a second +# occurrence -- selftest 080's control problem, which this session has now met +# five times in five different files. +_ledcase="$(sed -n '/^\t\t_led_rc=\$?$/,/^\t\tesac$/p' "$_rv" | grep -vE '^[[:space:]]*#')" +check "premise: the gate's status block was found in the runner" \ + "$(printf '%s' "$_ledcase" | grep -c 'case "\$_led_rc" in')" "1" +check "the runner captures the gate's status rather than only its success" \ + "$(grep -c '_led_rc=\$?' "$_rv")" "1" +check "a refusal keeps the regenerate-the-ledger wording" \ + "$(printf '%s' "$_ledcase" | grep -c 'has a check the ledger has never seen')" "1" +check "an integrity failure says regenerating will not help" \ + "$(printf '%s' "$_ledcase" | grep -c 'will not help')" "1" +check "and it is a different sentence from the refusal, not the same one twice" \ + "$(printf '%s' "$_ledcase" | grep -c 'could not run the ledger gate at all')" "1" +check "a clean status says nothing and does not fail the major" \ + "$(printf '%s' "$_ledcase" | grep -cE '^[[:space:]]+0\)[[:space:]]*;;[[:space:]]*$')" "1" +check "both failure arms fail the major" \ + "$(printf '%s' "$_ledcase" | grep -c 'verfail=1')" "2" + +# ---- an unresolvable ref is not the bootstrap case -------------------------- +# +# The three states are the point of this part, and one path had collapsed two of +# them. The ref-resolution check lived inside the `auto` resolver, so it covered +# the production call site and nothing else: an EXPLICIT ref that did not resolve +# fell through to the file-absent branch and was reported as rc=0, with a message +# asserting "this change introduces it" about a ref that does not exist -- one +# clause after saying the distance from HEAD was unknown. The code knew it could +# not resolve the ref and contradicted itself inside one sentence. +# +# Unreachable from the runner, which always passes `auto`. A trap for these arms, +# for anyone driving the tool by hand, and for whoever later passes a concrete ref +# because `auto` was inconvenient. Scoped exactly that way by OffgridwithJD, who +# checked the production path first rather than leading with the headline. + +_rr="$_lw/refres"; rm -rf "$_rr"; mkdir -p "$_rr" +( cd "$_rr" && git init -q . && git config user.email t@t && git config user.name t + printf 'suites_not_covered 7\n' > b.txt && git add b.txt && git commit -qm base + git branch -q hasbudget + git rm -q b.txt && git commit -qm "a commit without the budget" + git branch -q nobudget + git checkout -q hasbudget ) >/dev/null 2>&1 +printf 's\tp\ta\tnever\t-\n' > "$_rr/led" +printf 'RESULT\ts\tp\ta\tPASS\t\nchecks run: 1\n' > "$_rr/log" +printf 's\n' > "$_rr/reg" + +check "premise: the hasbudget branch carries the budget" \ + "$(cd "$_rr" && git show hasbudget:b.txt >/dev/null 2>&1 && echo yes || echo no)" "yes" +check "premise: and the nobudget branch does not, which is the bootstrap shape" \ + "$(cd "$_rr" && git show nobudget:b.txt >/dev/null 2>&1 && echo yes || echo no)" "no" + +check "a ref that does not resolve is an integrity failure, not a bootstrap" \ + "$(cd "$_rr" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against refs/heads/no-such-ref-xyz log >/dev/null 2>&1; echo $?)" "2" +check "and it says the ref does not resolve, rather than claiming the file is new" \ + "$(cd "$_rr" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against refs/heads/no-such-ref-xyz log 2>&1 | grep -c 'that ref does not resolve here')" "1" +check "and never says a change introduces a file at a ref that is not there" \ + "$(cd "$_rr" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against refs/heads/no-such-ref-xyz log 2>&1 | grep -c 'this change introduces it')" "0" + +# The bootstrap case must still be the bootstrap case: a ref that EXISTS, without +# the file. Without this arm the fix above is satisfied by refusing everything. +check "a ref that exists without the budget is still the bootstrap case" \ + "$(cd "$_rr" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against nobudget log >/dev/null 2>&1; echo $?)" "0" +check "and it is that case that says the change introduces the file" \ + "$(cd "$_rr" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against nobudget log 2>&1 | grep -c 'this change introduces it')" "1" + +# And a ref that exists WITH the file still compares, so the third state is intact. +check "a ref that exists with the budget still compares" \ + "$(cd "$_rr" && python3 "$_led" gate --ledger led --budget b.txt --registered reg \ + --against hasbudget log 2>&1 | grep -c 'ceiling against hasbudget')" "1" diff --git a/test/selftest/420-a-deleted-part-must-be-visible.sh b/test/selftest/420-a-deleted-part-must-be-visible.sh new file mode 100644 index 00000000..2f3dfcbc --- /dev/null +++ b/test/selftest/420-a-deleted-part-must-be-visible.sh @@ -0,0 +1,68 @@ +# ---- a deleted selftest part must not shrink the run silently --------------- +# +# WHY THIS EXISTS. harness_selftest sources `selftest/*.sh`. A glob cannot notice +# a deletion: the file goes, the loop runs one fewer time, the suite reports a +# smaller number, and every remaining check still passes. Measured on #918 -- a +# log with all 96 records of part 410 removed, and `checks run:` adjusted to +# match, reconciled with itself AND merged into the ledger at rc=0. Nothing in +# either harness could tell that from a run where part 410 simply had less to +# say. Reported by @linuxhikerpm. +# +# THE MANIFEST IS INDEPENDENT OF THE GLOB, which is the entire point. Comparing +# the glob against itself is what the runner already does; comparing it against a +# COMMITTED list makes deleting a part a two-line diff a reviewer sees rather +# than a number that quietly falls. +# +# This lives in a part rather than in the driver because selftest 200 requires +# the driver to hold no checks -- they all live in parts -- and that rule is +# right: an assertion in the driver is one nobody looks for. + +_dp_dir="$PGC_TESTDIR/selftest" +_dp_manifest="$_dp_dir/parts.manifest" + +check "the parts manifest exists, because without it a deletion is invisible" \ + "$([ -f "$_dp_manifest" ] && echo present || echo missing)" "present" + +_dp_ondisk="$(cd "$_dp_dir" && ls -1 ./*.sh 2>/dev/null | sed 's|^\./||' | sort)" +_dp_listed="$(grep -v '^[[:space:]]*$' "$_dp_manifest" 2>/dev/null | sort)" + +check "premise: the manifest names something, so the comparison has two sides" \ + "$([ "$(printf '%s\n' "$_dp_listed" | grep -c .)" -ge 20 ] && echo yes || echo no)" "yes" + +_dp_only_disk="$(comm -23 <(printf '%s\n' "$_dp_ondisk") <(printf '%s\n' "$_dp_listed"))" +_dp_only_list="$(comm -13 <(printf '%s\n' "$_dp_ondisk") <(printf '%s\n' "$_dp_listed"))" + +check "every part on disk is named in the manifest" \ + "$(printf '%s' "$_dp_only_disk" | grep -c . || true)" "0" +[ -z "$_dp_only_disk" ] || printf ' on disk but unlisted: %s\n' $_dp_only_disk + +check "and every name in the manifest is a part on disk, so a deletion reddens" \ + "$(printf '%s' "$_dp_only_list" | grep -c . || true)" "0" +[ -z "$_dp_only_list" ] || printf ' listed but gone: %s\n' $_dp_only_list + +# inputs == sum(buckets), printed from the data rather than retyped. +_dp_both="$(comm -12 <(printf '%s\n' "$_dp_ondisk") <(printf '%s\n' "$_dp_listed") | grep -c . || true)" +_dp_n_disk="$(printf '%s\n' "$_dp_ondisk" | grep -c . || true)" +echo " parts: on disk=$_dp_n_disk | in both=$_dp_both, only on disk=$(printf '%s' "$_dp_only_disk" | grep -c . || true), only in the manifest=$(printf '%s' "$_dp_only_list" | grep -c . || true)" +check "the three buckets account for every part on disk" \ + "$((_dp_both + $(printf '%s' "$_dp_only_disk" | grep -c . || true)))" "$_dp_n_disk" + +# THE COMPARISON MUST BE ABLE TO FAIL, on a fixture rather than on the tree. +_dp_fix="$PGC_WORKDIR/deletedpart"; rm -rf "$_dp_fix"; mkdir -p "$_dp_fix" +printf '%s\n' aaa.sh bbb.sh ccc.sh > "$_dp_fix/manifest" +printf '%s\n' aaa.sh ccc.sh > "$_dp_fix/ondisk" +check "premise: a part present in the manifest and missing on disk is named" \ + "$(comm -13 "$_dp_fix/ondisk" "$_dp_fix/manifest" | tr -d '[:space:]')" "bbb.sh" +printf '%s\n' aaa.sh bbb.sh ccc.sh ddd.sh > "$_dp_fix/ondisk" +check "premise: and a part added without listing it is named too" \ + "$(comm -23 "$_dp_fix/ondisk" "$_dp_fix/manifest" | tr -d '[:space:]')" "ddd.sh" + +# WHAT THIS DOES NOT CATCH, stated rather than implied. A part that still exists +# and contributes nothing -- an early return, a condition that never fires -- is +# invisible here. A runtime "every part contributed at least one check" arm was +# written first and REMOVED: parts 010 and 020 legitimately contribute none, 020 +# because it is setup with no check calls at all and 010 because its only checks +# are on its failure path. A rule with two false positives on a healthy tree is +# not a rule, and the deletion this part exists for is caught above. +unset _dp_dir _dp_manifest _dp_ondisk _dp_listed _dp_only_disk _dp_only_list \ + _dp_both _dp_n_disk _dp_fix diff --git a/test/selftest/parts.manifest b/test/selftest/parts.manifest new file mode 100644 index 00000000..1a5f64d9 --- /dev/null +++ b/test/selftest/parts.manifest @@ -0,0 +1,44 @@ +010-stand-up-a-squatter-on-a.sh +020-point-a-real-suite-setup-at.sh +030-assertions.sh +040-the-detection-primitive-itself.sh +050-the-list-must-be-read-the.sh +060-the-list-stays-sorted-which-is.sh +070-and-comm-s-two-inputs-must.sh +080-no-suite-pipes-a-captured-string.sh +090-no-suite-hands-every-run-the.sh +100-the-assertions-that-refuse-an-empty.sh +110-the-harness-must-say-which-binary.sh +120-a-failing-suite-must-surface-the.sh +130-the-sanitizer-subset-must-cover-the.sh +140-a-cluster-that-will-not-start.sh +150-the-verdict-must-not-assert-a.sh +160-and-the-log-report-must-show.sh +170-and-lib-sh-must-ask-these.sh +180-the-port-walk-must-wrap-not.sh +190-an-in-tree-build-must-not.sh +200-additions-go-in-their-own-file.sh +210-no-suite-assigns-a-bash-special.sh +220-an-opt-in-upgrade-guard-must.sh +230-a-suite-connecting-by-socket-must.sh +240-the-nightly-enumeration-must-not.sh +250-the-coverage-runner-must-refuse.sh +260-an-ordered-comparison-must-use-the.sh +270-a-set-options-call-must-use-values.sh +280-the-shared-cluster-config-must-not.sh +290-a-preflight-that-built-nothing.sh +300-a-test-script-must-be-runnable.sh +310-a-compiled-artifact-must-not-be.sh +320-a-check-that-could-not-run.sh +330-the-incomplete-path-must-run-whole.sh +340-the-binary-must-be-built-from.sh +350-the-pytest-corpus-must-be.sh +360-an-unrunnable-pytest-test-must.sh +370-the-plan-marker-guard-must.sh +380-the-pytest-cluster-helpers.sh +390-a-registered-suite-must-account.sh +400-a-check-result-must-be-machine.sh +410-a-check-must-have-been-red.sh +420-a-deleted-part-must-be-visible.sh +430-the-self-test-must-not-report.sh +440-a-count-grep-never-produced.sh diff --git a/test/sorted_pathkeys.sh b/test/sorted_pathkeys.sh index f948f50d..85b477e0 100755 --- a/test/sorted_pathkeys.sh +++ b/test/sorted_pathkeys.sh @@ -275,7 +275,7 @@ ansp "and it answers in C order, matching heap" colh colc \ ALTCOLL="$(q "SELECT collname FROM pg_collation WHERE collname IN ('en_US.utf8','en_US.UTF-8','en_US','und-x-icu') ORDER BY 1 LIMIT 1;")" if [ -z "$ALTCOLL" ] || \ [ "$(q "SELECT (min(k) COLLATE \"C\") = (SELECT min(k COLLATE \"$ALTCOLL\") FROM colh) FROM colh;" 2>/dev/null)" != "f" ]; then - echo "SKIP the collation-change demonstration: this server has no collation that" + check_skip "the collation-change demonstration" "SKIP the collation-change demonstration: this server has no collation that" "this server has no suitable collation" echo " disagrees with C on ASCII, so the arm could not fail and is not run." echo " The refusal it demonstrates is asserted above on COLLATE \"C\"." else diff --git a/test/unique_conc.sh b/test/unique_conc.sh index 0c0fc30d..304119d8 100755 --- a/test/unique_conc.sh +++ b/test/unique_conc.sh @@ -389,7 +389,7 @@ if [ "$CITEXT" = 1 ]; then check "2c post-fix: exactly one row equal to 'abc'" \ "$(ctl_q "SELECT count(*) FROM s_ci WHERE v = 'abc';")" "1" else - echo "SKIP 2c citext case test (citext extension not available)" + check_skip "the 2c citext case" "SKIP 2c citext case test (citext extension not available)" "citext extension not available" fi # =========================================================================== @@ -521,7 +521,7 @@ if [ "$PG_MAJOR" -ge 15 ]; then check "6 NULLS NOT DISTINCT: exactly one NULL row" \ "$(ctl_q "SELECT count(*) FROM s_nn WHERE k IS NULL;")" "1" else - echo "SKIP 6 NULLS NOT DISTINCT test (PostgreSQL < 15)" + check_skip "the NULLS NOT DISTINCT case" "SKIP 6 NULLS NOT DISTINCT test (PostgreSQL < 15)" "PostgreSQL < 15" fi # ===========================================================================