diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3219661..a0be535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,8 @@ jobs: # it by two, retiring a legacy file moves it by one -- so gating on it # would punish ordinary progress and reward deleting hard examples. # - # scripts/catalogue_validating.txt names the files that validate. A + # scripts/catalogue_validation_baseline.txt names the files that pass + # validation -- which is not the same as running correctly. A # file LEAVING that list fails this job. A file joining it does not; # it prints a reminder to run --update. # diff --git a/scripts/catalogue_report.py b/scripts/catalogue_report.py index ffe0a64..3e48bb6 100644 --- a/scripts/catalogue_report.py +++ b/scripts/catalogue_report.py @@ -10,10 +10,16 @@ So this reports two different things, and only one of them can fail the build: -* **The supported list.** `scripts/catalogue_validating.txt` names the files - that validate today. A file dropping out of it is a regression and fails. - A file newly validating is an improvement and does not -- it just prints a - reminder to add it. +* **The validation baseline.** `scripts/catalogue_validation_baseline.txt` + names the files that *pass `orchestrator validate`* today. A file dropping + out of it is a regression and fails. A file newly validating is an + improvement and does not -- it just prints a reminder to add it. + + It says nothing more than that. Passing validation is not the same as + running, producing correct artifacts, or being a supported example, and + calling this a "supported list" conflated four different things. + `examples/supported/` is the stronger contract: those are executed and + their behaviour asserted. This file is the weaker, wider net. * **Everything else, grouped by the validator's own first error.** Counts by signature rather than by a taxonomy written down somewhere, because a @@ -22,7 +28,7 @@ Usage: python scripts/catalogue_report.py # report; fail on regressions - python scripts/catalogue_report.py --update # rewrite the supported list + python scripts/catalogue_report.py --update # rewrite the baseline python scripts/catalogue_report.py --json # machine-readable """ @@ -41,7 +47,7 @@ REPO = Path(__file__).resolve().parent.parent EXAMPLES = REPO / "examples" -SUPPORTED_LIST = REPO / "scripts" / "catalogue_validating.txt" +BASELINE = REPO / "scripts" / "catalogue_validation_baseline.txt" #: Collapses the variable parts of an error so that "Tool 'a' not found" and #: "Tool 'b' not found" are one signature rather than two. @@ -156,18 +162,18 @@ def compare( ) -def load_supported() -> Optional[List[str]]: - if not SUPPORTED_LIST.exists(): +def load_baseline() -> Optional[List[str]]: + if not BASELINE.exists(): return None return [ line.strip() - for line in SUPPORTED_LIST.read_text().splitlines() + for line in BASELINE.read_text().splitlines() if line.strip() and not line.startswith("#") ] -def write_supported(paths: List[str]) -> None: - SUPPORTED_LIST.write_text( +def write_baseline(paths: List[str]) -> None: + BASELINE.write_text( "# Examples that pass `orchestrator validate`.\n" "#\n" "# A file leaving this list is a regression and fails CI. A file\n" @@ -183,7 +189,7 @@ def write_supported(paths: List[str]) -> None: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--update", action="store_true", - help="rewrite the supported list from what validates now") + help="rewrite the baseline from what validates now") parser.add_argument("--json", action="store_true", help="machine-readable output") parser.add_argument("--jobs", type=int, default=8) args = parser.parse_args() @@ -206,13 +212,13 @@ def main() -> int: failures[rel] = first_error(output) if args.update: - write_supported(validating) - print(f"wrote {SUPPORTED_LIST.relative_to(REPO)} with {len(validating)} entries") + write_baseline(validating) + print(f"wrote {BASELINE.relative_to(REPO)} with {len(validating)} entries") return 0 signatures = Counter(failures.values()) - supported = load_supported() - regressed, gained = compare(supported or [], validating) + baseline = load_baseline() + regressed, gained = compare(baseline or [], validating) if args.json: # Nothing but JSON on stdout: this used to print the human report @@ -224,9 +230,9 @@ def main() -> int: "by_signature": dict(signatures.most_common()), "failures": failures, "regressed": regressed, - "gained": gained if supported is not None else [], + "gained": gained if baseline is not None else [], }, indent=2)) - return 1 if (supported is not None and regressed) else 0 + return 1 if (baseline is not None and regressed) else 0 print(f"catalogue: {len(validating)}/{len(files)} validate\n") print(f"{'count':>5} first error") @@ -234,8 +240,8 @@ def main() -> int: for signature, count in signatures.most_common(): print(f"{count:>5} {signature}") - if supported is None: - print(f"\nno {SUPPORTED_LIST.relative_to(REPO)} yet; create it with --update") + if baseline is None: + print(f"\nno {BASELINE.relative_to(REPO)} yet; create it with --update") return 0 if gained: @@ -252,7 +258,7 @@ def main() -> int: "headline count is not the contract; these named files are.") return 1 - print(f"\nall {len(supported)} listed example(s) still validate") + print(f"\nall {len(baseline)} listed example(s) still validate") return 0 diff --git a/scripts/catalogue_validating.txt b/scripts/catalogue_validation_baseline.txt similarity index 89% rename from scripts/catalogue_validating.txt rename to scripts/catalogue_validation_baseline.txt index 926eb76..18db8ee 100644 --- a/scripts/catalogue_validating.txt +++ b/scripts/catalogue_validation_baseline.txt @@ -1,4 +1,8 @@ -# Examples that pass `orchestrator validate`. +# Examples that pass `orchestrator validate` -- and nothing more. +# +# Passing validation is not running, is not producing correct output, +# and is not being a supported example. examples/supported/ is the +# stronger contract: those are executed and their behaviour asserted. # # A file leaving this list is a regression and fails CI. A file # joining it is an improvement; run: diff --git a/tests/test_catalogue_report.py b/tests/test_catalogue_report.py index b506408..1675392 100644 --- a/tests/test_catalogue_report.py +++ b/tests/test_catalogue_report.py @@ -23,7 +23,7 @@ pytestmark = [pytest.mark.unit] REPO = Path(__file__).resolve().parent.parent -SUPPORTED_LIST = REPO / "scripts" / "catalogue_validating.txt" +BASELINE = REPO / "scripts" / "catalogue_validation_baseline.txt" def _module(): @@ -155,7 +155,7 @@ def test_a_swap_of_equal_size_is_still_a_regression(): def test_every_listed_example_exists(): """A stale entry would fail CI forever with a confusing message.""" missing = [ - line for line in SUPPORTED_LIST.read_text().splitlines() + line for line in BASELINE.read_text().splitlines() if line.strip() and not line.startswith("#") and not (REPO / line.strip()).exists() ] @@ -163,6 +163,6 @@ def test_every_listed_example_exists(): def test_the_list_is_not_empty(): - assert report.load_supported(), ( - "an empty supported list would make the regression gate vacuous" + assert report.load_baseline(), ( + "an empty baseline would make the regression gate vacuous" )