|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""A failing member names its issue (task N2 of the 2026-09-28 ecosystem design, |
| 3 | +mcpp-community/mcpp `.agents/docs/2026-09-28-ecosystem-design-and-optimisation-plan.md`). |
| 4 | +
|
| 5 | +WHY. The two full sweeps of 2026-09-26 on `main` failed -- one on the |
| 6 | +`mirror-cn-reachable` job, one on the member `pangocairo` (linux) -- and nothing |
| 7 | +recorded either: the job that opens the sweep issue failed as well, and a red |
| 8 | +weekly sweep that no issue names is read by nobody (#482). |
| 9 | +
|
| 10 | +`tests/known-red.tsv` lists the members known to fail, per leg, each with the |
| 11 | +open issue that owns it. After the members have run, this script reads every |
| 12 | +shard's timing rows (`<seconds>\t<member>\t<ok|FAIL>`, in the artifacts |
| 13 | +`timings-<platform>-<toolchain>-<shard>`), and: |
| 14 | +
|
| 15 | +- lists every failing member with its issue, in the run summary; |
| 16 | +- fails when a failing member has no issue on its leg; |
| 17 | +- fails when the members' job failed and no member row says FAIL (a shard that |
| 18 | + failed before testing: nothing can be attributed); |
| 19 | +- lists a known-red member that passed, so its line can be removed. |
| 20 | +
|
| 21 | + red_members.py report <timings dir> <known-red.tsv> [--members-result R] |
| 22 | + red_members.py selftest |
| 23 | +
|
| 24 | +A leg is `<platform>-<toolchain>` (`linux-default`, `linux-llvm`, |
| 25 | +`macos-default`, `windows-default`); `*` in known-red.tsv matches every leg. |
| 26 | +""" |
| 27 | +from __future__ import annotations |
| 28 | + |
| 29 | +import pathlib |
| 30 | +import sys |
| 31 | +import tempfile |
| 32 | + |
| 33 | + |
| 34 | +def read_known(path: pathlib.Path) -> dict[tuple[str, str], str]: |
| 35 | + known: dict[tuple[str, str], str] = {} |
| 36 | + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): |
| 37 | + if not line.strip() or line.lstrip().startswith("#"): |
| 38 | + continue |
| 39 | + parts = line.split("\t") |
| 40 | + if len(parts) != 3 or not all(p.strip() for p in parts): |
| 41 | + raise SystemExit(f"{path}:{number}: expected <leg>\\t<member>\\t<issue>, got {line!r}") |
| 42 | + leg, member, issue = (p.strip() for p in parts) |
| 43 | + known[(leg, member)] = issue |
| 44 | + return known |
| 45 | + |
| 46 | + |
| 47 | +def read_rows(timings: pathlib.Path) -> list[tuple[str, str, str]]: |
| 48 | + """(leg, member, status) for every row of every shard.""" |
| 49 | + rows = [] |
| 50 | + for tsv in sorted(timings.glob("timings-*/timings.tsv")): |
| 51 | + # timings-<platform>-<toolchain>-<shard> |
| 52 | + name = tsv.parent.name.split("-") |
| 53 | + if len(name) < 4: |
| 54 | + continue |
| 55 | + leg = f"{name[1]}-{name[2]}" |
| 56 | + for line in tsv.read_text(encoding="utf-8").splitlines(): |
| 57 | + parts = line.split("\t") |
| 58 | + if len(parts) == 3: |
| 59 | + rows.append((leg, parts[1], parts[2].strip())) |
| 60 | + return rows |
| 61 | + |
| 62 | + |
| 63 | +def report(timings: pathlib.Path, known_path: pathlib.Path, members_result: str, |
| 64 | + out=sys.stdout) -> int: |
| 65 | + known = read_known(known_path) |
| 66 | + rows = read_rows(timings) |
| 67 | + failing = sorted({(leg, m) for leg, m, s in rows if s == "FAIL"}) |
| 68 | + passing = {(leg, m) for leg, m, s in rows if s == "ok"} |
| 69 | + |
| 70 | + def issue_of(leg: str, member: str) -> str | None: |
| 71 | + return known.get((leg, member)) or known.get(("*", member)) |
| 72 | + |
| 73 | + unattributed = [(leg, m) for leg, m in failing if not issue_of(leg, m)] |
| 74 | + print("### Red members", file=out) |
| 75 | + print("", file=out) |
| 76 | + if failing: |
| 77 | + print("| leg | member | issue |", file=out) |
| 78 | + print("|---|---|---|", file=out) |
| 79 | + for leg, m in failing: |
| 80 | + print(f"| {leg} | `{m}` | {issue_of(leg, m) or '**none**'} |", file=out) |
| 81 | + else: |
| 82 | + print("No member failed.", file=out) |
| 83 | + healed = sorted((leg, m) for (leg, m) in known |
| 84 | + if leg != "*" and (leg, m) in passing) |
| 85 | + if healed: |
| 86 | + print("", file=out) |
| 87 | + print("Listed in tests/known-red.tsv and passing now (remove the line once " |
| 88 | + "its issue is closed):", file=out) |
| 89 | + for leg, m in healed: |
| 90 | + print(f"- {leg} `{m}` ({known[(leg, m)]})", file=out) |
| 91 | + status = 0 |
| 92 | + if unattributed: |
| 93 | + print("", file=out) |
| 94 | + print("A failing member without an issue fails this job: open one and add " |
| 95 | + "`<leg>\\t<member>\\t<issue>` to tests/known-red.tsv, or fix the member.", |
| 96 | + file=out) |
| 97 | + status = 1 |
| 98 | + if members_result == "failure" and not failing: |
| 99 | + print("", file=out) |
| 100 | + print("The members' job failed, and no member row says FAIL: a shard failed " |
| 101 | + "before testing, and nothing can be attributed to a member.", file=out) |
| 102 | + status = 1 |
| 103 | + return status |
| 104 | + |
| 105 | + |
| 106 | +def selftest() -> int: |
| 107 | + def run(rows_by_artifact: dict[str, str], known: str, result: str) -> tuple[int, str]: |
| 108 | + with tempfile.TemporaryDirectory() as d: |
| 109 | + root = pathlib.Path(d) |
| 110 | + for artifact, text in rows_by_artifact.items(): |
| 111 | + (root / artifact).mkdir() |
| 112 | + (root / artifact / "timings.tsv").write_text(text, encoding="utf-8") |
| 113 | + known_path = root / "known-red.tsv" |
| 114 | + known_path.write_text(known, encoding="utf-8") |
| 115 | + import io |
| 116 | + buf = io.StringIO() |
| 117 | + code = report(root, known_path, result, out=buf) |
| 118 | + return code, buf.getvalue() |
| 119 | + |
| 120 | + header = "# <leg>\t<member>\t<issue>\n" |
| 121 | + cases = [ |
| 122 | + # (name, rows, known, members result, expected status, text that must appear) |
| 123 | + ("all green", {"timings-linux-default-0": "10\ta\tok\n"}, header, "success", 0, "No member failed"), |
| 124 | + ("a red member without an issue", {"timings-linux-default-0": "10\ta\tFAIL\n"}, header, |
| 125 | + "failure", 1, "**none**"), |
| 126 | + ("a red member with its issue", {"timings-linux-default-0": "10\ta\tFAIL\n"}, |
| 127 | + header + "linux-default\ta\t#1\n", "failure", 0, "| linux-default | `a` | #1 |"), |
| 128 | + ("an issue on another leg does not cover", {"timings-linux-llvm-1": "10\ta\tFAIL\n"}, |
| 129 | + header + "linux-default\ta\t#1\n", "failure", 1, "**none**"), |
| 130 | + ("a wildcard leg covers", {"timings-windows-default-0": "10\ta\tFAIL\n"}, |
| 131 | + header + "*\ta\t#2\n", "failure", 0, "#2"), |
| 132 | + ("a shard that failed before testing", {"timings-linux-default-0": "10\ta\tok\n"}, header, |
| 133 | + "failure", 1, "failed before testing"), |
| 134 | + ("a known-red member that passes is named", {"timings-macos-default-0": "10\ta\tok\n"}, |
| 135 | + header + "macos-default\ta\t#3\n", "success", 0, "passing now"), |
| 136 | + ] |
| 137 | + failed = 0 |
| 138 | + for name, rows, known, result, want_status, want_text in cases: |
| 139 | + status, text = run(rows, known, result) |
| 140 | + ok = status == want_status and want_text in text |
| 141 | + print(f"{'ok' if ok else 'FAIL'}: {name}") |
| 142 | + if not ok: |
| 143 | + failed += 1 |
| 144 | + print(text) |
| 145 | + return 1 if failed else 0 |
| 146 | + |
| 147 | + |
| 148 | +def main(argv: list[str]) -> int: |
| 149 | + if argv[:1] == ["selftest"]: |
| 150 | + return selftest() |
| 151 | + if argv[:1] == ["report"] and len(argv) >= 3: |
| 152 | + result = "" |
| 153 | + if "--members-result" in argv: |
| 154 | + result = argv[argv.index("--members-result") + 1] |
| 155 | + return report(pathlib.Path(argv[1]), pathlib.Path(argv[2]), result) |
| 156 | + print(__doc__) |
| 157 | + return 2 |
| 158 | + |
| 159 | + |
| 160 | +if __name__ == "__main__": |
| 161 | + sys.exit(main(sys.argv[1:])) |
0 commit comments