diff --git a/.github/workflows/gc-root-dominance.yml b/.github/workflows/gc-root-dominance.yml index 9f99738a70..b83116955e 100644 --- a/.github/workflows/gc-root-dominance.yml +++ b/.github/workflows/gc-root-dominance.yml @@ -345,13 +345,14 @@ jobs: - name: Check root-store dominance (dependency-scale) run: | set -euo pipefail - # Floors from the corpus as of this commit (81 modules, ~12900 - # functions, ~7700 root stores), set below that with room for the - # dependency's own churn. `zod` growing is fine; `zod` no longer - # compiling natively is the finding, and these are what make it one. + # The corpus retained 81 modules and ~14,000 root stores while + # compiler pruning reduced its function count from 12,909 to a + # stable 5,615-5,920. Keep the breadth floor below that measured + # population; module and root-store floors independently reject a + # compile that silently loses the dependency or the gate's subject. python3 scripts/gc_root_dominance_check.py ir-corpus-dep \ --moving-only \ - --min-files 60 --min-binds 4000 --min-funcs 6000 \ + --min-files 60 --min-binds 4000 --min-funcs 5000 \ --allowlist scripts/gc_root_dominance_allowlist.json \ --seeded-violations 40 \ -v @@ -362,7 +363,7 @@ jobs: python3 scripts/gc_root_dominance_check.py ir-corpus-dep \ --unrooted-allocas \ --moving-only \ - --min-files 60 --min-binds 4000 --min-funcs 6000 \ + --min-files 60 --min-binds 4000 --min-funcs 5000 \ --allowlist scripts/gc_root_dominance_allowlist.json \ -v @@ -401,7 +402,7 @@ jobs: set -euo pipefail python3 scripts/gc_root_dominance_check.py ir-corpus-dep \ --stale-registers --moving-only \ - --min-files 60 --min-binds 4000 --min-funcs 6000 \ + --min-files 60 --min-binds 4000 --min-funcs 5000 \ --max-stale 118 - name: Upload the IR corpus on failure diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc7a369410..e884953480 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -502,6 +502,17 @@ jobs: python3 scripts/check_llvm_corpus_currency.py --self-test python3 scripts/check_llvm_corpus_currency.py + # #9925. Both native root-dominance corpus generators scraped the + # production statepoint pass constant from inprocess.rs, so splitting + # that module left the scheduled gate unable to emit either corpus. The + # shared reader follows the unique declaration across codegen modules; + # keep its parser and the live repository lookup in required lint. + - name: Statepoint pass source reader + if: ${{ !cancelled() }} + run: | + python3 scripts/read_statepoint_rewrite_passes.py --self-test + python3 scripts/read_statepoint_rewrite_passes.py --check + # #7877, second round. The knob audit above covers env-var names; it says # nothing about the paths and numbers the same pages state. Both drifted: # the memory-model source map pointed at a `gc.rs` deleted in the module diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index cf5d7f881b..d982dc1d45 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -122,16 +122,15 @@ # function into one and producing exactly the line-order false positives the # docstring above says real dominance avoids. # -# `-` is in the identifier class because LLVM's own identifiers allow it and -# LLVM's own passes USE it. `rewrite-statepoints-for-gc` splits critical edges -# into landing pads and names the halves `eh.lpad.8.split-lp`, -# `…split-lp.split-lp`, and so on: 677 such labels in a 21-module native -# corpus. Under the old class those lines matched LABEL_SHAPED_RE but not -# LABEL_RE, so every native module raised MalformedIR and the mode could not -# read its own corpus. (Loudly, at least — the parser's refusal-not-skip rule -# working as designed.) Widening is safe for the shadow corpus: perry's writer -# emits no hyphens, so no line changes classification there. -LABEL_RE = re.compile(r"^([-\w.$][-\w.$]*):\s*(?:;.*)?$") +# `-` is in the bare identifier class because LLVM's own identifiers allow it +# and LLVM's own passes USE it. LLVM quotes labels that contain other bytes, +# including the `$`-bearing names produced when a repsel-specialised function +# is inlined. A declaration prints as `"name$part":` and its references as +# `%"name$part"`; both must normalize to the same CFG key. Keep escape spelling +# intact rather than decoding it: LLVM prints the same spelling at declaration +# and use sites, and equality is the only operation this parser needs. +LLVM_LABEL_TOKEN = r'(?:"(?:[^"\\]|\\.)*"|[-\w.$]+)' +LABEL_RE = re.compile(rf"^({LLVM_LABEL_TOKEN}):\s*(?:;.*)?$") # Anything that ends in `:` and is not an instruction is label-SHAPED. If the # strict form above declines it, that is a parser gap and must be loud. LABEL_SHAPED_RE = re.compile(r"^[^\s=]+:\s*(?:;.*)?$") @@ -157,17 +156,28 @@ BIND_RE = re.compile(r"call void @js_shadow_slot_bind\(i32 (\d+), ptr %([\w.$]+)\)") CLEAR_RE = re.compile(r"call void @js_shadow_slot_set\(i32 (\d+), i64 0\)") STORE_RE = re.compile(r"^\s*store\s+([\w\[\]x* ]+?)\s+([^,]+),\s*ptr %([\w.$]+)") -# `[-\w.$]` throughout, for the reason spelled out on LABEL_RE: LLVM's own -# `split-lp` landing-pad labels carry hyphens, and a branch regex that cannot -# name them drops the edge rather than failing. -BR_UNCOND_RE = re.compile(r"^\s*br label %([-\w.$]+)") -BR_COND_RE = re.compile(r"^\s*br i1 [^,]+, label %([-\w.$]+), label %([-\w.$]+)") -SWITCH_LABEL_RE = re.compile(r"label %([-\w.$]+)") +# All CFG-edge readers use the same token grammar as LABEL_RE. A narrower +# branch regex drops the edge rather than failing, which makes reachable blocks +# look dead and can turn a real dominance violation into a clean result. +BR_UNCOND_RE = re.compile(rf"^\s*br label %({LLVM_LABEL_TOKEN})") +BR_COND_RE = re.compile( + rf"^\s*br i1 [^,]+, label %({LLVM_LABEL_TOKEN}), " + rf"label %({LLVM_LABEL_TOKEN})" +) +SWITCH_LABEL_RE = re.compile(rf"label %({LLVM_LABEL_TOKEN})") # Invoke edges (#7302): normal destination + unwind destination. The invoke # terminates its block; the continuation label follows immediately in the # emitted text and both successors must appear in the CFG or the landing pad # (and everything reached through it) would be dropped as unreachable. -INVOKE_EDGE_RE = re.compile(r"\binvoke\b.*\bto label %([-\w.$]+) unwind label %([-\w.$]+)") +INVOKE_EDGE_RE = re.compile( + rf"\binvoke\b.*\bto label %({LLVM_LABEL_TOKEN}) unwind label " + rf"%({LLVM_LABEL_TOKEN})" +) + + +def llvm_label_name(token): + """Normalize a bare or quoted LLVM block token for CFG comparisons.""" + return token[1:-1] if token.startswith('"') else token # ------------------------------------------------- statepoint IR vocabulary @@ -373,7 +383,7 @@ def parse_file(path): continue lm = LABEL_RE.match(line) if lm: - curblk = lm.group(1) + curblk = llvm_label_name(lm.group(1)) if curblk not in cur.insns: cur.blocks.append(curblk) cur.insns[curblk] = [] @@ -422,21 +432,21 @@ def build_cfg(f): t = ins.text m = BR_COND_RE.match(t) if m: - f.succs[b].add(m.group(1)) - f.succs[b].add(m.group(2)) + f.succs[b].add(llvm_label_name(m.group(1))) + f.succs[b].add(llvm_label_name(m.group(2))) continue m = BR_UNCOND_RE.match(t) if m: - f.succs[b].add(m.group(1)) + f.succs[b].add(llvm_label_name(m.group(1))) continue if t.strip().startswith("switch"): for lbl in SWITCH_LABEL_RE.findall(t): - f.succs[b].add(lbl) + f.succs[b].add(llvm_label_name(lbl)) continue m = INVOKE_EDGE_RE.search(t) if m: - f.succs[b].add(m.group(1)) - f.succs[b].add(m.group(2)) + f.succs[b].add(llvm_label_name(m.group(1))) + f.succs[b].add(llvm_label_name(m.group(2))) for b, ss in list(f.succs.items()): for s in ss: f.preds[s].add(b) @@ -3517,7 +3527,9 @@ def _is_phi(ins): return body.strip().startswith("phi ") -_PHI_EDGE_RE = re.compile(r"\[\s*([^,\[\]]+?)\s*,\s*%([-\w.$]+)\s*\]") +_PHI_EDGE_RE = re.compile( + rf"\[\s*([^,\[\]]+?)\s*,\s*%({LLVM_LABEL_TOKEN})\s*\]" +) def phi_incoming(ins): @@ -3525,7 +3537,10 @@ def phi_incoming(ins): edges, in LLVM's printed order. `operand_text` is `%reg` for a register operand or the constant's own spelling (`0.000000e+00`, `null`, ...) — never itself tainted, which is why callers filter on the leading `%`.""" - return _PHI_EDGE_RE.findall(ins.text) + return [ + (value, llvm_label_name(predecessor)) + for value, predecessor in _PHI_EDGE_RE.findall(ins.text) + ] def transparent_use_graph(f): @@ -4251,16 +4266,28 @@ def _sp(tok="tok", callee="js_gc_loop_safepoint", live=()): } """ -# A repsel specialisation, whose name LLVM must quote. Skipped silently for as -# long as this checker has existed; 175 of 2452 defines in the native corpus. +# A repsel specialisation whose function and inlined block names LLVM must +# quote. The phi's tainted edge is the quoted block, so this fixture also goes +# false-clean if declarations, branch targets, or phi predecessors disagree on +# how `%"name$part"` is normalized. _SELFTEST_SP_QUOTED_NAME = """\ define internal double @"perry_fn_selftest__probe$typed_f64"(double %a) gc "statepoint-example" { entry.0: %rs4gc.b1 = bitcast double %a to i64 %rs4gc.s1 = inttoptr i64 %rs4gc.b1 to ptr addrspace(1) %raw = ptrtoint ptr addrspace(1) %rs4gc.s1 to i64 + br i1 true, label %"inlined$pshape.exit", label %safe.1 + +"inlined$pshape.exit": ; preds = %entry.0 __SAFEPOINT__ - %r = call double @js_object_get_field_by_name_f64(i64 %raw, i64 0) + br label %join.2 + +safe.1: ; preds = %entry.0 + br label %join.2 + +join.2: ; preds = %"inlined$pshape.exit", %safe.1 + %merged = phi i64 [ %raw, %"inlined$pshape.exit" ], [ 0, %safe.1 ] + %r = call double @js_object_get_field_by_name_f64(i64 %merged, i64 0) ret double %r } """.replace("__SAFEPOINT__", _sp()) @@ -4562,9 +4589,27 @@ def statepoint_self_test(): "names bare, LLVM's printer quotes them).", file=sys.stderr) ok = False + elif funcs[0].blocks != [ + "entry.0", "inlined$pshape.exit", "safe.1", "join.2"]: + print("self-test FAIL: a quoted basic-block declaration must " + "normalize to the same name used by its CFG references, got " + f"{funcs[0].blocks!r}", file=sys.stderr) + ok = False + elif funcs[0].succs["entry.0"] != {"inlined$pshape.exit", "safe.1"}: + print("self-test FAIL: a conditional branch to a quoted block " + f"must retain both CFG edges, got {funcs[0].succs['entry.0']!r}", + file=sys.stderr) + ok = False + elif phi_incoming(funcs[0].insns["join.2"][0]) != [ + ("%raw", "inlined$pshape.exit"), ("0", "safe.1")]: + print("self-test FAIL: a quoted phi predecessor must normalize to " + "the declared block name, got " + f"{phi_incoming(funcs[0].insns['join.2'][0])!r}", + file=sys.stderr) + ok = False if len(_scan_statepoints([paths["quoted"]], moving_only=True)) != 1: - print("self-test FAIL: the quoted-name fixture carries the same " - "planted hazard as the unrooted one and must report it", + print("self-test FAIL: the quoted-name fixture carries a planted " + "phi-edge hazard through the quoted block and must report it", file=sys.stderr) ok = False diff --git a/scripts/gc_root_dominance_corpus.sh b/scripts/gc_root_dominance_corpus.sh index 3df473b14c..1102a74ca6 100755 --- a/scripts/gc_root_dominance_corpus.sh +++ b/scripts/gc_root_dominance_corpus.sh @@ -47,8 +47,9 @@ # # That pass string is single-sourced from the Rust # (`STATEPOINT_REWRITE_PASSES`) and checked, not copied: a reproduction of a -# pipeline that has silently drifted is a corpus about nothing. See -# `rs4gc_pass_string`. +# pipeline that has silently drifted is a corpus about nothing. The shared +# reader searches perry-codegen's Rust sources for the unique declaration, so +# moving the constant between modules cannot darken both corpus generators. # # THE CORPUS DOES NOT LINK, and that is load-bearing (#8810) # --------------------------------------------------------- @@ -206,30 +207,7 @@ fi # corpus generated by a pass pipeline production stopped using is a corpus # about nothing, and it would report a serene zero. rs4gc_pass_string() { - local src="crates/perry-codegen/src/inprocess.rs" - if [ ! -f "$src" ]; then - echo "::error::$src not found; run from the repository root" >&2 - return 1 - fi - local value - # rustfmt may keep this declaration on one line or wrap the string onto the - # following line. Read the declaration as a record so formatting cannot - # silently disconnect this corpus from production's pass pipeline. - value="$(perl -0777 -ne ' - if (/^pub\(crate\) const STATEPOINT_REWRITE_PASSES: &str\s*=\s*"([^"]+)";/m) { - print "$1\n"; - } - ' "$src")" - if [ -z "$value" ]; then - echo "::error::could not read STATEPOINT_REWRITE_PASSES out of $src." >&2 - echo "The native corpus reproduces production's statepoint rewrite, and it" >&2 - echo "single-sources the pass string from that const so the two cannot" >&2 - echo "drift. If the const was renamed or reformatted, update this reader --" >&2 - echo "do NOT hardcode the string here, which is the drift this exists to" >&2 - echo "prevent." >&2 - return 1 - fi - printf '%s\n' "$value" + python3 scripts/read_statepoint_rewrite_passes.py } find_opt() { diff --git a/scripts/gc_root_dominance_dep_corpus.sh b/scripts/gc_root_dominance_dep_corpus.sh index 0c719872e4..43e4bf79fb 100755 --- a/scripts/gc_root_dominance_dep_corpus.sh +++ b/scripts/gc_root_dominance_dep_corpus.sh @@ -128,11 +128,13 @@ fi # # So the registry is the import graph and this is the check on it: the module # name perry derives from a path is the path with every non-alphanumeric -# character replaced by `_`, so each source names exactly one expected `.ll`. +# character replaced by `_`. Release worktrees can add their directory name as +# a prefix, so require exactly one emitted module with the source-path suffix. dark=() for src in test-files/gc-dep-corpus/*.ts; do sanitized="$(printf '%s' "$src" | tr -c 'A-Za-z0-9' '_')" - if [ ! -f "$OUTDIR/dep__${sanitized}.ll" ]; then + matches=("$OUTDIR"/dep__*"${sanitized}.ll") + if [ "${#matches[@]}" -ne 1 ] || [ ! -f "${matches[0]}" ]; then dark+=("$src") fi done diff --git a/scripts/gc_root_dominance_dep_native_corpus.sh b/scripts/gc_root_dominance_dep_native_corpus.sh index 8d988358fd..60692c47f9 100755 --- a/scripts/gc_root_dominance_dep_native_corpus.sh +++ b/scripts/gc_root_dominance_dep_native_corpus.sh @@ -29,18 +29,11 @@ case "$PERRY_BIN" in /*) ;; *) PERRY_BIN="$PWD/$PERRY_BIN" ;; esac [ -x "$PERRY_BIN" ] || { echo "::error::$PERRY_BIN not found or not executable" >&2; exit 2; } ENTRY="${ENTRY:-test-files/gc-dep-corpus/main.ts}" -# Single-sourced from the Rust const, never retyped (same reason the curated -# script gives: a fourth copy is how the pass string drifts from production). -# Join continuation lines up to the `;` first: rustfmt wraps the initializer -# when the line grows (#8068 did), and a single-line match then reads nothing. -PASSES="$(awk '/const STATEPOINT_REWRITE_PASSES: &str/ { - buf = $0 - while (buf !~ /;[[:space:]]*$/ && (getline line) > 0) buf = buf " " line - if (match(buf, /"[^"]*"/)) print substr(buf, RSTART + 1, RLENGTH - 2) - exit - }' crates/perry-codegen/src/inprocess.rs)" -[ -n "$PASSES" ] || { echo "could not read STATEPOINT_REWRITE_PASSES" >&2; exit 2; } -OPT_BIN="${PERRY_LLVM_OPT:-/opt/homebrew/opt/llvm/bin/opt}" +# Single-sourced from the Rust const through the same reader as the curated +# native corpus. The reader requires one literal declaration anywhere under +# perry-codegen/src, so a module split cannot leave this arm on a stale path. +PASSES="$(python3 scripts/read_statepoint_rewrite_passes.py)" || exit 2 +OPT_BIN="${PERRY_LLVM_OPT:-}" if [ ! -x "$OPT_BIN" ]; then for c in "${LLVM_SYS_221_PREFIX:-}/bin/opt" /opt/homebrew/opt/llvm/bin/opt /usr/local/opt/llvm/bin/opt; do [ -n "$c" ] && [ -x "$c" ] && OPT_BIN="$c" && break diff --git a/scripts/read_statepoint_rewrite_passes.py b/scripts/read_statepoint_rewrite_passes.py new file mode 100755 index 0000000000..a0dd1fa13b --- /dev/null +++ b/scripts/read_statepoint_rewrite_passes.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Read production's statepoint pass pipeline from its unique Rust constant.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + + +REPO = Path(__file__).resolve().parent.parent +CODEGEN_SRC = REPO / "crates" / "perry-codegen" / "src" +DECLARATION = re.compile( + r"\bpub\(crate\)\s+const\s+STATEPOINT_REWRITE_PASSES\s*:\s*&str\s*=\s*" + r'"([^"\\]*)"\s*;', + re.MULTILINE | re.DOTALL, +) + + +def extract(text: str) -> str | None: + match = DECLARATION.search(text) + return match.group(1) if match else None + + +def find_declaration(root: Path = CODEGEN_SRC) -> tuple[Path, str]: + matches = [] + for path in sorted(root.rglob("*.rs")): + value = extract(path.read_text(encoding="utf-8")) + if value is not None: + matches.append((path, value)) + + if len(matches) != 1: + locations = ", ".join(str(path) for path, _value in matches) or "none" + raise ValueError( + "expected exactly one literal STATEPOINT_REWRITE_PASSES declaration " + f"under {root}, found {len(matches)} ({locations})" + ) + path, value = matches[0] + if not value or "rewrite-statepoints-for-gc" not in value: + raise ValueError( + f"{path} does not contain a usable statepoint rewrite pipeline" + ) + return path, value + + +def self_test() -> int: + failures = [] + expected = "always-inline,function(mem2reg),rewrite-statepoints-for-gc" + one_line = ( + 'pub(crate) const STATEPOINT_REWRITE_PASSES: &str = "' + expected + '";' + ) + wrapped = ( + "pub(crate) const STATEPOINT_REWRITE_PASSES: &str =\n" + f' "{expected}";\n' + ) + for label, source in (("one-line", one_line), ("rustfmt-wrapped", wrapped)): + if extract(source) != expected: + failures.append(f"{label} declaration was not read") + if extract(one_line.replace("STATEPOINT_REWRITE_PASSES", "OTHER_PASSES")) is not None: + failures.append("a differently named constant was accepted") + + try: + path, value = find_declaration() + if not path.is_relative_to(CODEGEN_SRC): + failures.append("the repository declaration escaped perry-codegen/src") + if value != extract(path.read_text(encoding="utf-8")): + failures.append("the repository scan disagrees with the source parser") + except (OSError, ValueError) as error: + failures.append(str(error)) + + for failure in failures: + print(f"statepoint-pass reader self-test FAILED: {failure}", file=sys.stderr) + if failures: + return 1 + print("statepoint-pass reader self-test: OK") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + return self_test() + + try: + path, value = find_declaration() + except (OSError, ValueError) as error: + print(f"statepoint-pass reader: {error}", file=sys.stderr) + return 2 + + if args.check: + print( + "statepoint-pass source OK: " + f"{path.relative_to(REPO)} -> {value}" + ) + else: + print(value) + return 0 + + +if __name__ == "__main__": + sys.exit(main())