From 82522fc4accaea09dca18fdf5d89f1db13b8aad6 Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Wed, 14 Oct 2026 22:41:51 -0500 Subject: [PATCH 1/5] CI: run the full main lane on demand (workflow_dispatch), writing nothing of main's A long-lived branch (the Phase 2 integration branch, draft PR #1776) had never run the heavy workers, which run only in the merge queue and on main pushes. `gh workflow run ci.yml --ref ` now runs them. - ci.yml triggers on workflow_dispatch. A dispatch is not a pull request, so every job runs as on a push to main. - Concurrency: a dispatch gets its own group (ci-dispatch-), so it never cancels a push, PR or merge-queue run on any ref, main included, and none of those cancels it. - Outward writes, audited job by job: dev-image is the only ci.yml job with a write permission (packages: write). It advanced ci-main on any `push` event (safe only because push is filtered to main) and now does so only on a push to refs/heads/main. A dispatch pushes only its own ci-; the buildx gha cache is ref-scoped by GitHub, and artifacts stay in the run. - gate-selftests: a dispatch carries no base sha, and `selftests.sh --changed ''` selects 0 rows (measured), a silent skip. An empty base now falls back to origin/main, so a dispatch selects what the ref changes against main, as its PR would. That step's waiver is re-pinned (the scope/detect pin is unchanged). - tools/ci_tier_check.sh gains [dispatch-write]: every ci.yml job holding a write permission, its own or the workflow's (missing counts as the repository default), must match its reviewed pin in WRITERS. Action `uses:` pins are excluded, so Dependabot bumps do not reopen the review. Proven once with planted copies, recorded in the PR (PRINCIPLES section 5, no permanent self-test). Each planted violation went red: a new writer, the ci-main gate loosened, the workflow permissions removed, a stale pin, and the edited select step's stale waiver. A Dependabot-style bump and the tree stayed green. - docs/CI.md: when to dispatch, what runs, the commands, what it never touches, concurrency, where its checks show, and the new rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KC99CmwatKssQYggkBCgwF --- .github/workflows/ci.yml | 27 ++++++++++++++++++----- docs/CI.md | 46 ++++++++++++++++++++++++++++++++++++++++ tools/ci_tier_check.sh | 26 ++++++++++++++++++++--- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48992a60f..3dd4a1dd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,14 @@ on: # true on push AND on merge_group — never `== 'push'`. merge_group: types: [checks_requested] + # The full main lane on demand, for a long-lived branch before it enters the + # queue: `gh workflow run ci.yml --ref ` (docs/CI.md). A dispatch is + # not a pull request, so every heavy worker runs as on a push to main. It + # must never write main's state, because it can run from any ref: the only + # writer in this file is `dev-image`, which advances the rolling ci-main tag + # only on a push to refs/heads/main, and tools/ci_tier_check.sh pins every + # job holding a write permission to its reviewed text. + workflow_dispatch: permissions: contents: read @@ -31,8 +39,12 @@ env: # up (2026-08-18: three main-push runs — 214m/118m/81m — alive at once after # a valgrind flake, none superseding the others). Pairs with the per-job # timeout-minutes, which bounds a single hang; this bounds the *fan-out*. +# A dispatch gets a group of its own (`ci-dispatch-`), so it never +# cancels a push run on the same ref, main's included, nor a PR or merge-queue +# run, and none of those cancels it. A newer dispatch on the same ref cancels +# the older one. concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ci-${{ github.event_name == 'workflow_dispatch' && format('dispatch-{0}', github.ref) || github.event.pull_request.number || github.ref }} cancel-in-progress: true # Pull requests run the release suite plus build, lint/LSP, self-test, @@ -128,15 +140,17 @@ jobs: env: OWNER: ${{ github.repository_owner }} IS_FORK: ${{ github.event.pull_request.head.repo.fork == true }} - # The event is read from the runner's GITHUB_EVENT_NAME, not an - # expression: tools/ci_tier_check.sh reds event-derived expression - # values on a required path (#1264). Only push advances ci-main. + # The event and ref are read from the runner's GITHUB_EVENT_NAME and + # GITHUB_REF, not expressions: tools/ci_tier_check.sh reds + # event-derived expression values on a required path (#1264). Only a + # push to main advances ci-main: a workflow_dispatch from any ref, main + # included, pushes only its own ci-. run: | BASE="ghcr.io/${OWNER,,}/eigenscript-dev" if [ "$IS_FORK" = "true" ]; then echo "image=$BASE:ci-main" >> "$GITHUB_OUTPUT" echo "tags=" >> "$GITHUB_OUTPUT" - elif [ "$GITHUB_EVENT_NAME" = "push" ]; then + elif [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then echo "image=$BASE:ci-${{ github.sha }}" >> "$GITHUB_OUTPUT" echo "tags=$BASE:ci-${{ github.sha }},$BASE:ci-main" >> "$GITHUB_OUTPUT" else @@ -207,6 +221,9 @@ jobs: run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" base='${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}' + # A workflow_dispatch carries no base: select what the dispatched ref + # changes against main, as its PR would. An empty base selects nothing. + [ -n "$base" ] || base=origin/main bash tools/selftests.sh --changed "$base" --list > selection.txt cat selection.txt n=$(sed -n 's/^selftests: enrolment .*; \([0-9][0-9]*\) self-tests selected$/\1/p' selection.txt) diff --git a/docs/CI.md b/docs/CI.md index d6dcae450..55d7bb219 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -189,6 +189,43 @@ on the commit that will land, before it lands — see **Platform tiers** below. #1138 and #1158 both carried lanes only CI could run. Contributors never wait on it and never rebase to satisfy it; the queue does both. +### The main lane on demand (`workflow_dispatch`) + +A long-lived branch, such as an integration branch that lives through many +review rounds, can run the full main lane before it enters the queue: + + gh workflow run ci.yml --ref + gh run list --workflow ci.yml --event workflow_dispatch --branch + +- **What runs.** A dispatch is not a pull request, so every job runs as on a + push to `main`. That includes every heavy worker: the ASan shards, both TSan + variants, the native macOS suite, valgrind, the bench gate, the extension + suites and the db lane. The bench gate compares the branch with `origin/main`. + The changed-gate self-tests select what the branch changes against `main`. +- **When.** Dispatch on an integration branch before it is queued, and again + after a rebase that pulls in a lot. The queue still runs everything on the + candidate commit, so a dispatch is never a substitute for it. +- **What it never touches.** A dispatch writes nothing that belongs to `main`: + - the only `ci.yml` job that can write is `build dev/ci image`. It pushes the + run's own `ci-` image, and only a push to `main` advances the rolling + `ci-main` tag; + - its build cache is scoped by GitHub to the branch; + - artifacts stay in the run. + + `tools/ci_tier_check.sh` pins every `ci.yml` job that holds a write + permission (`[dispatch-write]`, below). +- **Concurrency.** A dispatch has its own group (`ci-dispatch-`). It never + cancels, and is never cancelled by, a push, pull-request or merge-queue run, + `main`'s included; a newer dispatch on the same ref cancels the older one. +- **Where the results show.** The dispatch reports its checks on the branch's + head commit. If the branch has a pull request, those checks appear on it too, + under the same names as the pull-request run's, and a failure there is a real + failure of that check on that commit. + +GitHub accepts a dispatch only for a workflow whose file on the default +branch declares `workflow_dispatch`, so this works once the trigger is on +`main`. + ## Nightly (`.github/workflows/nightly.yml`) - `macos-15-intel`, the full suite — the 35-minute job that used to set the PR @@ -559,6 +596,15 @@ self-tests` job. A missing PyYAML is exit 2, never a pass. - `[uncovered]` — a `ci.yml` job is neither required nor the worker of exactly one required `if: always()` aggregator. A failing non-required job does not stop the queue, yet it colours the badge: the `macos-15-intel` shape. +- `[dispatch-write]` — a `ci.yml` job that holds a write permission (its own, + or the workflow's, where a missing workflow-level block counts as the + repository default, which may write) must match its reviewed pin in + `WRITERS`. `workflow_dispatch` runs every non-PR path from any ref, so a + new writer, or an edit to the writer there is, stays red until someone + re-reviews it for a dispatch from a branch and re-pins it. Every outward + write is gated on `refs/heads/main`. Action pins (`uses:`) are left out of + the hash, so a Dependabot bump does not reopen the review. A pin that + matches no writing job is red. Whether an aggregator's script really fails on every non-success worker result remains a code-review question. The gate's other accepted limits diff --git a/tools/ci_tier_check.sh b/tools/ci_tier_check.sh index 0b618d476..89fcb7de7 100755 --- a/tools/ci_tier_check.sh +++ b/tools/ci_tier_check.sh @@ -24,7 +24,7 @@ def V(code, msg): bad.append(code); print(f"FAIL [{code}] {msg}") req = [l for l in open(os.environ["CT_REQ"], encoding="utf-8").read().split("\n") if l and not l.startswith("#")] if not req or len(req) != int(os.environ["CT_GREP"]) or len(set(req)) != len(req): V("vacuous", f"required-checks.txt: parsed {len(req)} names, {len(set(req))} distinct, grep counts {os.environ['CT_GREP']}") -REQ = set(req); W = {} +REQ = set(req); W = {}; PERMS = {} for fn in sorted(os.listdir(os.environ["CT_WF"])): if not fn.endswith((".yml", ".yaml")): continue try: doc = yaml.safe_load(open(os.path.join(os.environ["CT_WF"], fn), encoding="utf-8")) @@ -34,6 +34,7 @@ for fn in sorted(os.listdir(os.environ["CT_WF"])): on = {on: {}} if isinstance(on, str) else {str(k): {} for k in on} if isinstance(on, list) else \ {str(k): (v if isinstance(v, dict) else {}) for k, v in (on or {}).items()} W[fn] = (on, doc["jobs"], doc.get("env") or {}) + PERMS[fn] = doc.get("permissions", "write-all") # absent: the repository default, which may write def triggers(on, ev): if ev not in on: return False c = on[ev] @@ -51,7 +52,7 @@ EV = re.compile(r"\bgithub\b(?!\s*(?:\.\s*(?!" + EVP + r")\w|\[\s*['\"](?!" + EV IND = re.compile(r"\b(env|vars)\s*(?:\.\s*|\[\s*['\"])([\w-]+)['\"]?\s*\]?|\b(needs|steps)\s*(?:\.\s*|\[\s*['\"])([\w-]+)['\"]?\s*\]?" r"\s*(?:\.\s*|\[\s*['\"])(outputs|result|outcome|conclusion)['\"]?\s*\]?(?:\s*(?:\.\s*|\[\s*['\"])([\w-]+)['\"]?\s*\]?)?", re.I) WAIVE = {("ci.yml", "scope", "detect"): "35784ca855b3e3fd", - ("ci.yml", "gate-selftests", "select"): "411e6c561962764a"} + ("ci.yml", "gate-selftests", "select"): "cd5e284eee927730"} used = set() def exprs(v): return re.findall(r"\$\{\{(.*?)\}\}", str(v), re.S) def bad_expr(e, wf, jid, pr_ok, depth=0): @@ -170,10 +171,29 @@ for jid in sorted(HEAVY_WORKERS): V("heavy-missing", f"ci.yml:{jid}: declared heavy worker is missing") elif cond(ci[jid].get("if", "")) != "github.event_name != 'pull_request'": V("heavy-worker", f"ci.yml:{jid}: worker must skip only pull_request (and therefore run on merge_group and push)") +# workflow_dispatch runs every non-PR path of ci.yml from ANY ref, so a job that +# can write outside its run must be pinned to its reviewed text. A new writer, or +# an edit to the one there is, stays red until someone re-reviews it for a dispatch +# from a branch (every outward write gated on refs/heads/main) and re-pins it here. +# Action pins (`uses:`) are left out of the hash, so a Dependabot bump does not +# reopen the review. +WRITERS = {"dev-image": "d98ad4b818cdc372"} # pushes ci- from any ref; ci-main only on a push to refs/heads/main +def writes(p): return p == "write-all" or (isinstance(p, dict) and "write" in p.values()) +def writer_pin(j): + body = {k: v for k, v in j.items() if k != "steps"} + body["steps"] = [{k: v for k, v in st.items() if k != "uses"} if isinstance(st, dict) else st for st in j.get("steps") or []] + return hashlib.sha256(yaml.safe_dump(body, sort_keys=True).encode()).hexdigest()[:16] +wrote = set() +for jid, j in ci.items(): + if not writes(j["permissions"] if "permissions" in j else PERMS["ci.yml"]): continue + wrote.add(jid) + if WRITERS.get(jid) != writer_pin(j): + V("dispatch-write", f"ci.yml:{jid} can write outside its run and is not a pinned, reviewed writer (its text hashes to {writer_pin(j)}): workflow_dispatch runs it from any ref, so gate every outward write on refs/heads/main, then pin it in WRITERS") +if set(WRITERS) - wrote: V("dispatch-write", f"writer pin(s) {sorted(set(WRITERS) - wrote)} match no ci.yml job that can write: stale; remove them") if set(WAIVE) - used: V("event-condition", f"waiver(s) {sorted(set(WAIVE) - used)} matched nothing — stale; remove or re-pin") if bad: print(f"ci-tier: FAIL — {len(bad)} violation(s): {' '.join(sorted(set(bad)))}"); sys.exit(1) print(f"ci-tier: OK — ci.yml jobs={len(ci)} (awk={os.environ['CT_AWK']}) required={counts['required']} worker={counts['worker']}; " - f"{len(req)} required names, each produced once on pull_request+merge_group; {len(path)} jobs on required paths, no push-only condition, no continue-on-error") + f"{len(req)} required names, each produced once on pull_request+merge_group; {len(path)} jobs on required paths, no push-only condition, no continue-on-error; {len(wrote)} writing job(s), each pinned") PY } live() { From f39bff9cf28892aa4a6404f39643a81c925d5f7f Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Wed, 14 Oct 2026 22:43:21 -0500 Subject: [PATCH 2/5] docs/CI.md: GitHub reads the dispatch trigger from the dispatched ref's ci.yml Measured: a dispatch of ci.yml on ci-dispatch (whose copy declares the trigger) was accepted before merge (runs 38677615755, cancelled by the newer 38677618300 in the same per-ref dispatch group), while one on the stale branch cloud-1159 (no trigger in its copy) was refused with HTTP 422, "Workflow does not have 'workflow_dispatch' trigger". A long-lived branch must take this change (rebase or merge main) before its first dispatch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KC99CmwatKssQYggkBCgwF --- docs/CI.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/CI.md b/docs/CI.md index 55d7bb219..9b1062fae 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -222,9 +222,10 @@ review rounds, can run the full main lane before it enters the queue: under the same names as the pull-request run's, and a failure there is a real failure of that check on that commit. -GitHub accepts a dispatch only for a workflow whose file on the default -branch declares `workflow_dispatch`, so this works once the trigger is on -`main`. +GitHub reads the trigger from the dispatched ref's own copy of `ci.yml`. A +branch whose copy predates the trigger is refused with HTTP 422, "Workflow +does not have 'workflow_dispatch' trigger" (measured on a stale branch). So +rebase or merge `main` into a long-lived branch before its first dispatch. ## Nightly (`.github/workflows/nightly.yml`) From 68ed2aae420e5ae2abdd7cd6a9f3437995e512f6 Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Thu, 15 Oct 2026 00:39:44 -0500 Subject: [PATCH 3/5] CI dispatch: check the properties, not the writer's text (/code-review of #1785) The altitude correction (finding 13): [dispatch-write] pinned the writing job's text, so a dangerous edit cleared with the same one-line re-pin as a harmless one (plant A, re-pinned with the printed 187c693a40a0ae19, was green), while action swaps, a PAT and workflow-level defaults/BASH_ENV never reached the pin. WRITERS and its second hash recipe are deleted, and tools/ci_tier_check.sh checks what the writes do: - [ci-main]: the step that computes the image push's tags is RUN, under bash -e, across event x ref x tip (52 runs, with fork PRs and an unreadable tip). ci-main must appear exactly on a push to refs/heads/main at main's tip. on.push must be exactly [main]; each push takes its tags from that step; a custom shell, BASH_ENV/ENV/GITHUB_* overrides and unevaluable expressions are refused. - [writer-actions]: a job that can write uses only checkout, buildx, login and build-push (compared without @ref), no ./ or docker:// action, and no script but the tag step. - [secrets]: ci.yml reads no secret but GITHUB_TOKEN, in dot or bracket syntax, and passes none to a reusable workflow. - [event-condition] is now an ALLOWLIST of event-invariant github fields; ref_protected, ['ref_protected'] and workflow_ref were green before. 19 plants, recorded in the PR, before (the f39bff9c checker) and after: plant B (timeout 30 -> 31) is now green, and everything that slipped past the pin is red. The matrix found finding 8 in the tag step on its first run: a re-run of an older main push moved ci-main back. ci-main now moves only when `git ls-remote origin refs/heads/main` equals this SHA (not RUN_ATTEMPT), and an unreadable tip fails loudly. Also: - bench measures a dispatch against the branch's merge-base with main (the queue's measure), not main's tip; - dispatch groups never cancel in progress: a newer dispatch waits, a third replaces the waiting one before any job starts, so no cancelled dispatch leaves aggregators failing under required names; - the README badge reads push runs only (?branch=main&event=push); - docs/CI.md: a dispatch runs the branch's own ci.yml (its PR run already could); what a dispatch writes, on main too; one run per group, case-insensitive group names, a later push not stopping a dispatch; the rollup leaving dispatch suites out (measured on 82522fc4: 49 check runs, no rollup); the rule list; the dispatch self-test base; - precheck's comment and SKIP message cover the dispatch base. Closes #1787 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KC99CmwatKssQYggkBCgwF --- .github/workflows/ci.yml | 71 +++++++++++++------- README.md | 2 +- docs/CI.md | 117 +++++++++++++++++++++++--------- tools/ci_tier_check.sh | 140 +++++++++++++++++++++++++++++++-------- tools/precheck.sh | 5 +- 5 files changed, 251 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3dd4a1dd0..914af4619 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,12 +13,15 @@ on: merge_group: types: [checks_requested] # The full main lane on demand, for a long-lived branch before it enters the - # queue: `gh workflow run ci.yml --ref ` (docs/CI.md). A dispatch is - # not a pull request, so every heavy worker runs as on a push to main. It - # must never write main's state, because it can run from any ref: the only - # writer in this file is `dev-image`, which advances the rolling ci-main tag - # only on a push to refs/heads/main, and tools/ci_tier_check.sh pins every - # job holding a write permission to its reviewed text. + # queue: `gh workflow run ci.yml --ref ` (docs/CI.md, #1787). A + # dispatch is not a pull request, so every heavy worker runs as on a push to + # main. It runs the DISPATCHED ref's own copy of this file, so what follows + # holds for main's text; a branch that edits it can change it, exactly as its + # pull request run already could. In this text the one name main owns that is + # written here, the rolling ci-main tag, moves only on a push to + # refs/heads/main at main's tip; every non-fork ref pushes its own ci-. + # tools/ci_tier_check.sh checks those properties ([ci-main], [writer-actions], + # [secrets]). workflow_dispatch: permissions: @@ -34,18 +37,22 @@ permissions: env: ASAN_SHARDS: 3 -# Only one CI run per ref in flight: a new push (or PR update) cancels the -# older run for the same branch. Prevents superseded/hung runs from piling +# One run per concurrency group in flight: a new push to main (or a PR update) +# cancels the older run in its group. Prevents superseded/hung runs from piling # up (2026-08-18: three main-push runs — 214m/118m/81m — alive at once after # a valgrind flake, none superseding the others). Pairs with the per-job # timeout-minutes, which bounds a single hang; this bounds the *fan-out*. -# A dispatch gets a group of its own (`ci-dispatch-`), so it never -# cancels a push run on the same ref, main's included, nor a PR or merge-queue -# run, and none of those cancels it. A newer dispatch on the same ref cancels -# the older one. +# A dispatch has its own group (`ci-dispatch-`), apart from push, PR and +# merge-queue runs, main's included. GitHub compares group names +# case-insensitively, so refs differing only in case share one. A dispatch is +# never cancelled in progress: a newer dispatch on the same ref waits, and a +# third replaces the waiting one before it starts any job. So no cancelled +# dispatch leaves its `if: always()` aggregators failing under required names. +# Nothing else stops a running dispatch either, a later push to its branch +# included: cancel it with `gh run cancel `. concurrency: group: ci-${{ github.event_name == 'workflow_dispatch' && format('dispatch-{0}', github.ref) || github.event.pull_request.number || github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} # Pull requests run the release suite plus build, lint/LSP, self-test, # precheck, and docs gates. Heavy variants report their required names with a @@ -133,8 +140,10 @@ jobs: # GHCR requires a lowercase owner; compute the ref once and hand it to # the downstream jobs through a job output. `image` is what the Linux - # legs run in; `tags` is what this job pushes (empty on fork PRs — - # nothing is pushed; main pushes also advance the rolling ci-main tag). + # legs run in; `tags` is what this job pushes: empty on fork PRs, this + # commit's ci- on every other run, and the rolling ci-main tag on a + # push to main at main's tip only. tools/ci_tier_check.sh runs this step + # across event x ref x tip to check that ([ci-main]). - name: Compute image ref id: ref env: @@ -143,20 +152,28 @@ jobs: # The event and ref are read from the runner's GITHUB_EVENT_NAME and # GITHUB_REF, not expressions: tools/ci_tier_check.sh reds # event-derived expression values on a required path (#1264). Only a - # push to main advances ci-main: a workflow_dispatch from any ref, main - # included, pushes only its own ci-. + # push to main whose SHA is main's tip moves ci-main: a dispatch from + # any ref, main included, and a re-run of an older push push only + # ci-. Do not gate on GITHUB_RUN_ATTEMPT: a re-run at the tip is + # legitimate. run: | BASE="ghcr.io/${OWNER,,}/eigenscript-dev" + SHA="${{ github.sha }}" if [ "$IS_FORK" = "true" ]; then echo "image=$BASE:ci-main" >> "$GITHUB_OUTPUT" echo "tags=" >> "$GITHUB_OUTPUT" - elif [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then - echo "image=$BASE:ci-${{ github.sha }}" >> "$GITHUB_OUTPUT" - echo "tags=$BASE:ci-${{ github.sha }},$BASE:ci-main" >> "$GITHUB_OUTPUT" - else - echo "image=$BASE:ci-${{ github.sha }}" >> "$GITHUB_OUTPUT" - echo "tags=$BASE:ci-${{ github.sha }}" >> "$GITHUB_OUTPUT" + exit 0 + fi + TAGS="$BASE:ci-$SHA" + if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then + # Move ci-main only while this SHA is main's tip, so re-running an + # older push run cannot point it back at an older image. + tip=$(git ls-remote origin refs/heads/main | cut -f1) + [ -n "$tip" ] || { echo "::error::cannot read main's tip; ci-main not moved"; exit 1; } + if [ "$tip" = "$SHA" ]; then TAGS="$TAGS,$BASE:ci-main"; else echo "main has moved to $tip: ci-main stays"; fi fi + echo "image=$BASE:ci-$SHA" >> "$GITHUB_OUTPUT" + echo "tags=$TAGS" >> "$GITHUB_OUTPUT" - name: Build and push dev image if: github.event.pull_request.head.repo.fork != true @@ -1547,12 +1564,20 @@ jobs: # not origin/main — otherwise a PR is charged for the Ir cost of the PRs # ahead of it and falsely rejected (#1273 code review). Read from the # event payload inside the script, never via ${{ }} (tier gate rule). + # On a dispatch the baseline is the branch's merge-base with main: the + # branch's own changes, as the queue will measure them, not main's + # changes since the fork counted against it (or hiding its own). run: | base=origin/main if [ "$GITHUB_EVENT_NAME" = merge_group ]; then base=$(jq -r '.merge_group.base_sha' "$GITHUB_EVENT_PATH") [ -n "$base" ] && [ "$base" != null ] || { echo "merge_group event without base_sha"; exit 1; } git fetch --no-tags --depth=1 origin "$base" + elif [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + if [ "$(git rev-parse --is-shallow-repository)" = true ]; then git fetch --no-tags --unshallow origin main + else git fetch --no-tags origin main; fi + base=$(git merge-base origin/main HEAD) + echo "dispatch: the merge-base of $GITHUB_REF with origin/main ($(git rev-parse origin/main)) is $base" else git fetch --no-tags --depth=1 origin main fi diff --git a/README.md b/README.md index b4eb4cb2f..46cb564af 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- CI + CI Release License: MIT Stars diff --git a/docs/CI.md b/docs/CI.md index 9b1062fae..fd62724cb 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -192,35 +192,57 @@ on it and never rebase to satisfy it; the queue does both. ### The main lane on demand (`workflow_dispatch`) A long-lived branch, such as an integration branch that lives through many -review rounds, can run the full main lane before it enters the queue: +review rounds, can run the full main lane before it enters the queue (#1787): gh workflow run ci.yml --ref gh run list --workflow ci.yml --event workflow_dispatch --branch + gh run list --workflow ci.yml --commit - **What runs.** A dispatch is not a pull request, so every job runs as on a push to `main`. That includes every heavy worker: the ASan shards, both TSan variants, the native macOS suite, valgrind, the bench gate, the extension - suites and the db lane. The bench gate compares the branch with `origin/main`. - The changed-gate self-tests select what the branch changes against `main`. + suites and the db lane. + - The bench gate measures the branch against its merge-base with `main`, so + the branch is charged for its own changes, as the queue will measure them. + - The changed-gate self-tests select what the branch changes against `main`. - **When.** Dispatch on an integration branch before it is queued, and again after a rebase that pulls in a lot. The queue still runs everything on the candidate commit, so a dispatch is never a substitute for it. -- **What it never touches.** A dispatch writes nothing that belongs to `main`: - - the only `ci.yml` job that can write is `build dev/ci image`. It pushes the - run's own `ci-` image, and only a push to `main` advances the rolling - `ci-main` tag; - - its build cache is scoped by GitHub to the branch; - - artifacts stay in the run. - - `tools/ci_tier_check.sh` pins every `ci.yml` job that holds a write - permission (`[dispatch-write]`, below). -- **Concurrency.** A dispatch has its own group (`ci-dispatch-`). It never - cancels, and is never cancelled by, a push, pull-request or merge-queue run, - `main`'s included; a newer dispatch on the same ref cancels the older one. -- **Where the results show.** The dispatch reports its checks on the branch's - head commit. If the branch has a pull request, those checks appear on it too, - under the same names as the pull-request run's, and a failure there is a real - failure of that check on that commit. +- **Which `ci.yml` runs.** The dispatched ref's own copy, with its own + `tools/ci_tier_check.sh`. What follows holds for `main`'s text. A branch that + edits `ci.yml` can change it, exactly as its pull-request run already could, + since a pull request also runs the branch's own workflow file. +- **What it writes.** In `main`'s `ci.yml`, the only job that can write is + `build dev/ci image`: + - Every non-fork run pushes the commit's own `ci-` image tag, a + dispatch included. + - The rolling `ci-main` tag is the one name written here that `main` owns + (fork PRs run in it). It moves only on a push to `main` whose SHA is still + `main`'s tip, so neither a dispatch from any ref nor a re-run of an older + push moves it (`[ci-main]`, below). + - The build cache goes to the run's ref scope. A dispatch on a branch writes + the branch's scope; a dispatch on `main` writes `main`'s, as a push does. + - Artifacts stay in the run. + + A dispatch on `main` also adds runs to `main`'s commit, but the README badge + reads only push runs (`badge.svg?branch=main&event=push`). +- **Concurrency.** There is one run per concurrency group. + - A dispatch's group is `ci-dispatch-`, separate from push, + pull-request and merge-queue runs, `main`'s included. GitHub compares group + names case-insensitively, so refs that differ only in case share one. + - The group never cancels a running dispatch. A newer dispatch on the same + ref waits, and a third replaces the waiting one before it starts any job. + - A later push to the branch does not stop a dispatch either. To drop an + outdated one, run `gh run cancel ` and dispatch again. A cancelled + run's `if: always()` aggregators then conclude failure under their required + names on that commit; that failure records the cancellation, not a test + result. +- **Where the results show.** In the dispatch run itself: `gh run view + `, or `gh run list --commit `. GitHub's status rollup for a + commit, which pull requests and the queue read, leaves out `workflow_dispatch` + suites. (Measured: a commit that carries only a dispatch's 49 check runs has + no rollup at all.) So a dispatch neither appears among a pull request's + checks nor gates it. GitHub reads the trigger from the dispatched ref's own copy of `ci.yml`. A branch whose copy predates the trigger is refused with HTTP 422, "Workflow @@ -516,8 +538,9 @@ index headers and compiles `eigenlsp.c` with the trio in its flag bundle. Main CI did not finish green from 2026-09-16 to 2026-09-22 although every required check passed on every merge: one lane that no pull request had to pass (`macos / macos-15-intel`, main lane only) hit its timeout on nearly every -push. The README badge is the status of the whole `ci.yml` workflow on `main`, -so **any** `ci.yml` job that can fail there colours it, required or not. +push. The README badge is the status of the whole `ci.yml` workflow's latest +push run on `main` (`badge.svg?branch=main&event=push`), so **any** `ci.yml` +job that can fail there colours it, required or not. The fix copies Rust (tiers plus a merge queue), CPython and Go: @@ -579,7 +602,14 @@ self-tests` job. A missing PyYAML is exit 2, never a pass. those must use exactly `github.event_name != 'pull_request'`. A step `if:` may mention the event only as `github.event_name ==/!= 'pull_request'`, or via the PR payload `github.event.pull_request.*` (empty on push and in the - queue alike). Dot and bracket syntax are both read. The same rule covers + queue alike). Beyond those two forms, a condition may read only the `github` + fields that are the same for the push, merge-queue, pull-request and dispatch + runs of this repository: `repository`, `repository_owner`, `repository_id`, + `repository_owner_id`, `server_url`, `api_url`, `graphql_url`, `workspace`, + `workflow`, `job`, `action`, `token` and `retention_days`. Any other field + (such as `ref_protected`, `workflow_ref`, `sha` or `actor`), or a bare + `github`, is red: that is an allowlist (#1787), not a list of known event + fields. Dot and bracket syntax are both read. The same rule covers indirection: an `env`, job `outputs`, workflow `env` or matrix value on a required path may not read the event (outside those two forms), and an `if:` that reads `env.*`, `needs.*.outputs` or `steps.*.outputs` is traced @@ -597,15 +627,36 @@ self-tests` job. A missing PyYAML is exit 2, never a pass. - `[uncovered]` — a `ci.yml` job is neither required nor the worker of exactly one required `if: always()` aggregator. A failing non-required job does not stop the queue, yet it colours the badge: the `macos-15-intel` shape. -- `[dispatch-write]` — a `ci.yml` job that holds a write permission (its own, - or the workflow's, where a missing workflow-level block counts as the - repository default, which may write) must match its reviewed pin in - `WRITERS`. `workflow_dispatch` runs every non-PR path from any ref, so a - new writer, or an edit to the writer there is, stays red until someone - re-reviews it for a dispatch from a branch and re-pins it. Every outward - write is gated on `refs/heads/main`. Action pins (`uses:`) are left out of - the hash, so a Dependabot bump does not reopen the review. A pin that - matches no writing job is red. +- `[ci-main]` — `ci-main` is the one name `ci.yml` writes that `main` owns. + The check finds the step that computes the image push's tags and runs it, + under `bash -e`, across every event (push, pull request, merge queue, + dispatch), ref (`main`, a branch, a pull request's merge ref, the queue's + ref) and tip state (this SHA is `main`'s tip, is not, or the tip cannot be + read), plus fork pull requests. It requires `ci-main` exactly on a push to + `refs/heads/main` at `main`'s tip. Every non-fork run pushing its own + `ci-` is by design. + - `on.push` must be exactly `branches: [main]`. + - Each image push must take its tags from that one step. + - The check refuses what its matrix cannot model: a custom shell + (`defaults:` or `shell:`), `BASH_ENV`/`ENV`/`GITHUB_*` in any env scope, + or an expression it cannot evaluate. +- `[writer-actions]` — a `ci.yml` job that can write uses only the allowlisted + actions `actions/checkout`, `docker/setup-buildx-action`, + `docker/login-action` and `docker/build-push-action`. + - The allowlist compares action identities without the `@ref`, so a + Dependabot bump stays green. + - No `./` or `docker://` action, and no reusable-workflow call. + - No script except the tag step that `[ci-main]` runs. + + A job can write when it holds a write permission: its own, or the workflow's, + where a missing block counts as the repository default, which may write. +- `[secrets]` — `ci.yml` reads no secret but `secrets.GITHUB_TOKEN`, in dot or + bracket syntax, and passes none to a reusable workflow. So `permissions:` + alone decide what it can write. + +These rules were each proven once, when written, against planted copies +recorded in PR #1785 (`docs/PRINCIPLES.md` §5); none is re-proven on every +run. Whether an aggregator's script really fails on every non-success worker result remains a code-review question. The gate's other accepted limits @@ -673,7 +724,9 @@ The suite runs each gate's live check; `tools/selftests.sh --changed origin/main runs checker self-tests whose scripts, fixtures or helpers changed, including uncommitted and untracked files, and `make precheck` includes that selection. The required `gate self-tests (changed rows)` check uses the -PR base SHA or merge-group base SHA, while nightly runs `tools/selftests.sh --all` +PR base SHA or merge-group base SHA, the pushed range on `main`, or +`origin/main` on a `workflow_dispatch` (which carries no base SHA), while +nightly runs `tools/selftests.sh --all` and reports failures through its tracking issue. `tools/selftests.txt` owns the trigger globs, bounded commands and existing result pins; changing it or the driver selects every row. diff --git a/tools/ci_tier_check.sh b/tools/ci_tier_check.sh index 89fcb7de7..3f74d0e99 100755 --- a/tools/ci_tier_check.sh +++ b/tools/ci_tier_check.sh @@ -13,7 +13,7 @@ check() { CT_AWK=$(awk '/^jobs:/ {j=1; next} j && /^[^ #]/ {j=0} j && /^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/ {n++} END {print n+0}' "$WF_DIR/ci.yml") \ CT_GREP=$(grep -cvE '^(#|$)' "$REQ_FILE") CT_WF="$WF_DIR" CT_REQ="$REQ_FILE" python3 - <<'PY' -import hashlib, itertools, os, re, sys; from fnmatch import fnmatchcase +import hashlib, itertools, os, re, subprocess, sys, tempfile; from fnmatch import fnmatchcase def instrument(m): print(f"ci-tier: INSTRUMENT ERROR — {m}; nothing was checked"); sys.exit(2) try: import yaml @@ -24,7 +24,7 @@ def V(code, msg): bad.append(code); print(f"FAIL [{code}] {msg}") req = [l for l in open(os.environ["CT_REQ"], encoding="utf-8").read().split("\n") if l and not l.startswith("#")] if not req or len(req) != int(os.environ["CT_GREP"]) or len(set(req)) != len(req): V("vacuous", f"required-checks.txt: parsed {len(req)} names, {len(set(req))} distinct, grep counts {os.environ['CT_GREP']}") -REQ = set(req); W = {}; PERMS = {} +REQ = set(req); W = {}; PERMS = {}; DOCS = {} for fn in sorted(os.listdir(os.environ["CT_WF"])): if not fn.endswith((".yml", ".yaml")): continue try: doc = yaml.safe_load(open(os.path.join(os.environ["CT_WF"], fn), encoding="utf-8")) @@ -35,6 +35,7 @@ for fn in sorted(os.listdir(os.environ["CT_WF"])): {str(k): (v if isinstance(v, dict) else {}) for k, v in (on or {}).items()} W[fn] = (on, doc["jobs"], doc.get("env") or {}) PERMS[fn] = doc.get("permissions", "write-all") # absent: the repository default, which may write + DOCS[fn] = doc def triggers(on, ev): if ev not in on: return False c = on[ev] @@ -47,8 +48,15 @@ def needs(j): n = j.get("needs") or []; return [n] if isinstance(n, str) else li def always(j): return cond(j.get("if", "")) == "always()" G = r"github\s*(?:\.\s*{0}\b|\[\s*['\"]{0}['\"]\s*\])" PR_ATOM = re.compile(G.format("event_name") + r"\s*[=!]=\s*['\"]pull_request['\"]|['\"]pull_request['\"]\s*[=!]=\s*" + G.format("event_name"), re.I) -PR_BODY = re.compile(G.format("event") + r"\s*(?:\.\s*pull_request\b|\[\s*['\"]pull_request['\"]\s*\])", re.I); EVP = r"(?:event_name|event|ref|ref_name|ref_type|head_ref|base_ref)\b" -EV = re.compile(r"\bgithub\b(?!\s*(?:\.\s*(?!" + EVP + r")\w|\[\s*['\"](?!" + EVP + r")\w+['\"]\s*\]))", re.I) +PR_BODY = re.compile(G.format("event") + r"\s*(?:\.\s*pull_request\b|\[\s*['\"]pull_request['\"]\s*\])", re.I) +# An ALLOWLIST (#1787): a condition may read only github fields that are the same for the +# push, merge_group, pull_request and workflow_dispatch runs of this repository. Any other +# field (ref_protected, workflow_ref, actor, sha, ...) or a bare `github` can select a lane. +INVARIANT = {"repository", "repository_owner", "repository_id", "repository_owner_id", "server_url", + "api_url", "graphql_url", "workspace", "workflow", "job", "action", "token", "retention_days"} +GH = re.compile(r"\bgithub\b(?:\s*\.\s*([A-Za-z_][\w-]*)|\s*\[\s*['\"]([^'\"]*)['\"]\s*\])?", re.I) +def reads_event(t): + return next((m.group(0).strip() for m in GH.finditer(t) if (m.group(1) or m.group(2) or "").lower() not in INVARIANT), None) IND = re.compile(r"\b(env|vars)\s*(?:\.\s*|\[\s*['\"])([\w-]+)['\"]?\s*\]?|\b(needs|steps)\s*(?:\.\s*|\[\s*['\"])([\w-]+)['\"]?\s*\]?" r"\s*(?:\.\s*|\[\s*['\"])(outputs|result|outcome|conclusion)['\"]?\s*\]?(?:\s*(?:\.\s*|\[\s*['\"])([\w-]+)['\"]?\s*\]?)?", re.I) WAIVE = {("ci.yml", "scope", "detect"): "35784ca855b3e3fd", @@ -57,7 +65,8 @@ used = set() def exprs(v): return re.findall(r"\$\{\{(.*?)\}\}", str(v), re.S) def bad_expr(e, wf, jid, pr_ok, depth=0): """Why expression `e` (in wf:jid) may differ between push and merge_group, or None."""; t = PR_BODY.sub("_", PR_ATOM.sub("_", e)) if pr_ok else e - if EV.search(t): return f"`{e.strip()}` reads the event" + hit = reads_event(t) + if hit: return f"`{e.strip()}` reads `{hit}`, not one of the event-invariant github fields" if re.search(r"\b(env|vars|steps|needs)\b", IND.sub("_", t), re.I): return f"`{e.strip()}` reads a context this gate cannot resolve" for m in IND.finditer(t): (ctx, name, kind, job, field, key), j = m.groups(), W[wf][1].get(jid, {}) @@ -73,9 +82,11 @@ def bad_expr(e, wf, jid, pr_ok, depth=0): if v is None: return f"`needs.{job}.outputs.{key}` is not declared" why = next((w for x in exprs(v) for w in [bad_expr(x, wf, job, pr_ok, depth + 1)] if w), None) else: - src = next((yaml.safe_dump(s, sort_keys=True) for s in j.get("steps") or [] if isinstance(s, dict) and s.get("id") == job), None); pin = WAIVE.get((wf, jid, job)) - if pin and src is not None and hashlib.sha256(src.encode()).hexdigest()[:16] == pin: used.add((wf, jid, job)); continue - return f"`steps.{job}.outputs.{key}` comes from a script (not waived, or the waived script changed)" + st = next((s for s in j.get("steps") or [] if isinstance(s, dict) and s.get("id") == job), None); pin = WAIVE.get((wf, jid, job)) + now = None if st is None else hashlib.sha256(yaml.safe_dump(st, sort_keys=True).encode()).hexdigest()[:16] + if pin and now == pin: used.add((wf, jid, job)); continue + return (f"`steps.{job}.outputs.{key}` comes from a script that is not waived" if not pin else + f"`steps.{job}.outputs.{key}` comes from a waived script that changed (it now hashes to {now}): review it, then re-pin WAIVE") if why: return f"{m.group(0).strip()} -> {why}" return None def names(jid, j): @@ -137,7 +148,7 @@ for wf, jid in sorted(path): if isinstance(st, dict) for k, v in (st.get("env") or {}).items()] for k, v in vals: for x in exprs(v): - if EV.search(PR_BODY.sub("_", PR_ATOM.sub("_", x))): + if reads_event(PR_BODY.sub("_", PR_ATOM.sub("_", x))): V("event-condition", f"{wf}:{jid}: `{k}: ${{{{{x}}}}}` derives a value from the event") if j.get("continue-on-error") not in (None, False): V("continue-on-error", f"{wf}:{jid} sets continue-on-error: its failure would not fail the check") for i, st in enumerate(j.get("steps") or []): @@ -171,29 +182,106 @@ for jid in sorted(HEAVY_WORKERS): V("heavy-missing", f"ci.yml:{jid}: declared heavy worker is missing") elif cond(ci[jid].get("if", "")) != "github.event_name != 'pull_request'": V("heavy-worker", f"ci.yml:{jid}: worker must skip only pull_request (and therefore run on merge_group and push)") -# workflow_dispatch runs every non-PR path of ci.yml from ANY ref, so a job that -# can write outside its run must be pinned to its reviewed text. A new writer, or -# an edit to the one there is, stays red until someone re-reviews it for a dispatch -# from a branch (every outward write gated on refs/heads/main) and re-pins it here. -# Action pins (`uses:`) are left out of the hash, so a Dependabot bump does not -# reopen the review. -WRITERS = {"dev-image": "d98ad4b818cdc372"} # pushes ci- from any ref; ci-main only on a push to refs/heads/main +# workflow_dispatch runs every non-PR path of ci.yml from ANY ref (#1787), so the +# writes are checked by what they do, not pinned by their text: +# [ci-main] the rolling ci-main tag is the one name ci.yml writes that main owns +# (fork PRs run in it). The step that computes the pushed tags is RUN across +# event x ref x "this SHA is main's tip"; ci-main must appear exactly on a push to +# refs/heads/main at main's tip (no dispatch, queue run or re-run of an older push). +# ci- tags go out from every non-fork ref by design. What the matrix cannot +# model (a custom shell, BASH_ENV, an expression it cannot evaluate) is refused. +# [writer-actions] a job that can write uses only allowlisted actions (compared +# without @ref), no ./ or docker:// actions, and no script but that tag step. +# [secrets] ci.yml reads no secret but GITHUB_TOKEN, so only permissions: can write. def writes(p): return p == "write-all" or (isinstance(p, dict) and "write" in p.values()) -def writer_pin(j): - body = {k: v for k, v in j.items() if k != "steps"} - body["steps"] = [{k: v for k, v in st.items() if k != "uses"} if isinstance(st, dict) else st for st in j.get("steps") or []] - return hashlib.sha256(yaml.safe_dump(body, sort_keys=True).encode()).hexdigest()[:16] -wrote = set() +ALLOWED = {"actions/checkout", "docker/setup-buildx-action", "docker/login-action", "docker/build-push-action"} +EVENTS = ("push", "pull_request", "merge_group", "workflow_dispatch") +REFS = ("refs/heads/main", "refs/heads/a-branch", "refs/pull/1/merge", "refs/heads/gh-readonly-queue/main/pr-1-0123abc") +def env_value(v, fork): + """A tag-step env value, or None when the matrix cannot evaluate it.""" + known = {"github.repository_owner": "Owner", "github.event.pull_request.head.repo.fork==true": "true" if fork else "false"} + out = str(v) + for x in exprs(out): + if re.sub(r"\s+", "", x) not in known: return None + out = out.replace("${{" + x + "}}", known[re.sub(r"\s+", "", x)]) + return out +def ci_main(jid, j, tag): + """Run the tag step across the matrix; returns how many runs it judged.""" + where = f"ci.yml:{jid} step {tag.get('id')!r}" + if DOCS["ci.yml"].get("defaults") or j.get("defaults") or "shell" in tag: + V("ci-main", f"{where}: a custom shell (defaults: or shell:) can change what the step computes; the matrix runs it under bash -e only"); return 0 + for scope, d in (("workflow", W["ci.yml"][2]), ("job", j.get("env") or {}), ("step", tag.get("env") or {})): + bad_env = [k for k in d if str(k).upper() in ("BASH_ENV", "ENV") or str(k).upper().startswith("GITHUB_")] + if bad_env: V("ci-main", f"{where}: {scope} env sets {bad_env}, which can change what the step computes"); return 0 + script = str(tag.get("run", "")) + odd = sorted({x.strip() for x in exprs(script)} - {"github.sha"}) + if odd or any(env_value(v, False) is None for v in (tag.get("env") or {}).values()): + V("ci-main", f"{where}: uses an expression the matrix cannot evaluate ({odd or 'in its env'})"); return 0 + runs = 0 + with tempfile.TemporaryDirectory() as t: + with open(os.path.join(t, "git"), "w") as f: # a git that answers only `ls-remote origin refs/heads/main` + f.write('#!/bin/sh\n[ "$*" = "ls-remote origin refs/heads/main" ] || { echo "unexpected git $*" >&2; exit 97; }\n' + '[ -n "$TIP" ] || exit 128\nprintf "%s\\trefs/heads/main\\n" "$TIP"\n') + os.chmod(os.path.join(t, "git"), 0o755) + cases = [(e, r, tip, False) for e in EVENTS for r in REFS for tip in (True, False, None)] + cases += [("pull_request", r, True, True) for r in REFS] # fork pull requests + for ev, ref, tip, fork in cases: + sha, out = "1" * 40, os.path.join(t, "out") + open(out, "w").close() + env = {"PATH": t + ":" + os.environ["PATH"], "HOME": t, "GITHUB_EVENT_NAME": ev, "GITHUB_REF": ref, + "GITHUB_SHA": sha, "GITHUB_OUTPUT": out, "TIP": {True: sha, False: "2" * 40, None: ""}[tip], + **{str(k): env_value(v, fork) for k, v in (tag.get("env") or {}).items()}} + p = subprocess.run(["bash", "-e", "-c", re.sub(r"\$\{\{\s*github\.sha\s*\}\}", sha, script)], env=env, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=20) + runs += 1 + tags = [x for line in open(out) if line.startswith("tags=") for x in line[5:].strip().split(",") if x] + has = any(x.endswith(":ci-main") for x in tags) + want = ev == "push" and ref == "refs/heads/main" and tip is True and not fork + what = f"event={ev} ref={ref} tip={'?' if tip is None else tip}{' fork' if fork else ''}" + if p.returncode != 0 and not (tip is None and ev == "push" and ref == "refs/heads/main"): + V("ci-main", f"{where} failed for {what}: {p.stderr.strip()[-200:]}") + elif has != want: + V("ci-main", f"{where} {'pushes' if has else 'does not push'} ci-main for {what}; ci-main is main's: only a push to refs/heads/main at main's tip may move it") + return runs +if W["ci.yml"][0].get("push") != {"branches": ["main"]}: + V("ci-main", f"ci.yml: on.push is {W['ci.yml'][0].get('push')!r}, not exactly branches: [main]") +writers, judged = [], 0 for jid, j in ci.items(): if not writes(j["permissions"] if "permissions" in j else PERMS["ci.yml"]): continue - wrote.add(jid) - if WRITERS.get(jid) != writer_pin(j): - V("dispatch-write", f"ci.yml:{jid} can write outside its run and is not a pinned, reviewed writer (its text hashes to {writer_pin(j)}): workflow_dispatch runs it from any ref, so gate every outward write on refs/heads/main, then pin it in WRITERS") -if set(WRITERS) - wrote: V("dispatch-write", f"writer pin(s) {sorted(set(WRITERS) - wrote)} match no ci.yml job that can write: stale; remove them") + writers.append(jid) + if "uses" in j: V("writer-actions", f"ci.yml:{jid} can write and calls a reusable workflow ({j['uses']}) this check cannot vet") + steps = [st for st in j.get("steps") or [] if isinstance(st, dict)] + push = [st for st in steps if re.sub(r"@[^@]*$", "", str(st.get("uses", ""))).lower() == "docker/build-push-action"] + tag_ids = {m.group(1) for st in push for m in [re.fullmatch(r"\$\{\{\s*steps\.([\w-]+)\.outputs\.tags\s*\}\}", str((st.get("with") or {}).get("tags", "")).strip())] if m} + if len(tag_ids) != len(push) or len(tag_ids) > 1: + V("ci-main", f"ci.yml:{jid}: each image push must take its tags from one step (`tags: ${{{{ steps..outputs.tags }}}}`), the step the matrix runs") + tag = next((st for st in steps if st.get("id") in tag_ids), None) + for i, st in enumerate(steps): + u = str(st.get("uses", "")).strip() + if u and (u.startswith(("./", "docker://")) or re.sub(r"@[^@]*$", "", u).lower() not in ALLOWED): + V("writer-actions", f"ci.yml:{jid} step {i} uses {u!r}: a job that can write uses only {sorted(ALLOWED)} (any @ref), no ./ or docker:// action") + elif "run" in st and st is not tag: + V("writer-actions", f"ci.yml:{jid} step {i} ({st.get('name', '?')}) runs a script in a job that can write; only the tag step may, because the matrix runs it") + if tag is not None: judged += ci_main(jid, j, tag) +if judged == 0: V("ci-main", "no image push in ci.yml took its tags from a step the matrix could run: the ci-main property was not checked") +def strings(x): + if isinstance(x, str): yield x + elif isinstance(x, dict): + for k, v in x.items(): yield from strings(k); yield from strings(v) + elif isinstance(x, list): + for v in x: yield from strings(v) +SECRET = re.compile(r"\bsecrets\b(?:\s*\.\s*([A-Za-z_][\w-]*)|\s*\[\s*['\"]([^'\"]*)['\"]\s*\])?", re.I) +for x in {e for v in strings(DOCS["ci.yml"]) for e in exprs(v)}: + for m in SECRET.finditer(x): + if (m.group(1) or m.group(2) or "").upper() != "GITHUB_TOKEN": + V("secrets", f"ci.yml reads `{m.group(0)}` in `${{{{{x.strip()}}}}}`: only secrets.GITHUB_TOKEN may appear, so permissions: alone decide what ci.yml can write") +for jid, j in ci.items(): + if "secrets" in j: V("secrets", f"ci.yml:{jid} passes secrets ({j['secrets']!r}) to a reusable workflow") if set(WAIVE) - used: V("event-condition", f"waiver(s) {sorted(set(WAIVE) - used)} matched nothing — stale; remove or re-pin") if bad: print(f"ci-tier: FAIL — {len(bad)} violation(s): {' '.join(sorted(set(bad)))}"); sys.exit(1) print(f"ci-tier: OK — ci.yml jobs={len(ci)} (awk={os.environ['CT_AWK']}) required={counts['required']} worker={counts['worker']}; " - f"{len(req)} required names, each produced once on pull_request+merge_group; {len(path)} jobs on required paths, no push-only condition, no continue-on-error; {len(wrote)} writing job(s), each pinned") + f"{len(req)} required names, each produced once on pull_request+merge_group; {len(path)} jobs on required paths, no push-only condition, no continue-on-error; " + f"ci-main only on a push to main at its tip ({judged} event x ref x tip runs); {len(writers)} writing job(s), allowlisted actions only; no secret but GITHUB_TOKEN") PY } live() { diff --git a/tools/precheck.sh b/tools/precheck.sh index acc9a1366..aea22d61e 100644 --- a/tools/precheck.sh +++ b/tools/precheck.sh @@ -118,12 +118,13 @@ lane 0 & p0=$! lane 1 & p1=$! wait "$p0" "$p1" # The self-tests can build scratch fixtures: run serially after the static lanes. -# CI owns this row separately, with the event's base SHA rather than origin/main. +# CI owns this row separately: the PR's or merge group's base SHA, the pushed range on +# main, or origin/main on a workflow_dispatch (which carries no base). i=0 while IFS='|' read -r class cmd; do if [ "$class" = selftest ]; then if [ "${PRECHECK_SELFTESTS:-1}" = 0 ]; then - echo "SKIP|0|CI runs the driver separately with the event base SHA" > "$OUT/$i.st" + echo "SKIP|0|CI runs the driver separately (event base SHA, or origin/main on a dispatch)" > "$OUT/$i.st" else run_gate "$i" "$class" $cmd cat "$OUT/$i.log" From e0b6e01551a74854bbe04a27f93df40774abbd79 Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Thu, 15 Oct 2026 00:48:09 -0500 Subject: [PATCH 4/5] CI dispatch: keep the tag step bash-3.2 clean; exact allowlist wording tools/ci_tier_check.sh now RUNS the dev-image tag step, and precheck runs the checker, so on macOS's bash 3.2 `${OWNER,,}` failed 51 of the 52 matrix runs with "bad substitution" (measured with a bash 3.2 build). The step lowercases with tr instead; the checker and all 19 plants now give the same result under bash 3.2 and bash 5. docs/CI.md: the [event-condition] allowlist is the github fields whose value does not depend on the event (github.token is per run, so "the same in every run" was not exact). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KC99CmwatKssQYggkBCgwF --- .github/workflows/ci.yml | 5 +++-- docs/CI.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 914af4619..ba13ba714 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,9 +155,10 @@ jobs: # push to main whose SHA is main's tip moves ci-main: a dispatch from # any ref, main included, and a re-run of an older push push only # ci-. Do not gate on GITHUB_RUN_ATTEMPT: a re-run at the tip is - # legitimate. + # legitimate. Keep it bash-3.2 clean (tr, not ${OWNER,,}): precheck runs + # it, and macOS ships bash 3.2. run: | - BASE="ghcr.io/${OWNER,,}/eigenscript-dev" + BASE="ghcr.io/$(printf %s "$OWNER" | tr '[:upper:]' '[:lower:]')/eigenscript-dev" SHA="${{ github.sha }}" if [ "$IS_FORK" = "true" ]; then echo "image=$BASE:ci-main" >> "$GITHUB_OUTPUT" diff --git a/docs/CI.md b/docs/CI.md index fd62724cb..e0e5c8cf0 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -603,8 +603,8 @@ self-tests` job. A missing PyYAML is exit 2, never a pass. may mention the event only as `github.event_name ==/!= 'pull_request'`, or via the PR payload `github.event.pull_request.*` (empty on push and in the queue alike). Beyond those two forms, a condition may read only the `github` - fields that are the same for the push, merge-queue, pull-request and dispatch - runs of this repository: `repository`, `repository_owner`, `repository_id`, + fields whose value does not depend on the event that started the run: + `repository`, `repository_owner`, `repository_id`, `repository_owner_id`, `server_url`, `api_url`, `graphql_url`, `workspace`, `workflow`, `job`, `action`, `token` and `retention_days`. Any other field (such as `ref_protected`, `workflow_ref`, `sha` or `actor`), or a bare From f27fe34f4b5d86053bafc30db32b4dddae8e8267 Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Thu, 15 Oct 2026 01:07:14 -0500 Subject: [PATCH 5/5] ci_tier_check: close the [ci-main]/[writer-actions] model gaps Self-review of the first property version (e0b6e015) found 10 edits that write ci-main, or redirect what the matrix models, while it stayed green. Each is now red, with the plant recorded in the PR (32 plants, the same table under bash 3.2 and bash 5): - [writer-actions]: each allowlisted action takes only the inputs vetted for it (INPUTS); build-push-action's `outputs:` or a registry `cache-to:` pushes any name, and a registry cache is shared across refs (O1-O3); checkout `repository:` moves origin away from this repo (O4). - [ci-main]: the matrix merges workflow, job and step env as the runner does (a job-level env was ignored), and refuses env that redirects bash or git: PATH, GIT_*, BASH_FUNC_*, SHELLOPTS, BASHOPTS, LD_* (J2, J3), plus `working-directory:` (W1). It reads tags= in any case with quotes stripped (Q1, U4) and refuses a tags<< output (H1). Controls stay green: a new platforms: value (C4), a plain job env (C5). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KC99CmwatKssQYggkBCgwF --- docs/CI.md | 18 +++++++++++++--- tools/ci_tier_check.sh | 49 ++++++++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/docs/CI.md b/docs/CI.md index e0e5c8cf0..397e59fec 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -637,14 +637,26 @@ self-tests` job. A missing PyYAML is exit 2, never a pass. `ci-` is by design. - `on.push` must be exactly `branches: [main]`. - Each image push must take its tags from that one step. - - The check refuses what its matrix cannot model: a custom shell - (`defaults:` or `shell:`), `BASH_ENV`/`ENV`/`GITHUB_*` in any env scope, - or an expression it cannot evaluate. + - The matrix runs the step with the workflow, job and step env merged as the + runner merges them, and reads its `tags` output in any case, quotes + stripped. + - The check refuses what its matrix cannot model: + - a custom shell or directory (`defaults:`, `shell:`, + `working-directory:`); + - env in any scope that redirects bash or git (`PATH`, `BASH_ENV`, `ENV`, + `SHELLOPTS`, `BASHOPTS`, `GITHUB_*`, `GIT_*`, `BASH_FUNC_*`, `LD_*`); + - an expression it cannot evaluate; + - a multi-line `tags<<` output. - `[writer-actions]` — a `ci.yml` job that can write uses only the allowlisted actions `actions/checkout`, `docker/setup-buildx-action`, `docker/login-action` and `docker/build-push-action`. - The allowlist compares action identities without the `@ref`, so a Dependabot bump stays green. + - Each action takes only the inputs vetted for it. So + `docker/build-push-action` takes no `outputs:` and only a `type=gha` + `cache-to:`: an `outputs:` or a registry cache can push to any name, and a + registry cache is shared across refs. + - A new input goes red until someone vets it and adds it to `INPUTS`. - No `./` or `docker://` action, and no reusable-workflow call. - No script except the tag step that `[ci-main]` runs. diff --git a/tools/ci_tier_check.sh b/tools/ci_tier_check.sh index 3f74d0e99..d89fc7bab 100755 --- a/tools/ci_tier_check.sh +++ b/tools/ci_tier_check.sh @@ -49,9 +49,9 @@ def always(j): return cond(j.get("if", "")) == "always()" G = r"github\s*(?:\.\s*{0}\b|\[\s*['\"]{0}['\"]\s*\])" PR_ATOM = re.compile(G.format("event_name") + r"\s*[=!]=\s*['\"]pull_request['\"]|['\"]pull_request['\"]\s*[=!]=\s*" + G.format("event_name"), re.I) PR_BODY = re.compile(G.format("event") + r"\s*(?:\.\s*pull_request\b|\[\s*['\"]pull_request['\"]\s*\])", re.I) -# An ALLOWLIST (#1787): a condition may read only github fields that are the same for the -# push, merge_group, pull_request and workflow_dispatch runs of this repository. Any other -# field (ref_protected, workflow_ref, actor, sha, ...) or a bare `github` can select a lane. +# An ALLOWLIST (#1787): a condition may read only github fields whose value does not depend +# on the event (push, merge_group, pull_request or workflow_dispatch). Any other field +# (ref_protected, workflow_ref, actor, sha, ...) or a bare `github` can select a lane. INVARIANT = {"repository", "repository_owner", "repository_id", "repository_owner_id", "server_url", "api_url", "graphql_url", "workspace", "workflow", "job", "action", "token", "retention_days"} GH = re.compile(r"\bgithub\b(?:\s*\.\s*([A-Za-z_][\w-]*)|\s*\[\s*['\"]([^'\"]*)['\"]\s*\])?", re.I) @@ -189,12 +189,15 @@ for jid in sorted(HEAVY_WORKERS): # event x ref x "this SHA is main's tip"; ci-main must appear exactly on a push to # refs/heads/main at main's tip (no dispatch, queue run or re-run of an older push). # ci- tags go out from every non-fork ref by design. What the matrix cannot -# model (a custom shell, BASH_ENV, an expression it cannot evaluate) is refused. +# model (a custom shell or directory, env that redirects bash or git, an expression +# it cannot evaluate, tags it cannot read) is refused. # [writer-actions] a job that can write uses only allowlisted actions (compared -# without @ref), no ./ or docker:// actions, and no script but that tag step. +# without @ref) with the inputs vetted for each (no outputs:, cache-to: only type=gha), +# no ./ or docker:// actions, and no script but that tag step. # [secrets] ci.yml reads no secret but GITHUB_TOKEN, so only permissions: can write. def writes(p): return p == "write-all" or (isinstance(p, dict) and "write" in p.values()) -ALLOWED = {"actions/checkout", "docker/setup-buildx-action", "docker/login-action", "docker/build-push-action"} +INPUTS = {"actions/checkout": set(), "docker/setup-buildx-action": set(), "docker/login-action": {"registry", "username", "password"}, + "docker/build-push-action": {"context", "file", "platforms", "push", "tags", "cache-from", "cache-to"}} EVENTS = ("push", "pull_request", "merge_group", "workflow_dispatch") REFS = ("refs/heads/main", "refs/heads/a-branch", "refs/pull/1/merge", "refs/heads/gh-readonly-queue/main/pr-1-0123abc") def env_value(v, fork): @@ -208,15 +211,17 @@ def env_value(v, fork): def ci_main(jid, j, tag): """Run the tag step across the matrix; returns how many runs it judged.""" where = f"ci.yml:{jid} step {tag.get('id')!r}" - if DOCS["ci.yml"].get("defaults") or j.get("defaults") or "shell" in tag: - V("ci-main", f"{where}: a custom shell (defaults: or shell:) can change what the step computes; the matrix runs it under bash -e only"); return 0 + if DOCS["ci.yml"].get("defaults") or j.get("defaults") or "shell" in tag or "working-directory" in tag: + V("ci-main", f"{where}: a custom shell or directory (defaults:, shell:, working-directory:) can change what the step computes; the matrix runs it under bash -e in the checkout"); return 0 + envs = {} # workflow < job < step, as the runner merges them for scope, d in (("workflow", W["ci.yml"][2]), ("job", j.get("env") or {}), ("step", tag.get("env") or {})): - bad_env = [k for k in d if str(k).upper() in ("BASH_ENV", "ENV") or str(k).upper().startswith("GITHUB_")] - if bad_env: V("ci-main", f"{where}: {scope} env sets {bad_env}, which can change what the step computes"); return 0 + bad_env = [k for k in d if re.match(r"(PATH|BASH_ENV|ENV|SHELLOPTS|BASHOPTS)$|GITHUB_|GIT_|BASH_FUNC_|LD_", str(k), re.I)] + if bad_env: V("ci-main", f"{where}: {scope} env sets {bad_env}, which can change what the step or its git computes"); return 0 + envs.update(d) script = str(tag.get("run", "")) odd = sorted({x.strip() for x in exprs(script)} - {"github.sha"}) - if odd or any(env_value(v, False) is None for v in (tag.get("env") or {}).values()): - V("ci-main", f"{where}: uses an expression the matrix cannot evaluate ({odd or 'in its env'})"); return 0 + if odd or any(env_value(v, False) is None for v in envs.values()): + V("ci-main", f"{where}: uses an expression the matrix cannot evaluate ({odd or 'in its workflow, job or step env'})"); return 0 runs = 0 with tempfile.TemporaryDirectory() as t: with open(os.path.join(t, "git"), "w") as f: # a git that answers only `ls-remote origin refs/heads/main` @@ -230,15 +235,18 @@ def ci_main(jid, j, tag): open(out, "w").close() env = {"PATH": t + ":" + os.environ["PATH"], "HOME": t, "GITHUB_EVENT_NAME": ev, "GITHUB_REF": ref, "GITHUB_SHA": sha, "GITHUB_OUTPUT": out, "TIP": {True: sha, False: "2" * 40, None: ""}[tip], - **{str(k): env_value(v, fork) for k, v in (tag.get("env") or {}).items()}} + **{str(k): env_value(v, fork) for k, v in envs.items()}} p = subprocess.run(["bash", "-e", "-c", re.sub(r"\$\{\{\s*github\.sha\s*\}\}", sha, script)], env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=20) runs += 1 - tags = [x for line in open(out) if line.startswith("tags=") for x in line[5:].strip().split(",") if x] + text = open(out).read() # output names are case-insensitive; build-push-action strips quotes + tags = [x.strip().strip("\"'") for line in text.split("\n") if line[:5].lower() == "tags=" for x in line[5:].split(",")] has = any(x.endswith(":ci-main") for x in tags) want = ev == "push" and ref == "refs/heads/main" and tip is True and not fork what = f"event={ev} ref={ref} tip={'?' if tip is None else tip}{' fork' if fork else ''}" - if p.returncode != 0 and not (tip is None and ev == "push" and ref == "refs/heads/main"): + if re.search(r"^tags<<", text, re.M | re.I): + V("ci-main", f"{where} writes tags as a multi-line output for {what}, which the matrix does not read; write one tags= line") + elif p.returncode != 0 and not (tip is None and ev == "push" and ref == "refs/heads/main"): V("ci-main", f"{where} failed for {what}: {p.stderr.strip()[-200:]}") elif has != want: V("ci-main", f"{where} {'pushes' if has else 'does not push'} ci-main for {what}; ci-main is main's: only a push to refs/heads/main at main's tip may move it") @@ -257,9 +265,12 @@ for jid, j in ci.items(): V("ci-main", f"ci.yml:{jid}: each image push must take its tags from one step (`tags: ${{{{ steps..outputs.tags }}}}`), the step the matrix runs") tag = next((st for st in steps if st.get("id") in tag_ids), None) for i, st in enumerate(steps): - u = str(st.get("uses", "")).strip() - if u and (u.startswith(("./", "docker://")) or re.sub(r"@[^@]*$", "", u).lower() not in ALLOWED): - V("writer-actions", f"ci.yml:{jid} step {i} uses {u!r}: a job that can write uses only {sorted(ALLOWED)} (any @ref), no ./ or docker:// action") + u = str(st.get("uses", "")).strip(); ident = re.sub(r"@[^@]*$", "", u).lower(); w = st.get("with") or {} + if u and (u.startswith(("./", "docker://")) or ident not in INPUTS): + V("writer-actions", f"ci.yml:{jid} step {i} uses {u!r}: a job that can write uses only {sorted(INPUTS)} (any @ref), no ./ or docker:// action") + elif u and (set(map(str, w)) - INPUTS[ident] or any(not l.strip().startswith("type=gha") for l in str(w.get("cache-to", "type=gha")).splitlines() if l.strip())): + V("writer-actions", f"ci.yml:{jid} step {i} gives {ident} {sorted(map(str, w))}: in a job that can write, an action takes only the inputs vetted for it " + f"({sorted(INPUTS[ident])}, cache-to: only type=gha); outputs: or a registry cache can push any name") elif "run" in st and st is not tag: V("writer-actions", f"ci.yml:{jid} step {i} ({st.get('name', '?')}) runs a script in a job that can write; only the tag step may, because the matrix runs it") if tag is not None: judged += ci_main(jid, j, tag) @@ -281,7 +292,7 @@ if set(WAIVE) - used: V("event-condition", f"waiver(s) {sorted(set(WAIVE) - used if bad: print(f"ci-tier: FAIL — {len(bad)} violation(s): {' '.join(sorted(set(bad)))}"); sys.exit(1) print(f"ci-tier: OK — ci.yml jobs={len(ci)} (awk={os.environ['CT_AWK']}) required={counts['required']} worker={counts['worker']}; " f"{len(req)} required names, each produced once on pull_request+merge_group; {len(path)} jobs on required paths, no push-only condition, no continue-on-error; " - f"ci-main only on a push to main at its tip ({judged} event x ref x tip runs); {len(writers)} writing job(s), allowlisted actions only; no secret but GITHUB_TOKEN") + f"ci-main only on a push to main at its tip ({judged} event x ref x tip runs); {len(writers)} writing job(s), allowlisted actions and inputs only; no secret but GITHUB_TOKEN") PY } live() {