From 921de564608df2c683a1cd6026c09c7a8600eff9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 23:37:35 +0000 Subject: [PATCH 1/3] ci: hourly full run on main; push narrows to the affected set - push on main computes the Test Core package set with --affected against github.event.before; a zero or unresolvable sha falls back to the full list, loudly. - schedule '0 * * * *' + workflow_dispatch on ci.yml and lint.yml run the full battery, with their own concurrency group so a merge cannot cancel them. - select-shard-timings-run reads the scheduled run, not the push run. - a new workflow_run filer opens or refreshes one deduplicated card per watched workflow when a scheduled run is red. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012GKcPZbMoGq7WPzKLfRBTU --- .github/workflows/ci.yml | 84 +- .github/workflows/lint.yml | 55 +- .github/workflows/scheduled-full-run-card.yml | 255 ++++++ .github/workflows/shard-timings-refresh.yml | 82 +- scripts/ci/scheduled-full-run.mjs | 760 ++++++++++++++++++ scripts/ci/select-shard-packages.selftest.sh | 159 +++- scripts/ci/select-shard-packages.sh | 90 ++- scripts/ci/select-shard-timings-run.mjs | 210 ++++- 8 files changed, 1633 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/scheduled-full-run-card.yml create mode 100644 scripts/ci/scheduled-full-run.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cea13285a4..40c07856a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,12 +38,55 @@ on: # branch-protection-required check MUST carry this trigger, or queue builds # wait forever on a check that never reports. merge_group: + # ── THE HOURLY FULL RUN (#16467) ───────────────────────────────────────── + # `push` above now computes the Test Core package set with `--affected` + # against `github.event.before`, so a merge no longer re-tests the whole + # workspace it just tested in the queue. Something still has to run the FULL + # battery on `main`, on a clock, or "main is green" stops being a statement + # about the workspace and becomes a statement about the last diff. + # + # This is that run. Minute 0 rather than an offset minute ON PURPOSE, and + # against the sibling convention in test-nightly-tiers.yml: the reading this + # feeds — `scripts/ci/select-shard-timings-run.mjs` — wants one complete run + # per hour far more than it wants a short queue wait, and a run that starts + # a few minutes late is still the hour's run. What it must NOT do is collide + # with the next hour's, which the concurrency group below decides. + # + # ⛔ This is a NEW TRIGGER KEY, not a widening of `push:`. The ⛔ above still + # binds: `push:` stays filtered to `main`. + schedule: + - cron: '0 * * * *' + # The same full battery, on demand: an operator who has just landed a fix for + # a red hourly run should not have to wait up to an hour to see it answered. + # Treated as `full` by the selection script for the same reason `schedule` is. + workflow_dispatch: # Superseded runs on the same PR/branch waste runners and delay feedback; # cancel them. Push runs to main group by commit ref as well, so an in-flight # main run is cancelled only by a newer main push. +# +# ⭐ `github.event_name` IS LOAD-BEARING IN THIS KEY, and it is the whole +# reason the hourly run above can finish (#16467). On a `schedule` event +# `github.event.pull_request.number` is empty and `github.ref` is +# `refs/heads/main` — BYTE-IDENTICAL to what a push to `main` produces. Without +# the event name in the key the hourly run and the next merge share one group, +# `cancel-in-progress: true` applies, and the next merge kills the hourly run. +# That is not a hypothetical: this card's own measurement is that 36 of the +# last 60 push runs on `main` were already cancelled that way, at a merge +# cadence that would censor most hours. +# +# ⛔ Do not "simplify" this back to two segments. `scripts/ci/scheduled-full- +# run.mjs --check-concurrency` evaluates this expression against a push-shaped +# and a schedule-shaped context and reds when the two groups are equal, so the +# collision cannot come back silently. +# +# `cancel-in-progress` stays `true` for every event including `schedule`: two +# hourly runs overlapping means one of them is over an hour old, and an hour-old +# measurement of `main` is strictly the worse of the two. Test Core's wall clock +# is well inside the cadence (~39 shard-minutes across six shards), so this is +# the exceptional path, not the normal one. concurrency: - group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ci-${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -54,11 +97,13 @@ jobs: contents: read pull-requests: read outputs: - # On merge_group, everything counts as changed: dorny/paths-filter has no - # merge_group support, and the queue build is the last validation before - # main — the one place a skipped job can never be the right answer. A - # skipped step's output is the empty string (falsy), so `|| 'true'` - # supplies the merge-group value without touching PR/push behavior. + # On merge_group — and on the `schedule` / `workflow_dispatch` full runs + # (#16467) — everything counts as changed: dorny/paths-filter resolves a + # diff only on `pull_request` and `push`, and each of these three events + # is a place a skipped job can never be the right answer (the queue build + # is the last validation before main; the hourly run IS the full battery). + # A skipped step's output is the empty string (falsy), so `|| 'true'` + # supplies their value without touching PR/push behavior. docs: ${{ steps.changes.outputs.docs || 'true' }} core: ${{ steps.changes.outputs.core || 'true' }} console: ${{ steps.changes.outputs.console || 'true' }} @@ -100,9 +145,24 @@ jobs: - name: Checkout repository uses: actions/checkout@v7 + # ⭐ SPELLED AS AN ALLOW-LIST, not as `!= 'merge_group'` (#16467). This + # action reads a diff, and the only two events that hand it one it can + # resolve unaided are `pull_request` (base vs head) and `push` + # (`event.before` vs `after`). `merge_group` was the first event with + # neither, and the exclusion was written as its name; `schedule` and + # `workflow_dispatch` are the second and third, and under the old + # spelling they would have RUN the action with no resolvable base. + # + # That failure direction is the dangerous one: every filter output would + # come back `false`, `|| 'true'` supplies nothing (the output exists and + # says `false`), and every downstream `!= 'false'` guard SKIPS — so the + # hourly full run would have been an entirely green, entirely empty run + # of nothing at all. An allow-list makes each new event fail toward + # THE FILTER CONTRACT's half 1 instead: the step is skipped, its outputs + # are the empty string, `|| 'true'` fills them in, and everything runs. - uses: dorny/paths-filter@v4 id: changes - if: github.event_name != 'merge_group' + if: github.event_name == 'pull_request' || github.event_name == 'push' with: filters: | docs: @@ -473,13 +533,21 @@ jobs: # on this shard" exit below, every shard still attests, # and Test Core is an honest green. It is NOT the #10057 # case, which is pull_request-only (the script says why). - # push unchanged: the FULL list. + # push affected set against `github.event.before`, the commit + # `main` was on before this merge landed (#16467). A zero + # sha -- a first push, or a force-push that rewrote + # history -- falls back to the FULL list, loudly. + # schedule the FULL list: the hourly run IS the full battery, and + # workflow_ the on-demand rerun of it. Neither carries a diff base, + # dispatch and neither should: they are the run that says whether + # the whole workspace is green on `main`. - name: Compute this shard's package set env: OS_SHARD_EVENT_NAME: ${{ github.event_name }} OS_SHARD_PR_BASE_REF: ${{ github.event.pull_request.base.ref }} OS_SHARD_PR_PINNED_BASE_SHA: ${{ github.event.pull_request.base.sha }} OS_SHARD_MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }} + OS_SHARD_PUSH_BEFORE_SHA: ${{ github.event.before }} run: | bash scripts/ci/select-shard-packages.sh node scripts/partition-test-shards.mjs "$RUNNER_TEMP/turbo-ls.json" \ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8daeaee9c0..d285e88cf6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,6 +25,24 @@ on: # queue builds or the queue stalls. This workflow has no PR-only steps, so # the trigger alone is enough. merge_group: + # ── THE HOURLY FULL RUN (#16467), the half this file owns ──────────────── + # The card's ruling is that the hourly run covers BOTH required-context + # files, not just ci.yml, and the reason is measured rather than symmetric: + # the push-on-`main` run of THIS workflow is the only post-merge full-battery + # run of the families `scripts/ci/select-gate-families.sh` scopes away on + # merge groups (the PM dispatch-gates self-test, both ratchets, the + # verify-lock self-test, the comment-mask corpus). A scoped family that goes + # red on `main` after a queue build skipped it had, until this trigger, no + # run that would notice and no filer that would say so. + # + # The selector already treats every event that is neither `merge_group` nor + # `pull_request` as "run every family", so this trigger alone restores the + # full battery here — no step in this file changes. + # + # ⛔ A NEW TRIGGER KEY, not a widening of `push:`; see ci.yml's `on:` block. + schedule: + - cron: '0 * * * *' + workflow_dispatch: # ── MEASURED 2026-08-25 (#12211) — a NEGATIVE result, recorded so it is not # re-measured. After the stale-ledger outage the queue's check set was measured @@ -68,8 +86,14 @@ on: # Same policy as ci.yml: superseded runs on the same PR/branch waste runners # and delay feedback; cancel them. Push runs to main group by commit ref, so an # in-flight main run is cancelled only by a newer main push. +# +# ⭐ `github.event_name` is in the key for the reason ci.yml's concurrency block +# states in full (#16467): on a `schedule` event the remaining two segments are +# byte-identical to a push to `main`, so without it the next merge cancels the +# hourly full run. `scripts/ci/scheduled-full-run.mjs --check-concurrency` +# evaluates THIS expression too and reds if the two groups ever collapse again. concurrency: - group: lint-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: lint-${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true # ───────────────────────────────────────────────────────────────────────────── @@ -3402,6 +3426,35 @@ jobs: - name: Test Core package selection self-test run: pnpm check:select-shard-packages + # The hourly full run's own contract (#16467). Two halves, one script: + # + # --self-test the card's IDENTITY, DE-DUP and BODY, driven + # offline. The filer is a `workflow_run` workflow, so + # it can only ever run from the default branch and + # NOTHING on a pull request can exercise it — the + # same position merge-queue-triage.yml is in. A + # de-dup rule tested only by the live workflow gets + # its first real test on the night it files its + # second duplicate, and this one fires hourly. + # --check-concurrency + # evaluates ci.yml's and THIS file's + # `concurrency.group` expressions against a + # push-shaped and a schedule-shaped context and reds + # when the two collapse onto one group. They did + # until #16467: on a `schedule` event the other two + # segments are byte-identical to a push to `main`, so + # `cancel-in-progress` let the next merge kill the + # hourly run. It also refuses the two degenerate ways + # to make them differ — a run-unique key, and a + # constant one — since both switch cancellation off + # or on for every event in the file. + # + # Offline, no network, ~0.1 s. + - name: Hourly full run self-test and concurrency gate + run: | + node scripts/ci/scheduled-full-run.mjs --self-test + node scripts/ci/scheduled-full-run.mjs --check-concurrency + # Gate-family selection self-test (#16496). The "Select the gate families # this run pays for" step at the top of this job is a thin call into # scripts/ci/select-gate-families.sh, and its `merge_group` branch cannot diff --git a/.github/workflows/scheduled-full-run-card.yml b/.github/workflows/scheduled-full-run-card.yml new file mode 100644 index 0000000000..d96654f6dd --- /dev/null +++ b/.github/workflows/scheduled-full-run-card.yml @@ -0,0 +1,255 @@ +name: Scheduled Full Run Card + +# The reader for the hourly full run (#16467). +# +# ══════════════════════════════════════════════════════════════════════════════ +# WHY THIS EXISTS: A RED SCHEDULED RUN HAD NOBODY LOOKING AT IT. +# ══════════════════════════════════════════════════════════════════════════════ +# +# Since #16467 a `push` to `main` computes the Test Core package set with +# `--affected`, and `scripts/ci/select-gate-families.sh` already scopes several +# `Lint & Repo Gates` families away on merge groups. The hourly `schedule` run +# of `CI` and `Lint & Type Check` is therefore the ONLY run that exercises the +# whole battery on `main`. +# +# A scheduled run is on no pull request. It publishes no check that branch +# protection reads, it blocks nothing, and nobody is notified when it goes red. +# Narrowing `push` without this file would have left `main` less tested than +# before AND unwatched — strictly worse than not doing the card at all. +# +# ⚠️ IT WATCHES BOTH WORKFLOWS, and that is the part the card did not say. +# `merge-queue-triage.yml` is `workflows: [CI]`, so a red `Lint & Type Check` on +# `main` has had no filer at all — not since #16467, but ever. The families +# #16496/#16754 scope on merge groups (the PM dispatch-gates self-test, both +# ratchets, the verify-lock self-test, the comment-mask corpus) are exactly the +# ones that would go red here and nowhere else. +# +# ## ONE CARD PER WORKFLOW, deduplicated — the shape, and why not one card +# +# The identity, the de-dup rule and the body all live in +# `scripts/ci/scheduled-full-run.mjs`, driven offline by its `--self-test`. That +# is deliberate: a de-dup rule exercised only by the live workflow gets its +# first real test on the night it files its second duplicate, and this one fires +# hourly. +# +# IDENTITY a fixed title prefix per workflow — +# `hourly full run: red on main (CI)` — plus a PLAIN-TEXT body +# marker. ⛔ Never an HTML comment: this platform's body sanitizer +# is measured to eat short angle-bracket fragments, and a de-dup +# key that can be swallowed files a duplicate an hour. +# DE-DUP scan OPEN issues for that prefix or that marker, bounded pages; +# a scan that hits its page bound has NOT established absence and +# REFUSES rather than filing. The oldest match wins — it is the one +# any duplicates were closed against, and the one the devx seat +# already graded. +# REFRESH the body is rewritten in place, never a comment per run. ⛔ Labels +# are applied on CREATE only: grading is the seat's and a refresh +# must not undo it. +# BODY the run link, and the commits between the previous GREEN +# scheduled run of THIS workflow and this one. That range is the +# hour in which `main` broke, and an EMPTY range is a reading of +# its own: same tree, green then red, so it is a flake or an +# infrastructure fault and nobody should go hunting a commit. +# +# ⛔ Two cards, not one, because `CI` and `Lint & Type Check` are two batteries +# that go red for unrelated reasons. Under a single identity whichever filer ran +# second would OVERWRITE the other's diagnosis — the body is rewritten on a +# refresh. "One red scheduled run files exactly one card" holds per run, which +# is the unit that is red. +# +# ⛔ A CLOSED card is never reopened. Red again after it was answered is a +# regression, filed fresh. +# +# ## What this file cannot prove about itself, and what covers that +# +# A `workflow_run` workflow only ever runs from the default branch, so nothing +# on a pull request can trigger it — the same position `merge-queue-triage.yml` +# is in. Everything decidable offline is therefore pushed into the module and +# gated by `Lint & Repo Gates`; what is left here is the API paging and the +# event guard. + +on: + workflow_run: + # Byte-exact workflow NAMES (the `name:` at the top of each file), not + # paths. Quoted because `Lint & Type Check` starts a YAML alias unquoted. + workflows: ['CI', 'Lint & Type Check'] + types: [completed] + +permissions: {} + +concurrency: + # Per watched workflow: two reds in one hour are two different cards and must + # not race, while two reds of the SAME workflow are the same card and the + # later one carries the newer facts. ⛔ `cancel-in-progress: false` — a filer + # cancelled between its de-dup scan and its create is how a duplicate is born. + group: scheduled-full-run-card-${{ github.event.workflow_run.name }} + cancel-in-progress: false + +jobs: + file: + name: File or refresh the hourly full run card + # ⭐ THREE guards, and each one is load-bearing. + # + # event == 'schedule' — a push, PR or merge_group run of these workflows + # is read by branch protection and by merge-queue-triage.yml. Filing on + # those would mint a card for every red PR in the repo. + # conclusion in (failure, timed_out) — NOT `!= 'success'`. `cancelled` + # and `skipped` are not reds: a cancelled hourly run measured nothing, + # and a card saying "main is red" on the strength of a run that did not + # finish is a false statement that costs somebody an investigation. + # head_branch == 'main' — belt and braces. A scheduled run can only be on + # the default branch today; if that ever changes, this must not file. + if: >- + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.head_branch == 'main' && + (github.event.workflow_run.conclusion == 'failure' || + github.event.workflow_run.conclusion == 'timed_out') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read # list the previous green scheduled run + contents: read # checkout, and the compare API + issues: write # the card, and nothing else + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: File or refresh the card + id: card + uses: actions/github-script@v9 + with: + # Same transient-retry posture as merge-queue-triage.yml, and for the + # same reason: 403 is REMOVED from the exempt list because GitHub + # answers a secondary rate limit with 403 as well as 429, and this job + # pages issues. 400/401/404/422 stay exempt — a malformed request or a + # body past the 65536-character limit is this repo's own bug and is + # not improved by asking again. + retries: 3 + retry-exempt-status-codes: 400,401,404,422 + script: | + const mod = await import(`${process.env.GITHUB_WORKSPACE}/scripts/ci/scheduled-full-run.mjs`); + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + + const identity = mod.cardIdentity(run.name); + const runUrl = run.html_url; + + // The previous GREEN scheduled run of THIS workflow. `status: + // 'success'` is the filter that matters: a cancelled or a red + // earlier run is not a point the tree was known good at, so a range + // measured from one would name commits that were already suspect. + let previousGreen = null; + let commits = []; + let compareUrl = null; + let commitsTruncated = false; + let rangeNote = null; + try { + const previous = await github.rest.actions.listWorkflowRuns({ + owner, repo, + workflow_id: run.workflow_id, + event: 'schedule', + branch: 'main', + status: 'success', + per_page: 1, + }); + const hit = previous.data.workflow_runs?.[0]; + if (hit) previousGreen = { run_id: hit.id, head_sha: hit.head_sha }; + else rangeNote = 'No previous GREEN `schedule` run of this workflow is in the API window — either this is the first one, or every hourly run in the window was red.'; + } catch (error) { + rangeNote = `The previous-green lookup failed (${error.message}).`; + } + + if (previousGreen) { + try { + const cmp = await github.rest.repos.compareCommitsWithBasehead({ + owner, repo, + basehead: `${previousGreen.head_sha}...${run.head_sha}`, + }); + commits = (cmp.data.commits ?? []).map((c) => ({ + sha: c.sha, + title: String(c.commit?.message ?? '').split('\n')[0], + })); + compareUrl = cmp.data.html_url ?? null; + // The compare endpoint caps at 250 commits. Saying so beats + // presenting a truncated list as the whole hour. + commitsTruncated = (cmp.data.total_commits ?? commits.length) > commits.length; + } catch (error) { + // ⛔ Do NOT fall through to "nothing landed": that string is the + // FLAKE reading, and a failed lookup is not evidence of an + // empty range. Drop the range entirely and say why. + previousGreen = null; + rangeNote = `The commit range could not be read (${error.message}).`; + } + } + + const body = mod.renderBody({ + identity, + runUrl, + headSha: run.head_sha, + conclusion: run.conclusion, + sweptAt: new Date().toISOString(), + previousGreen, + commits, + compareUrl, + commitsTruncated, + rangeNote, + }); + + const existing = await mod.findExistingCard({ + identity, + perPage: 100, + maxPages: 10, + listPage: async (page) => { + const res = await github.rest.issues.listForRepo({ + owner, repo, state: 'open', sort: 'created', direction: 'asc', per_page: 100, page, + }); + return res.data; + }, + }); + + if (existing) { + await github.rest.issues.update({ owner, repo, issue_number: existing.number, body }); + core.info(`refreshed ${identity.marker} card #${existing.number} (${body.length} chars)`); + core.notice(`${run.name} is red on the hourly full run — card #${existing.number} refreshed.`); + core.setOutput('action', 'refreshed'); + core.setOutput('number', String(existing.number)); + return; + } + + const created = await github.rest.issues.create({ + owner, repo, + title: identity.title, + body, + // Additive on create; ⛔ nothing here ever replaces a whole label + // set (`check:whole-set-label-write` refuses that verb outright). + labels: identity.labels, + }); + core.info(`filed ${identity.marker} card #${created.data.number} (${body.length} chars)`); + core.notice(`${run.name} is red on the hourly full run — card #${created.data.number} filed.`); + core.setOutput('action', 'filed'); + core.setOutput('number', String(created.data.number)); + + - name: Publish the verdict to the run summary + # always(): when the step above threw — a refused scan, a rate limit — + # the run must still say what it was reacting to, or the red hourly run + # is invisible in both places at once. + if: always() + env: + CARD_ACTION: ${{ steps.card.outputs.action }} + CARD_NUMBER: ${{ steps.card.outputs.number }} + WATCHED: ${{ github.event.workflow_run.name }} + WATCHED_URL: ${{ github.event.workflow_run.html_url }} + WATCHED_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: | + { + echo "### Hourly full run: \`$WATCHED\` concluded \`$WATCHED_CONCLUSION\` on \`main\`" + echo + echo "- Run: $WATCHED_URL" + if [ -n "${CARD_ACTION:-}" ]; then + echo "- Card: ${CARD_ACTION} #${CARD_NUMBER}" + else + echo "- Card: NOT WRITTEN — the filing step did not report an outcome. Read this run's log:" + echo " a bounded issue scan that could not complete REFUSES to file rather than risk a" + echo " duplicate, and that refusal looks exactly like this." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/shard-timings-refresh.yml b/.github/workflows/shard-timings-refresh.yml index 2b2274d322..cd023d2347 100644 --- a/.github/workflows/shard-timings-refresh.yml +++ b/.github/workflows/shard-timings-refresh.yml @@ -57,15 +57,40 @@ # no refresh because it stamps a fresh `measuredAt` on numbers nobody measured. # That script carries the argument and the self-test; this file only drives it. # +# ⚠ WHICH RUNS: THE HOURLY `schedule` RUN OF ci.yml, NEVER A PUSH RUN (#16467). +# Until #16467 a push to `main` re-ran the FULL Test Core battery, so "a green +# push run on main" and "a measurement of the workspace" were the same thing. +# They are not any more: a push run computes its package set with `--affected` +# against `github.event.before`. Its six shards still conclude `success` and +# still upload six run summaries, so NOTHING in the eligibility test would have +# noticed — this lane would have kept regenerating the balancing dataset from +# measurements of whatever the last merge happened to touch, and the coverage +# check would not have caught it either (a package the affected set skipped is a +# cache HIT, carried at its old weight, which is a pass). The selector names the +# event, and `select-shard-timings-run.mjs --self-test` drives a schedule-shaped +# and a push-shaped run through it to prove which one comes back. +# +# ⛔ Do NOT make ci.yml's `Save Turbo cache (main only)` step fire on `schedule` +# as well. It is `github.event_name == 'push'` on purpose and that is now +# load-bearing here: if the hourly run seeded the cache it restores, the next +# hourly run would replay almost the whole workspace, the generator would refuse +# every replayed task, and this lane would measure nothing — by construction, +# every hour, forever. Affected-only pushes seeding a narrower cache is the +# direction that HELPS: it leaves more real misses for the hourly run to time. +# # ⚠ AND IT IS "RUNS", PLURAL, WHICH THE DOCUMENTED PROCEDURE DOES NOT SAY. -# Measured on this lane's first live run (34083991141): of the seven retained -# green push runs on main, the BEST measured 52 of the 71 packages the committed -# dataset holds, and the others measured 2, 3, 13, 18, 22 and 49. All seven were -# partial cache replays. That follows from the cache design rather than from luck -# — turbo's key is namespaced per shard and only main pushes write it, so a -# package whose inputs have not changed is a HIT, and the generator refuses hits -# rather than recording a replayed ~0.1s window as a suite's cost. "Download six -# artifacts from any green run" therefore measures a SLICE of the workspace. +# Measured on this lane's first live run (34083991141), when the runs it read +# were still push runs: of the seven retained green runs on main, the BEST +# measured 52 of the 71 packages the committed dataset holds, and the others +# measured 2, 3, 13, 18, 22 and 49. All seven were partial cache replays. That +# follows from the cache design rather than from luck — turbo's key is +# namespaced per shard and only main pushes write it, so a package whose inputs +# have not changed is a HIT, and the generator refuses hits rather than +# recording a replayed ~0.1s window as a suite's cost. "Download six artifacts +# from any green run" therefore measures a SLICE of the workspace. That number +# should IMPROVE under the hourly run, because a push no longer seeds the whole +# workspace's entries — but it is quoted as it was measured, and the next live +# run is what re-measures it. # # So the regeneration step accumulates runs, each under its own `--run ` # group — the grouping #16473 added, which sums a sliced package's slices within @@ -235,7 +260,15 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - node scripts/ci/select-shard-timings-run.mjs --candidates --limit 15 \ + # 24, not 15 (#16467). The window that matters is the artifact + # retention window — 1 day — and the runs inside it are now the + # hourly scheduled ones, so 24 is "everything still downloadable" + # rather than an arbitrary depth. Coverage is ACCUMULATED across + # runs (see the next step), so examining fewer than the window holds + # is coverage left on the table; the cost is two API reads per run + # examined and the loop below still stops at the first accumulation + # that covers the workspace. + node scripts/ci/select-shard-timings-run.mjs --candidates --limit 24 \ > "$RUNNER_TEMP/candidates.json" echo "Eligible runs, newest first:" node -e ' @@ -246,16 +279,19 @@ jobs: # ACCUMULATE runs until the workspace is covered — do not look for one run # that covers it, because there is no such run. # - # MEASURED on the first live run of this lane (run 34083991141), and it is - # the fact that shapes this step: of the seven retained green push runs on - # main, the BEST measured 52 of the 71 packages the committed dataset - # holds, and the rest measured 2, 3, 13, 18, 22 and 49. Every one of them - # was a partial cache replay. That is not bad luck, it is the cache design: - # turbo's key is namespaced per shard and only main pushes write it, so a - # package whose inputs have not changed is a HIT — and the generator - # refuses hits rather than recording a replayed ~0.1s window as a suite's - # cost. "Download six artifacts from any green run" therefore measures a - # SLICE of the workspace, never all of it. + # MEASURED on the first live run of this lane (run 34083991141), when the + # runs read were still push runs, and it is the fact that shapes this + # step: of the seven retained green runs on main, the BEST measured 52 of + # the 71 packages the committed dataset holds, and the rest measured 2, 3, + # 13, 18, 22 and 49. Every one of them was a partial cache replay. That is + # not bad luck, it is the cache design: turbo's key is namespaced per shard + # and only main pushes write it, so a package whose inputs have not changed + # is a HIT — and the generator refuses hits rather than recording a + # replayed ~0.1s window as a suite's cost. "Download six artifacts from any + # green run" therefore measures a SLICE of the workspace, never all of it. + # Since #16467 the runs read here are the HOURLY `schedule` runs, which are + # the full battery; the accumulation stays because the cache argument above + # is unchanged by which event ran the suite. # # So runs are accumulated. Each contributes its six summaries under its own # `--run ` group, which is exactly the grouping #16473 added: slices are @@ -456,8 +492,12 @@ jobs: echo echo "## Source" echo - echo "Measured across $RUN_COUNT accumulated run(s). No single green run measures the whole" - echo "workspace — turbo's cache is namespaced per shard and only main pushes write it, so a" + echo "Measured across $RUN_COUNT accumulated run(s) of the HOURLY \`schedule\` run of CI on" + echo "\`main\` — the full-battery run (#16467). A \`push\` run on \`main\` is affected-only and" + echo "is not a measurement of the workspace, so no push run feeds this file." + echo + echo "No single green run measures the whole workspace either — turbo's cache is namespaced" + echo "per shard and only main pushes write it, so a" echo "package whose inputs have not changed is a HIT and the generator refuses hits rather" echo "than recording a replay as a duration. Runs are therefore accumulated, each fenced by" echo "its own \`--run\` group, until every package the committed dataset holds is measured" diff --git a/scripts/ci/scheduled-full-run.mjs b/scripts/ci/scheduled-full-run.mjs new file mode 100644 index 0000000000..0b88cfc3c0 --- /dev/null +++ b/scripts/ci/scheduled-full-run.mjs @@ -0,0 +1,760 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// scheduled-full-run -- the two halves of the hourly full run's contract that +// nothing else can hold (#16467). +// +// node scripts/ci/scheduled-full-run.mjs --check-concurrency +// node scripts/ci/scheduled-full-run.mjs --self-test +// +// ## Why one file holds both halves +// +// `push` on `main` now tests the AFFECTED set. The thing that keeps "main is +// green" a statement about the whole workspace is the hourly `schedule` run of +// ci.yml and lint.yml. Two properties have to hold for that to be true, and +// neither is visible in the run that would be wrong: +// +// 1. THE RUN HAS TO SURVIVE. On a `schedule` event `github.event.pull_ +// request.number` is empty and `github.ref` is `refs/heads/main` -- byte- +// identical to a push to `main`. Under the old two-segment concurrency key +// the hourly run and the next merge shared one group with +// `cancel-in-progress: true`, so the next merge killed the hourly run. +// That is measured, not feared: 36 of the last 60 push runs on `main` were +// already cancelled that way. +// +// 2. A RED ONE HAS TO BE SEEN. A scheduled run is on no PR, blocks nothing +// and appears on no check list. `merge-queue-triage.yml` listens to +// `workflows: [CI]` on `merge_group` only, so before this card a red +// `Lint & Type Check` on `main` had no reader at all. +// +// Half 1 is `--check-concurrency`, a gate over the two workflow files. Half 2 +// is the card the filer workflow opens, whose IDENTITY, DE-DUP and BODY are +// exported from here so they can be driven offline -- a de-dup rule that is +// only ever exercised by the live workflow is a de-dup rule that gets its first +// test the night it files its second duplicate. +// +// ⛔ Both halves fail CLOSED. The expression evaluator below throws on any +// token it does not understand rather than guessing a value, because a guess +// makes two groups compare equal or unequal for a reason nobody wrote down. + +import { readFileSync } from 'node:fs'; +import process from 'node:process'; + +import { isEntrypoint } from '../invoked-as.mjs'; + +// The workflows that carry the hourly full run. Spelled as quoted repo-relative +// literals: the dispatch derivation reads a gate's path literals as the +// population it watches, so a card touching either file schedules this family. +export const WATCHED_WORKFLOWS = Object.freeze([ + '.github/workflows/ci.yml', + '.github/workflows/lint.yml', +]); + +// --------------------------------------------------------------------------- +// HALF 1 -- the concurrency key +// --------------------------------------------------------------------------- + +// The top-level `concurrency.group:` line, read as TEXT rather than through a +// YAML parser on purpose: this gate has to run with no dependency beyond node, +// and the shape it reads is one scalar at a known indentation. Anchored to +// column 0 for `concurrency:` so a job-level `concurrency:` block (indented) +// cannot be mistaken for the workflow-level one. +export function readConcurrencyGroup(yamlText) { + const lines = String(yamlText).split('\n'); + const start = lines.findIndex((line) => line === 'concurrency:'); + if (start === -1) throw new Error('no top-level `concurrency:` block'); + for (let i = start + 1; i < lines.length; i += 1) { + const line = lines[i]; + if (/^\S/.test(line)) break; // the block ended + const m = /^\s+group:\s*(.+?)\s*$/.exec(line); + if (m) return m[1]; + } + throw new Error('the top-level `concurrency:` block declares no `group:`'); +} + +// Resolve a dotted context path. A property that is absent is the EMPTY STRING, +// which is what the runner substitutes and what makes `||` fall through. +function readPath(context, path) { + let node = context; + for (const segment of path.split('.')) { + if (node === null || typeof node !== 'object' || !Object.hasOwn(node, segment)) return ''; + node = node[segment]; + } + return node ?? ''; +} + +// Evaluate one `${{ ... }}` expression. Deliberately tiny, and deliberately +// LOUD about anything outside its grammar: context paths, single-quoted +// literals, and `||` between them. GitHub's `||` yields the first truthy +// operand and otherwise the last, and the empty string is falsy. +export function evaluateExpression(expression, context) { + const operands = String(expression).split('||').map((s) => s.trim()); + let last = ''; + for (const operand of operands) { + if (operand === '') throw new Error(`empty operand in expression '${expression}'`); + let value; + if (/^'[^']*'$/.test(operand)) value = operand.slice(1, -1); + else if (/^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$/.test(operand)) { + value = readPath(context, operand); + } else { + throw new Error( + `scheduled-full-run cannot evaluate the operand '${operand}' in '${expression}'. ` + + 'This evaluator refuses rather than guesses: a guessed value decides whether two ' + + 'concurrency groups compare equal, which is the whole verdict. Extend the grammar ' + + 'and add a self-test case for it.' + ); + } + last = value; + if (value !== '' && value !== null && value !== undefined && value !== false) return String(value); + } + return String(last); +} + +// A `concurrency.group` value with its `${{ }}` holes filled in. +export function evaluateGroup(groupExpression, context) { + return String(groupExpression).replace(/\$\{\{([^}]*)\}\}/g, (_all, inner) => + evaluateExpression(inner, context) + ); +} + +// The event shapes the verdict below compares. Each is what the runner really +// hands a workflow for that event on THIS repo: `schedule` and +// `workflow_dispatch` on `main` carry no pull request and `refs/heads/main`, +// which is precisely why they used to collide with `push`. +export function eventContext(eventName, { workflow = 'W', prNumber = null, ref = null } = {}) { + const refs = { + push: 'refs/heads/main', + schedule: 'refs/heads/main', + workflow_dispatch: 'refs/heads/main', + pull_request: `refs/pull/${prNumber ?? 1}/merge`, + merge_group: 'refs/heads/gh-readonly-queue/main/pr-1-' + 'a'.repeat(40), + }; + return { + github: { + workflow, + event_name: eventName, + ref: ref ?? refs[eventName] ?? `refs/heads/${eventName}`, + event: prNumber === null ? {} : { pull_request: { number: prNumber } }, + }, + }; +} + +// The verdict. Findings are returned rather than thrown so the CLI can name all +// of them at once. +export function concurrencyVerdict(groupExpression, { workflow = 'W' } = {}) { + const findings = []; + const group = (eventName, options = {}) => + evaluateGroup(groupExpression, eventContext(eventName, { workflow, ...options })); + + const push = group('push'); + const schedule = group('schedule'); + const dispatch = group('workflow_dispatch'); + + // THE assertion this gate exists for. + if (schedule === push) { + findings.push( + `a \`schedule\` run and a \`push\` run share the concurrency group '${schedule}', so the next ` + + 'merge to main cancels the hourly full run. Put `github.event_name` in the key.' + ); + } + if (dispatch === push) { + findings.push( + `a \`workflow_dispatch\` run and a \`push\` run share the concurrency group '${dispatch}', so a ` + + 'merge cancels the on-demand full run.' + ); + } + if (dispatch === schedule) { + findings.push( + `a \`workflow_dispatch\` run and a \`schedule\` run share the concurrency group '${dispatch}', so ` + + 'an on-demand full run cancels the hourly one it was meant to pre-empt.' + ); + } + + // POSITIVE CONTROL, and it is not decoration. The cheap way to make every + // comparison above unequal is to put something run-unique in the key -- + // `github.run_id`, `github.sha` -- which also switches `cancel-in-progress` + // off for every event in the file. So the key must still collapse two runs of + // the SAME event on the same ref onto one group. + if (group('schedule') !== group('schedule')) { + findings.push('two `schedule` runs on the same ref do not share a group (the key is not stable)'); + } + if (group('push', { ref: 'refs/heads/main' }) !== group('push', { ref: 'refs/heads/main' })) { + findings.push('two `push` runs on the same ref do not share a group (the key is not stable)'); + } + // ...and it must still separate two different pull requests. + if (group('pull_request', { prNumber: 7 }) === group('pull_request', { prNumber: 8 })) { + findings.push('two different pull requests share one concurrency group (the key ignores the PR number)'); + } + // ...and a queue build must not be cancelled by a push to main. + if (group('merge_group') === push) { + findings.push('a `merge_group` build and a `push` run share one group'); + } + + return { ok: findings.length === 0, findings, groups: { push, schedule, dispatch } }; +} + +// --------------------------------------------------------------------------- +// HALF 2 -- the card a red scheduled run files +// --------------------------------------------------------------------------- + +// ⛔ The marker is PLAIN TEXT, never an HTML comment. This platform's body +// sanitizer is measured to eat short angle-bracket fragments, and a de-dup key +// that can be swallowed files a duplicate an hour. +const MARKER_PREFIX = 'os-hourly-full-run'; +const TITLE_PREFIX = 'hourly full run: red on main'; + +// ONE CARD PER WORKFLOW, not one card for both, and that is a decision rather +// than an oversight. `CI` and `Lint & Type Check` are two different batteries +// that go red for unrelated reasons; a single identity would make whichever +// filer ran second overwrite the other's diagnosis with its own, and the body +// is REWRITTEN on a refresh (a comment per run is the thing this shape avoids). +// The card's identity is therefore the workflow's name, and "one red scheduled +// run files exactly one card" holds per run, which is the unit that is red. +export function cardIdentity(workflowName) { + const name = String(workflowName ?? '').trim(); + if (name === '') throw new Error('cardIdentity: the workflow name is required -- it IS the identity'); + const slug = name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + if (slug === '') throw new Error(`cardIdentity: '${name}' slugs to nothing, so it cannot be an identity`); + return { + workflowName: name, + title: `${TITLE_PREFIX} (${name})`, + marker: `${MARKER_PREFIX}-${slug}`, + labels: ['bug', 'domain:devx', 'priority:p1'], + }; +} + +// Is this issue THIS card? Two ways in, because either can be edited away: the +// title prefix, and the body marker. A PULL REQUEST is never this card -- +// `listForRepo` returns PRs too, and a PR titled after this card would +// otherwise swallow the file and the real red would never be reported. +export function matchesCard(issue, identity) { + if (!issue || issue.pull_request) return false; + const title = String(issue.title ?? ''); + const body = String(issue.body ?? ''); + return title.startsWith(identity.title) || body.includes(identity.marker); +} + +// The de-dup decision, over a PAGED listing of OPEN issues. +// +// `listPage(n)` returns that page's issues; a page shorter than `perPage` ends +// the listing. A listing that hits `maxPages` without ending has NOT +// established absence, and filing on an unestablished absence is exactly how a +// filer mints a duplicate every hour -- so it throws rather than creating. +export async function findExistingCard({ identity, listPage, perPage = 100, maxPages = 10 }) { + const found = []; + let complete = false; + for (let page = 1; page <= maxPages; page += 1) { + const batch = (await listPage(page)) ?? []; + for (const issue of batch) if (matchesCard(issue, identity)) found.push(issue); + if (batch.length < perPage) { + complete = true; + break; + } + } + if (!complete) { + throw new Error( + `the open-issue scan hit its ${maxPages}-page bound without completing -- absence is NOT ` + + 'established, so nothing was filed. The verdict is in this run summary.' + ); + } + // The OLDEST open card wins: that is the one any duplicates were closed + // against, and the one whose grading the devx seat already did. + return found.sort((a, b) => a.number - b.number)[0] ?? null; +} + +// The card body. The run link and the commit range are the whole product: a red +// hourly run says "main broke somewhere in the last hour", and the commits +// between the previous GREEN scheduled run and this one are that hour. +export function renderBody({ + identity, + runUrl, + headSha, + conclusion, + sweptAt, + previousGreen = null, + commits = [], + compareUrl = null, + commitsTruncated = false, + // Why no range could be named, in the caller's words. The two reasons -- no + // previous green scheduled run in the API window, and a compare call that + // failed -- read identically in the body without it, and they point in + // opposite directions. + rangeNote = null, +}) { + const range = previousGreen + ? `\`${String(previousGreen.head_sha).slice(0, 10)}\` (run ${previousGreen.run_id}, the last green ` + + `\`${identity.workflowName}\` schedule run) .. \`${String(headSha).slice(0, 10)}\`` + : null; + + const suspects = previousGreen + ? commits.length > 0 + ? [ + `## What landed since the last green hourly run`, + '', + `Range: ${range}${compareUrl ? ` — [compare](${compareUrl})` : ''}`, + commitsTruncated + ? '\n_Truncated by the compare API; open the compare link for the full list._' + : '', + '', + ...commits.map((c) => `- \`${String(c.sha).slice(0, 10)}\` ${String(c.title).split('\n')[0]}`), + '', + ] + : [ + `## What landed since the last green hourly run`, + '', + `Range: ${range}`, + '', + '⚠️ Nothing landed in that range. The same tree was green an hour ago and is red now, so', + 'this is a FLAKE, an infrastructure fault, or a suite that depends on wall-clock time or on', + 'something outside the repository. ⛔ Do not go looking for the commit that broke it.', + '', + ] + : [ + `## What landed since the last green hourly run`, + '', + '⚠️ No commit range could be named for this red run.', + rangeNote ?? '', + 'Read the run log directly. ⛔ Do not take the absence of a range as "nothing changed": it is the', + 'absence of a READING, which is a different fact and points at this filer rather than at `main`.', + '', + ]; + + return [ + `${identity.marker} — machine-findable marker for this generated card. ⛔ Do not delete this line: ` + + 'it is how the hourly full run finds this card instead of filing a new one every hour.', + '', + `# ${identity.title}`, + '', + `_Swept ${sweptAt} · [run log](${runUrl}) · commit \`${headSha}\` · conclusion \`${conclusion}\`._`, + '', + `\`${identity.workflowName}\` runs the FULL battery on \`main\` every hour (#16467). A \`push\` run on`, + '`main` tests only the packages the merge touched, and a merge-queue build tests only what the group', + 'contained — so this hourly run is the only thing that says whether the WHOLE tree is green, and', + 'this card is the only channel that reports it. Nothing is blocked by it.', + '', + '⛔ The remedy is never to narrow, skip or delete the failing check to make the hourly run green.', + 'Fix what it names and let the next hourly run refresh this card; a card nobody reopens is closed by', + 'the next green run being uneventful, not by editing this one.', + '', + ...suspects, + '_Filed by `.github/workflows/scheduled-full-run-card.yml`. Generated by [Claude Code](https://claude.ai/code)_', + ] + .filter((line) => line !== '') + .join('\n'); +} + +// --------------------------------------------------------------------------- +// -- The self-test's own battery roster and floor --------------------------- +// +// Same shape as its siblings: what is pinned is the registered NAMES, and each +// count is a FLOOR -- a battery below it means cases stopped running, and the +// remedy is to find what stopped registering, never to lower the number. +const SELF_TEST_BATTERIES = Object.freeze({ + 'scheduled-full-run concurrency isolation': 20, + 'scheduled-full-run card identity and de-dup': 23, +}); +const SELF_TEST_BATTERY_FLOOR = 2; +const UNATTRIBUTED_BATTERY = '(no battery open)'; + +// Returned by `selfTest()` only after its verdict is printed, so a `return` +// that leaves the function early cannot report as a pass. +const SELF_TEST_VERDICT = 'scheduled-full-run self-test reached its verdict'; + +async function selfTest() { + const batterySeen = new Map(); + let openBattery = null; + const battery = (name) => { + openBattery = name; + }; + // AWAITS its case. Half the cases below drive `findExistingCard`, which is + // async because the live caller pages the API; a `check` that only CALLED an + // async case would register it, return, and let its rejection surface as an + // unhandled promise -- a self-test that reports a pass while a case failed. + const check = async (fn) => { + const b = openBattery ?? UNATTRIBUTED_BATTERY; + batterySeen.set(b, (batterySeen.get(b) ?? 0) + 1); + await fn(); + }; + const threw = (fn) => { + try { + fn(); + return false; + } catch { + return true; + } + }; + const threwAsync = async (fn) => { + try { + await fn(); + return false; + } catch { + return true; + } + }; + + battery('scheduled-full-run concurrency isolation'); + + // -- The reader. It must find the WORKFLOW-level group and refuse a file that + // has none, because "no group" would otherwise evaluate to two equal + // empty strings and read as a collision that is not there. + const SAMPLE = [ + 'name: X', + 'on:', + ' push:', + 'concurrency:', + " group: g-${{ github.event_name }}-${{ github.ref }}", + ' cancel-in-progress: true', + 'jobs:', + ' a:', + ' concurrency:', + ' group: job-level-must-not-be-read', + ].join('\n'); + await check(() => { + if (readConcurrencyGroup(SAMPLE) !== 'g-${{ github.event_name }}-${{ github.ref }}') { + throw new Error(`reader: got '${readConcurrencyGroup(SAMPLE)}'`); + } + }); + await check(() => { + if (!threw(() => readConcurrencyGroup('name: X\njobs:\n a:\n concurrency:\n group: g\n'))) { + throw new Error('reader: a job-level group was read as the workflow-level one'); + } + }); + await check(() => { + if (!threw(() => readConcurrencyGroup('concurrency:\n cancel-in-progress: true\n'))) { + throw new Error('reader: a `concurrency:` block with no `group:` was accepted'); + } + }); + + // -- The evaluator, including the falls-through and the refusal. + await check(() => { + const ctx = eventContext('pull_request', { workflow: 'CI', prNumber: 12 }); + const got = evaluateExpression('github.event.pull_request.number || github.ref', ctx); + if (got !== '12') throw new Error(`evaluator: expected '12', got '${got}'`); + }); + await check(() => { + const ctx = eventContext('push', { workflow: 'CI' }); + const got = evaluateExpression('github.event.pull_request.number || github.ref', ctx); + if (got !== 'refs/heads/main') throw new Error(`evaluator: '||' did not fall through, got '${got}'`); + }); + await check(() => { + if (evaluateExpression("'literal'", eventContext('push')) !== 'literal') { + throw new Error('evaluator: a quoted literal did not evaluate'); + } + }); + await check(() => { + if (!threw(() => evaluateExpression('github.event_name == 42', eventContext('push')))) { + throw new Error('evaluator: an operand outside the grammar was guessed instead of refused'); + } + }); + await check(() => { + const got = evaluateGroup('ci-${{ github.workflow }}-${{ github.event_name }}', eventContext('schedule', { workflow: 'CI' })); + if (got !== 'ci-CI-schedule') throw new Error(`evaluator: interpolation produced '${got}'`); + }); + + // -- The verdict. FIRING CONTROL first: the shape this gate exists to + // refuse -- today's key with `github.event_name` taken back out -- must + // be judged BAD, and it must be judged bad for the schedule/push pair by + // name. A gate that only ever reads the good shape is a gate that has + // never been shown to fail. + const BAD = 'ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + const GOOD = + 'ci-${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}'; + await check(() => { + const v = concurrencyVerdict(BAD, { workflow: 'CI' }); + if (v.ok) throw new Error('firing control: the pre-#16467 key was accepted'); + }); + await check(() => { + const v = concurrencyVerdict(BAD, { workflow: 'CI' }); + if (!v.findings.some((f) => f.includes('`schedule`') && f.includes('`push`'))) { + throw new Error(`firing control: the schedule/push collision was not named (${v.findings.join(' | ')})`); + } + }); + await check(() => { + const v = concurrencyVerdict(BAD, { workflow: 'CI' }); + if (v.groups.schedule !== v.groups.push) throw new Error('firing control: the two groups were not actually equal'); + }); + await check(() => { + const v = concurrencyVerdict(GOOD, { workflow: 'CI' }); + if (!v.ok) throw new Error(`the fixed key was rejected: ${v.findings.join(' | ')}`); + }); + await check(() => { + const v = concurrencyVerdict(GOOD, { workflow: 'CI' }); + if (v.groups.schedule === v.groups.push) throw new Error('the fixed key still collapses schedule onto push'); + }); + + // -- NONSENSE CONTROL. A key that separates every event by making every RUN + // unique passes the inequality above and silently disables + // `cancel-in-progress` everywhere. It must be refused. + await check(() => { + const v = concurrencyVerdict('ci-${{ github.run_id }}', { workflow: 'CI' }); + if (v.ok) throw new Error('nonsense control: a run-unique key was accepted'); + }); + await check(() => { + const v = concurrencyVerdict('ci-${{ github.run_id }}', { workflow: 'CI' }); + if (!v.findings.some((f) => f.includes('two different pull requests'))) { + throw new Error(`nonsense control: expected the PR-separation finding (${v.findings.join(' | ')})`); + } + }); + await check(() => { + // ...and a constant key, the other degenerate direction: every event in one + // group, which is the collision this gate is about at maximum strength. + const v = concurrencyVerdict('ci-fixed', { workflow: 'CI' }); + if (v.ok) throw new Error('nonsense control: a constant key was accepted'); + }); + + // -- THE REAL FILES. The two workflows are read from disk and judged, so the + // gate's production verdict is exercised by its own self-test too. + for (const file of WATCHED_WORKFLOWS) { + await check(() => { + const group = readConcurrencyGroup(readFileSync(file, 'utf8')); + const v = concurrencyVerdict(group, { workflow: file }); + if (!v.ok) throw new Error(`${file}: ${v.findings.join(' | ')}`); + }); + await check(() => { + const group = readConcurrencyGroup(readFileSync(file, 'utf8')); + const v = concurrencyVerdict(group, { workflow: file }); + if (v.groups.schedule === v.groups.push) { + throw new Error(`${file}: schedule and push still evaluate to '${v.groups.push}'`); + } + }); + } + + battery('scheduled-full-run card identity and de-dup'); + + const ci = cardIdentity('CI'); + const lint = cardIdentity('Lint & Type Check'); + + await check(() => { + if (ci.title !== 'hourly full run: red on main (CI)') throw new Error(`identity: CI title is '${ci.title}'`); + }); + await check(() => { + if (lint.marker !== 'os-hourly-full-run-lint-type-check') throw new Error(`identity: lint marker is '${lint.marker}'`); + }); + await check(() => { + if (ci.marker === lint.marker || ci.title === lint.title) { + throw new Error('identity: the two workflows share an identity, so one card would overwrite the other'); + } + }); + await check(() => { + if (/[<>]/.test(ci.marker) || /[<>]/.test(lint.marker)) { + throw new Error('identity: the marker carries an angle bracket, which the body sanitizer eats'); + } + }); + await check(() => { + if (!threw(() => cardIdentity(''))) throw new Error('identity: an empty workflow name was accepted'); + }); + await check(() => { + if (!threw(() => cardIdentity('!!!'))) throw new Error('identity: a name that slugs to nothing was accepted'); + }); + + // -- Matching. Both ways in, and the three things that are NOT this card. + const issue = (n, extra = {}) => ({ number: n, title: 'unrelated', body: '', ...extra }); + await check(() => { + if (!matchesCard(issue(1, { title: ci.title }), ci)) throw new Error('match: the exact title did not match'); + }); + await check(() => { + if (!matchesCard(issue(1, { title: `${ci.title} — since 09:00Z` }), ci)) { + throw new Error('match: a title with a suffix did not match its prefix'); + } + }); + await check(() => { + if (!matchesCard(issue(1, { title: 'renamed by a human', body: `x\n${ci.marker}\ny` }), ci)) { + throw new Error('match: the body marker did not match after a rename'); + } + }); + await check(() => { + if (matchesCard(issue(1, { title: ci.title, pull_request: { url: 'u' } }), ci)) { + throw new Error('match: a PULL REQUEST with the card title matched'); + } + }); + await check(() => { + if (matchesCard(issue(1, { title: lint.title, body: lint.marker }), ci)) { + throw new Error("match: the Lint card matched CI's identity"); + } + }); + await check(() => { + if (matchesCard(issue(1), ci)) throw new Error('match: an unrelated issue matched'); + }); + + // -- The de-dup decision. The firing control is the pair that matters: + // ONE red run with no existing card creates, and the SECOND finds it. + const page = (items) => async (n) => (n === 1 ? items : []); + await check(async () => { + if ((await findExistingCard({ identity: ci, listPage: page([]) })) !== null) { + throw new Error('de-dup: an empty board reported an existing card'); + } + }); + const firstCard = { number: 42, title: ci.title, body: ci.marker }; + await check(async () => { + const found = await findExistingCard({ identity: ci, listPage: page([issue(7), firstCard]) }); + if (found?.number !== 42) throw new Error(`de-dup: the existing card was not found (${found?.number})`); + }); + await check(async () => { + const found = await findExistingCard({ + identity: ci, + listPage: page([{ number: 99, title: ci.title }, firstCard]), + }); + if (found?.number !== 42) throw new Error(`de-dup: the OLDEST duplicate did not win (${found?.number})`); + }); + await check(async () => { + const found = await findExistingCard({ identity: ci, listPage: page([{ number: 42, title: lint.title, body: lint.marker }]) }); + if (found !== null) throw new Error("de-dup: the Lint card was taken for CI's, so one would overwrite the other"); + }); + await check(async () => { + // A truncated scan cannot establish absence, so it must refuse rather than + // file. Full pages forever is what that looks like. + const full = async () => Array.from({ length: 100 }, (_, i) => issue(i + 1)); + if (!(await threwAsync(() => findExistingCard({ identity: ci, listPage: full, maxPages: 3 })))) { + throw new Error('de-dup: a truncated scan was treated as established absence'); + } + }); + await check(async () => { + // ...and a listing that ends exactly on a full page is COMPLETE. + const pages = [Array.from({ length: 100 }, (_, i) => issue(i + 1)), [firstCard]]; + const found = await findExistingCard({ identity: ci, listPage: async (n) => pages[n - 1] ?? [], maxPages: 3 }); + if (found?.number !== 42) throw new Error('de-dup: a multi-page scan lost the card'); + }); + + // -- The body. What is pinned is that the parts a reader needs survive. + const body = renderBody({ + identity: ci, + runUrl: 'https://example.invalid/run/5', + headSha: 'abcdef1234567890', + conclusion: 'failure', + sweptAt: '2026-09-08T04:00:00.000Z', + previousGreen: { run_id: 4, head_sha: '1111111111222222' }, + commits: [{ sha: '9999999999000000', title: 'feat: a thing\n\nbody' }], + compareUrl: 'https://example.invalid/compare', + }); + await check(() => { + if (!body.startsWith(ci.marker)) throw new Error('body: the marker is not the first thing in it'); + }); + await check(() => { + for (const fragment of ['https://example.invalid/run/5', '9999999999', 'feat: a thing', '1111111111']) { + if (!body.includes(fragment)) throw new Error(`body: '${fragment}' is missing`); + } + }); + await check(() => { + if (body.includes('\n\nbody')) throw new Error('body: a commit message body leaked past its first line'); + }); + await check(() => { + const empty = renderBody({ + identity: ci, + runUrl: 'u', + headSha: 'a', + conclusion: 'failure', + sweptAt: 't', + previousGreen: { run_id: 4, head_sha: 'b' }, + commits: [], + }); + if (!empty.includes('FLAKE')) throw new Error('body: an empty commit range did not name the flake reading'); + }); + await check(() => { + const none = renderBody({ + identity: ci, + runUrl: 'u', + headSha: 'a', + conclusion: 'failure', + sweptAt: 't', + rangeNote: 'the compare call failed with HTTP 422', + }); + if (!none.includes('No commit range could be named') || !none.includes('HTTP 422')) { + throw new Error("body: a missing range, or the caller's reason for it, was not reported"); + } + }); + + // -- The floor. Evaluated before the verdict, so the success line can only be + // printed by a run in which the set of batteries that registered EQUALS + // the set declared. + const floorFailures = []; + const declared = Object.keys(SELF_TEST_BATTERIES); + if (declared.length < SELF_TEST_BATTERY_FLOOR) { + floorFailures.push( + `SELF_TEST_BATTERIES declares ${declared.length} batteries, below the pinned ${SELF_TEST_BATTERY_FLOOR}.` + ); + } + for (const [name, count] of batterySeen) { + if (declared.includes(name)) continue; + floorFailures.push(`battery "${name}" registered ${count} case(s) but is not declared.`); + } + for (const name of declared) { + const count = batterySeen.get(name) ?? 0; + if (count >= SELF_TEST_BATTERIES[name]) continue; + floorFailures.push( + count === 0 + ? `battery "${name}" DID NOT RUN -- 0 cases registered, ${SELF_TEST_BATTERIES[name]} pinned.` + : `battery "${name}" registered ${count} case(s), below its pinned floor of ${SELF_TEST_BATTERIES[name]}.` + ); + } + if (floorFailures.length > 0) { + throw new Error( + `scheduled-full-run self-test floor (${floorFailures.length} breach(es)):\n` + + floorFailures.map((f) => ` - ${f}`).join('\n') + + '\n A battery at or below its floor means cases STOPPED RUNNING -- the battery is the bug, ' + + 'not the number.' + ); + } + + console.log('scheduled-full-run: self-test OK'); + return SELF_TEST_VERDICT; +} + +function checkConcurrency() { + let bad = 0; + for (const file of WATCHED_WORKFLOWS) { + const group = readConcurrencyGroup(readFileSync(file, 'utf8')); + const verdict = concurrencyVerdict(group, { workflow: file }); + if (verdict.ok) { + console.log( + `scheduled-full-run: ${file} OK -- push '${verdict.groups.push}' and schedule ` + + `'${verdict.groups.schedule}' are different concurrency groups.` + ); + continue; + } + bad += 1; + for (const finding of verdict.findings) console.error(`scheduled-full-run: ${file}: ${finding}`); + } + if (bad > 0) { + console.error( + `scheduled-full-run: ${bad} workflow file(s) let a push to main cancel the hourly full run. ` + + 'That run is the only thing testing the whole workspace on main since push went affected-only, ' + + 'so this is a coverage refusal, not a style one.' + ); + process.exit(1); + } +} + +async function main() { + const argv = process.argv.slice(2); + + if (argv.includes('--self-test')) { + if ((await selfTest()) !== SELF_TEST_VERDICT) { + console.error( + '\nx scheduled-full-run self-test: selfTest() returned without reaching its verdict,\n' + + 'so no success line was printed. Exiting 0 here would report a self-test that never\n' + + 'finished as a self-test that passed.\n' + ); + process.exit(1); + } + return; + } + + if (argv.includes('--check-concurrency')) { + checkConcurrency(); + return; + } + + console.error( + 'usage: scheduled-full-run.mjs --check-concurrency\n' + + ' scheduled-full-run.mjs --self-test' + ); + process.exit(1); +} + +// Exports bindings, so an import for those exports alone must run nothing. +if (isEntrypoint(import.meta.url)) { + await main(); +} diff --git a/scripts/ci/select-shard-packages.selftest.sh b/scripts/ci/select-shard-packages.selftest.sh index f2e6c4a19e..11fac31bb0 100644 --- a/scripts/ci/select-shard-packages.selftest.sh +++ b/scripts/ci/select-shard-packages.selftest.sh @@ -165,15 +165,30 @@ git_q -C "$UP" branch release "$C1" SHALLOW="$FIX/shallow" git_q clone -q --depth 1 "file://$UP" "$SHALLOW" 2>/dev/null +# A SECOND shallow clone, for the push fetch case. One clone cannot serve both: +# the fetch under test mutates it, so the case that ran second would open on a +# checkout that already holds the object -- its precondition would fail, and if +# the precondition were dropped instead, the case would pass with the fetch +# removed from the script entirely. +SHALLOW_PUSH="$FIX/shallow-push" +git_q clone -q --depth 1 "file://$UP" "$SHALLOW_PUSH" 2>/dev/null ZEROS=0000000000000000000000000000000000000000 +# Well-formed, and in no fixture repository: the "resolvable shape, absent +# object" case, which is what a force-pushed-away previous tip looks like. +ABSENT=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef # ── The runner and the assertions ─────────────────────────────────────────── at() { git_q -C "$REPO" checkout -q --detach "$1"; } RT='' rc=0 -# run_case