From 248b057efd9bbf59ed824f282f3734f1626ae7de Mon Sep 17 00:00:00 2001 From: "claude[bot]" Date: Sun, 6 Sep 2026 04:30:16 +0000 Subject: [PATCH] ci: derive a job ceiling for lint, bundle-analysis and release (objectui#7270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three jobs declared no job-level `timeout-minutes`, so their only backstop was GitHub's 360-minute default. Two of them are required contexts, and this repository has twice had a transient fault converted into a `cancelled` check by a job ceiling (objectui#5304, objectui#6577) — a verdict the merge queue cannot tell from `failure`. Each ceiling is derived from that job's own measured run distribution, never inherited from `ci.yml` (objectui#7048 fences that), and the derivation is recorded beside the key: window, sample size, min/median/p95/max, and the rule. - `lint.yml::lint` 25 (n=300, max 7m14s) - `performance-budget.yml::bundle-analysis` 25 (n=300, max 5m23s) - `changeset-release.yml::release` 40 (n=25, max 7m15s, plus the 900-second wait cap the job declares for itself) `workflow-cache-save-bound.test.ts` gains a pin so removing a key or raising one of these ceilings is no longer invisible. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013uAaxiwgYDybsTNV9xwa1M --- .github/workflows/changeset-release.yml | 59 +++++++++++++-- .github/workflows/lint.yml | 52 ++++++++++++-- .github/workflows/performance-budget.yml | 53 ++++++++++++-- .../workflow-cache-save-bound.test.ts | 72 ++++++++++++++++++- 4 files changed, 213 insertions(+), 23 deletions(-) diff --git a/.github/workflows/changeset-release.yml b/.github/workflows/changeset-release.yml index 0496a9aa2c..9b1e679175 100644 --- a/.github/workflows/changeset-release.yml +++ b/.github/workflows/changeset-release.yml @@ -445,6 +445,47 @@ jobs: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.refresh_version_pr) runs-on: ubuntu-latest + + # ── The job ceiling, DERIVED FOR THIS JOB (objectui#7270) ──────────── + # Before this line the only backstop was GitHub's 360-minute default, on + # the lane that PUBLISHES. This ceiling is derived differently from the two + # that objectui#7270 set on `Lint` and `Bundle Analysis`, because this + # job's honest slowest run is not the slowest run in its sample: the `Wait + # for older release runs to finish` step below declares + # `MAX_WAIT_SECONDS: '900'`, so a contended release may legitimately spend + # 15 minutes waiting before it does any work at all. None of the sampled + # runs was contended, so that 15 minutes appears nowhere in the + # distribution below and has to be ADDED rather than measured. + # + # ⛔ NOT inherited from `ci.yml`'s 10/15/20/30/40 — objectui#7048 fences + # exactly that. The arithmetic below is this job's own. + # + # - Population: every successful `schedule` run of this workflow, window + # 2026-08-23T06:08Z .. 2026-09-06T00:23Z, n=25 `Changeset Release` + # jobs. Per-job wall clock from the Actions jobs endpoint + # (`completed_at` minus `started_at`), never the run's total. + # - min 1m23s / median 2m37s / p95 5m37s / max 7m15s. + # - ⚠️ n=25 is the whole recent population, not a sample cut short. This + # job is `skipped` on a push unless that push left the manifest version + # absent from npm, and the 400 most recent successful `push` runs + # (2026-08-30 .. 2026-09-06) skipped it 400 times out of 400. Paging + # further buys skips, not samples. + # - max/p95 = 1.29 — no fat tail inside the sample. + # + # Ceiling = 15min (the declared wait cap, taken at face value: it is + # already a hard, fail-open bound, so multiplying it would only buy slack + # for a clock that cannot overrun) + 3x the measured max work + # (3 x 7m15s = 21.8min) => 36.8min, rounded up to 40. The plain rule the + # other two jobs use would have given 25, which leaves a legitimately + # contended release (15min of waiting + 7m15s of work = 22m15s) inside 12% + # of its own ceiling — a ceiling set under a job's honest slowest run, + # which is the mirror hazard objectui#7048 fenced. + # + # ⛔ Raising this toward 360 is the ruled-out non-fix, and ⛔ lowering it + # under 23 minutes would let the clock truncate a release that the workflow + # itself says is behaving. + timeout-minutes: 40 + steps: # The queue the `concurrency:` key above cannot provide. GitHub offers exactly two # behaviours for a contended group — cancel the running one, or discard the pending @@ -591,13 +632,17 @@ jobs: # `actions/cache`'s own `save-always` deprecation text points at this # same split. # - # ⚠️ This job declares NO `timeout-minutes`, so its ceiling is GitHub's - # default of 360 minutes — and this is the PUBLISH lane, where a job that - # hangs after `Publish to npm` has already shipped and the concurrency - # note above records what a long-running release job does to the pushes - # behind it. ⛔ Adding a ceiling is NOT the fix and is not attempted here: - # the release act must never be truncated by a clock. The bound below is - # on the bookkeeping step alone, which runs after every release step. + # ⚠️ This job's ceiling is `timeout-minutes: 40`, DERIVED at the job + # header above from this job's own run distribution PLUS the 900-second + # wait cap this job declares for itself (objectui#7270). Until that line + # existed the ceiling was GitHub's 360-minute default — and this is the + # PUBLISH lane, where a job that hangs after `Publish to npm` has already + # shipped and the concurrency note above records what a long-running + # release job does to the pushes behind it. The constraint that used to + # argue against any ceiling at all is the one that shaped 40: the release + # act must never be truncated by a clock, so the number is generous + # rather than tight. The bound below is still on the bookkeeping step + # alone, which runs after every release step. # # The restore half is deliberately left UNBOUNDED: a restore stall fails # BEFORE anything is published, which is the honest failure. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6832840e2a..414d8c0a28 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -87,6 +87,43 @@ jobs: name: Lint runs-on: ubuntu-latest + # ── The job ceiling, DERIVED FOR THIS JOB (objectui#7270) ──────────── + # Before this line the only backstop was GitHub's 360-minute default. This + # repository has been bitten twice by an unbounded step whose only backstop + # was the job ceiling — the `apt-get` hang (objectui#5304) and the cache-save + # stall that ejected an all-green pull request (objectui#6577) — and both + # times the ceiling converted a transient fault into a `cancelled` check, + # which the merge queue cannot tell from `failure`. `Lint` is a required + # context on `pull_request` and `merge_group` alike, so at 360 the same + # fault holds a shared serial queue for six hours. + # + # ⛔ NOT inherited from `ci.yml`'s 10/15/20/30/40 — objectui#7048 fences + # exactly that. The arithmetic below is this job's own. + # + # - Population: the last 300 successful runs of this workflow, window + # 2026-09-05T11:51Z .. 2026-09-06T04:11Z, n=300 `Lint` jobs. Per-job + # wall clock from the Actions jobs endpoint (`completed_at` minus + # `started_at`), never the run's total. + # - min 9s / median 5m16s / p95 5m46s / max 7m14s. + # - Bimodal at the LOW end only: 11 runs finished in 9-14s, which is the + # `Decide whether this change needs a full run` gate skipping every + # step; the other 289 ran 3m27s-7m14s. A low mode cannot raise a + # ceiling, so it is recorded and not used. + # - By event, no material difference, so one ceiling covers all three: + # merge_group n=103 (median 5m17s, max 6m06s), pull_request n=142 + # (median 5m15s, max 6m39s), push n=55 (median 5m14s, max 7m14s). + # - max/p95 = 1.25 — no fat tail, so the multiplier is not widened. + # + # Ceiling = the smallest round number that is both >= 3x max (21.7min) and + # >= max + 15min (22.2min) => 25. That is ~3.5x the slowest run ever + # observed here, so it cannot fire on a healthy-but-slow runner; a wedge + # dies in 25 minutes instead of 6 hours. + # + # ⛔ Raising this toward 360 is the ruled-out non-fix (objectui#6577, + # objectui#7048): the hang just runs longer and the gate still reports + # `cancelled`. + timeout-minutes: 25 + steps: - name: Checkout code uses: actions/checkout@v7 @@ -257,13 +294,14 @@ jobs: # pull request (objectui#6577). `actions/cache`'s own `save-always` # deprecation text points at this same split. # - # ⚠️ This job declares NO `timeout-minutes`, so its ceiling is GitHub's - # default of 360 minutes. That makes the exposure worse here, not better: - # `Lint` is a required context on `pull_request` and `merge_group` alike, - # so a stalled save would hold the shared serial queue for up to six - # hours before reporting `cancelled`. ⛔ Adding a ceiling is NOT the fix - # and is not attempted here — it would change what this gate rejects. The - # bound below is on the bookkeeping step alone. + # ⚠️ This job's ceiling is `timeout-minutes: 25`, DERIVED at the job + # header above from this job's own run distribution (objectui#7270). + # Until that line existed the ceiling was GitHub's 360-minute default, + # and `Lint` is a required context on `pull_request` and `merge_group` + # alike, so a stalled save would have held the shared serial queue for up + # to six hours before reporting `cancelled`. The bound below is still on + # the bookkeeping step alone, and both numbers are needed: 25 minutes is + # the backstop, 5 minutes is what a healthy save is allowed. # # The restore half is deliberately left UNBOUNDED: a restore stall fails # BEFORE any verdict exists — a gate that did not run, which is honest — diff --git a/.github/workflows/performance-budget.yml b/.github/workflows/performance-budget.yml index f5516b83c7..2d21732d8f 100644 --- a/.github/workflows/performance-budget.yml +++ b/.github/workflows/performance-budget.yml @@ -110,6 +110,45 @@ jobs: bundle-analysis: name: Bundle Analysis runs-on: ubuntu-latest + + # ── The job ceiling, DERIVED FOR THIS JOB (objectui#7270) ──────────── + # Before this line the only backstop was GitHub's 360-minute default, on a + # job this repository treats as a required context (objectui#6245). Two + # incidents put a number on that exposure: objectui#5304 (`apt-get` hang) + # and objectui#6577 (cache-save stall), both of which turned a transient + # fault into a `cancelled` check — a gate that reports nothing. + # + # ⚠️ Measured while deriving this, and worth knowing before reusing the + # `Lint` argument here: this workflow subscribes only `push` and + # `pull_request`, NOT `merge_group`, and 300 sampled runs contain zero + # `merge_group` runs. So this job cannot hold the merge queue the way + # `Lint` can; what six hours costs here is the pull request and the runner. + # The ceiling is still derived, because that exposure is real on its own. + # + # ⛔ NOT inherited from `ci.yml`'s 10/15/20/30/40 — objectui#7048 fences + # exactly that. The arithmetic below is this job's own. + # + # - Population: the last 300 successful runs of this workflow, window + # 2026-09-03T11:40Z .. 2026-09-06T04:10Z, n=300 `Bundle Analysis` jobs. + # Per-job wall clock from the Actions jobs endpoint (`completed_at` + # minus `started_at`), never the run's total. + # - min 57s / median 3m35s / p95 4m37s / max 5m23s. + # - Broad rather than bimodal: 72 runs under 2m and 228 at or above it, + # spread continuously across the range (Turbo cache hit versus miss), + # with no isolated tail. + # - By event: pull_request n=191 (median 3m37s, max 5m23s), push n=109 + # (median 3m31s, max 4m47s). No material difference. + # - max/p95 = 1.17 — no fat tail, so the multiplier is not widened. + # + # Ceiling = the smallest round number that is both >= 3x max (16.2min) and + # >= max + 15min (20.4min) => 25; the second leg is the one that binds. + # That is ~4.6x the slowest run ever observed here. + # + # ⛔ Raising this toward 360 is the ruled-out non-fix (objectui#6577, + # objectui#7048): the hang just runs longer and the gate still reports + # `cancelled`. + timeout-minutes: 25 + steps: - name: Checkout code uses: actions/checkout@v7 @@ -152,12 +191,14 @@ jobs: # pull request (objectui#6577). `actions/cache`'s own `save-always` # deprecation text points at this same split. # - # ⚠️ This job declares NO `timeout-minutes`, so its ceiling is GitHub's - # default of 360 minutes, and `Bundle Analysis` is a required context - # (objectui#6245). A stalled save would therefore hold a required check - # open for up to six hours before reporting `cancelled`. ⛔ Adding a - # ceiling is NOT the fix and is not attempted here — it would change what - # this gate rejects. The bound below is on the bookkeeping step alone. + # ⚠️ This job's ceiling is `timeout-minutes: 25`, DERIVED at the job + # header above from this job's own run distribution (objectui#7270). + # Until that line existed the ceiling was GitHub's 360-minute default on + # a required context (objectui#6245), so a stalled save would have held + # that check open for up to six hours before reporting `cancelled`. The + # bound below is still on the bookkeeping step alone, and both numbers + # are needed: 25 minutes is the backstop, 5 minutes is what a healthy + # save is allowed. # # The restore half is deliberately left UNBOUNDED: a restore stall fails # BEFORE any verdict exists — a gate that did not run, which is honest — diff --git a/scripts/__tests__/workflow-cache-save-bound.test.ts b/scripts/__tests__/workflow-cache-save-bound.test.ts index a861932442..be529a56d5 100644 --- a/scripts/__tests__/workflow-cache-save-bound.test.ts +++ b/scripts/__tests__/workflow-cache-save-bound.test.ts @@ -40,9 +40,11 @@ import { parse as parseYaml } from 'yaml'; * * Fold any split back into one `actions/cache` step and nothing goes red. The * cache still works, every run still passes, and the next transient upload stall - * ejects the next green pull request — or, in the three jobs here that declare - * no `timeout-minutes` at all, holds a required context open against GitHub's - * 360-minute default. That is this repository's recurring "looks like + * ejects the next green pull request. Until objectui#7270 three of these jobs + * declared no `timeout-minutes` at all, so the same stall held a required + * context open against GitHub's 360-minute default instead; those three now + * carry ceilings derived from their own run distributions, and the last pin in + * this file is what keeps them. That is this repository's recurring "looks like * enforcement, isn't" class (objectui#3009, #3181, #3494). * * ## Deliberately NOT asserted @@ -281,4 +283,68 @@ describe('every workflow cache save is bounded and non-fatal (objectui#7048)', ( ).toBeLessThanOrEqual(max); } }); + + it('bounds the three jobs objectui#7270 found running under the 360-minute default', () => { + // Presence and bound in one assertion, because the two ways this can be + // undone need the same fix site and neither is visible in a run. + // + // - Delete the key and the job silently returns to GitHub's 360-minute + // default. Nothing goes red: a healthy job never approaches any + // ceiling, so the only symptom is the next transient hang holding a + // required context for six hours and then reporting `cancelled` — a + // verdict the merge queue cannot tell from `failure` (objectui#5304, + // objectui#6577 are the two times this repository has paid it). + // - Raise the number and objectui#7048's fence is crossed in the other + // direction: a larger ceiling buys a longer hang and still ends in + // `cancelled`. + // + // Each number is DERIVED FOR ITS OWN JOB from that job's measured run + // distribution, and the derivation is written beside the key in the + // workflow — window, sample size, min/median/p95/max, and the rule. + // objectui#7048 fences copying `ci.yml`'s 10/15/20/30/40, so the three + // values here are not a shared constant and two of them being equal is a + // coincidence of the arithmetic. Pinning them means a future change to one + // has to move the derivation beside it in the same commit. + const derived: Record = { + 'lint.yml::lint': 25, + 'performance-budget.yml::bundle-analysis': 25, + 'changeset-release.yml::release': 40, + }; + + const missing: string[] = []; + const raised: string[] = []; + + for (const [key, ceiling] of Object.entries(derived)) { + const [file, jobKey] = key.split('::'); + const entry = allJobs.find((e) => e.file === file && e.jobKey === jobKey); + expect(entry, `${file} must still define a \`${jobKey}:\` job`).toBeDefined(); + + const declared = entry!.job['timeout-minutes']; + if (typeof declared !== 'number') { + missing.push(`${file} :: job \`${jobKey}\``); + } else if (declared > ceiling) { + raised.push(`${file} :: job \`${jobKey}\` declares ${declared}, derived ${ceiling}`); + } + } + + expect( + missing, + 'these jobs declare no job-level `timeout-minutes`, so their only backstop is GitHub\'s ' + + '360-minute default again:\n' + + missing.map((o) => ` - ${o}`).join('\n') + + '\n\nTwo of them are required contexts and one is the publish lane. Restore the key ' + + 'TOGETHER WITH the derivation comment beside it — a ceiling with no recorded provenance ' + + 'is the next reader\'s excuse to guess at it (objectui#7270).', + ).toEqual([]); + + expect( + raised, + 'these job ceilings are above the value derived for them:\n' + + raised.map((o) => ` - ${o}`).join('\n') + + '\n\nRaising a ceiling does not fix a hang — it buys a longer one and the gate still ' + + 'reports `cancelled` (objectui#6577, objectui#7048). If a job genuinely got slower, ' + + 're-derive it from a fresh distribution, rewrite the comment beside the key, and move ' + + 'this pin in the same commit.', + ).toEqual([]); + }); });