ci: derive a job ceiling for lint, bundle-analysis and release (objectui#7270)

.github/workflows/changeset-release.yml

@@ -445,6 +445,47 @@ jobs:
       github.event_name == 'schedule' ||
       (github.event_name == 'workflow_dispatch' && inputs.refresh_version_pr)
     runs-on: ubuntu-latest
+
+    # ── The job ceiling, DERIVED FOR THIS JOB (objectui#7270) ────────────
+    # Before this line the only backstop was GitHub's 360-minute default, on
+    # the lane that PUBLISHES. This ceiling is derived differently from the two
+    # that objectui#7270 set on `Lint` and `Bundle Analysis`, because this
+    # job's honest slowest run is not the slowest run in its sample: the `Wait
+    # for older release runs to finish` step below declares
+    # `MAX_WAIT_SECONDS: '900'`, so a contended release may legitimately spend
+    # 15 minutes waiting before it does any work at all. None of the sampled
+    # runs was contended, so that 15 minutes appears nowhere in the
+    # distribution below and has to be ADDED rather than measured.
+    #
+    # ⛔ NOT inherited from `ci.yml`'s 10/15/20/30/40 — objectui#7048 fences
+    # exactly that. The arithmetic below is this job's own.
+    #
+    #   - Population: every successful `schedule` run of this workflow, window
+    #     2026-08-23T06:08Z .. 2026-09-06T00:23Z, n=25 `Changeset Release`
+    #     jobs. Per-job wall clock from the Actions jobs endpoint
+    #     (`completed_at` minus `started_at`), never the run's total.
+    #   - min 1m23s / median 2m37s / p95 5m37s / max 7m15s.
+    #   - ⚠️ n=25 is the whole recent population, not a sample cut short. This
+    #     job is `skipped` on a push unless that push left the manifest version
+    #     absent from npm, and the 400 most recent successful `push` runs
+    #     (2026-08-30 .. 2026-09-06) skipped it 400 times out of 400. Paging
+    #     further buys skips, not samples.
+    #   - max/p95 = 1.29 — no fat tail inside the sample.
+    #
+    # Ceiling = 15min (the declared wait cap, taken at face value: it is
+    # already a hard, fail-open bound, so multiplying it would only buy slack
+    # for a clock that cannot overrun) + 3x the measured max work
+    # (3 x 7m15s = 21.8min) => 36.8min, rounded up to 40. The plain rule the
+    # other two jobs use would have given 25, which leaves a legitimately
+    # contended release (15min of waiting + 7m15s of work = 22m15s) inside 12%
+    # of its own ceiling — a ceiling set under a job's honest slowest run,
+    # which is the mirror hazard objectui#7048 fenced.
+    #
+    # ⛔ Raising this toward 360 is the ruled-out non-fix, and ⛔ lowering it
+    # under 23 minutes would let the clock truncate a release that the workflow
+    # itself says is behaving.
+    timeout-minutes: 40
+
     steps:
       # The queue the `concurrency:` key above cannot provide. GitHub offers exactly two
       # behaviours for a contended group — cancel the running one, or discard the pending
@@ -591,13 +632,17 @@ jobs:
       # `actions/cache`'s own `save-always` deprecation text points at this
       # same split.
       #
-      # ⚠️ This job declares NO `timeout-minutes`, so its ceiling is GitHub's
-      # default of 360 minutes — and this is the PUBLISH lane, where a job that
-      # hangs after `Publish to npm` has already shipped and the concurrency
-      # note above records what a long-running release job does to the pushes
-      # behind it. ⛔ Adding a ceiling is NOT the fix and is not attempted here:
-      # the release act must never be truncated by a clock. The bound below is
-      # on the bookkeeping step alone, which runs after every release step.
+      # ⚠️ This job's ceiling is `timeout-minutes: 40`, DERIVED at the job
+      # header above from this job's own run distribution PLUS the 900-second
+      # wait cap this job declares for itself (objectui#7270). Until that line
+      # existed the ceiling was GitHub's 360-minute default — and this is the
+      # PUBLISH lane, where a job that hangs after `Publish to npm` has already
+      # shipped and the concurrency note above records what a long-running
+      # release job does to the pushes behind it. The constraint that used to
+      # argue against any ceiling at all is the one that shaped 40: the release
+      # act must never be truncated by a clock, so the number is generous
+      # rather than tight. The bound below is still on the bookkeeping step
+      # alone, which runs after every release step.
       #
       # The restore half is deliberately left UNBOUNDED: a restore stall fails
       # BEFORE anything is published, which is the honest failure.

.github/workflows/lint.yml

@@ -87,6 +87,43 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
 
+    # ── The job ceiling, DERIVED FOR THIS JOB (objectui#7270) ────────────
+    # Before this line the only backstop was GitHub's 360-minute default. This
+    # repository has been bitten twice by an unbounded step whose only backstop
+    # was the job ceiling — the `apt-get` hang (objectui#5304) and the cache-save
+    # stall that ejected an all-green pull request (objectui#6577) — and both
+    # times the ceiling converted a transient fault into a `cancelled` check,
+    # which the merge queue cannot tell from `failure`. `Lint` is a required
+    # context on `pull_request` and `merge_group` alike, so at 360 the same
+    # fault holds a shared serial queue for six hours.
+    #
+    # ⛔ NOT inherited from `ci.yml`'s 10/15/20/30/40 — objectui#7048 fences
+    # exactly that. The arithmetic below is this job's own.
+    #
+    #   - Population: the last 300 successful runs of this workflow, window
+    #     2026-09-05T11:51Z .. 2026-09-06T04:11Z, n=300 `Lint` jobs. Per-job
+    #     wall clock from the Actions jobs endpoint (`completed_at` minus
+    #     `started_at`), never the run's total.
+    #   - min 9s / median 5m16s / p95 5m46s / max 7m14s.
+    #   - Bimodal at the LOW end only: 11 runs finished in 9-14s, which is the
+    #     `Decide whether this change needs a full run` gate skipping every
+    #     step; the other 289 ran 3m27s-7m14s. A low mode cannot raise a
+    #     ceiling, so it is recorded and not used.
+    #   - By event, no material difference, so one ceiling covers all three:
+    #     merge_group n=103 (median 5m17s, max 6m06s), pull_request n=142
+    #     (median 5m15s, max 6m39s), push n=55 (median 5m14s, max 7m14s).
+    #   - max/p95 = 1.25 — no fat tail, so the multiplier is not widened.
+    #
+    # Ceiling = the smallest round number that is both >= 3x max (21.7min) and
+    # >= max + 15min (22.2min) => 25. That is ~3.5x the slowest run ever
+    # observed here, so it cannot fire on a healthy-but-slow runner; a wedge
+    # dies in 25 minutes instead of 6 hours.
+    #
+    # ⛔ Raising this toward 360 is the ruled-out non-fix (objectui#6577,
+    # objectui#7048): the hang just runs longer and the gate still reports
+    # `cancelled`.
+    timeout-minutes: 25
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v7
@@ -257,13 +294,14 @@ jobs:
       # pull request (objectui#6577). `actions/cache`'s own `save-always`
       # deprecation text points at this same split.
       #
-      # ⚠️ This job declares NO `timeout-minutes`, so its ceiling is GitHub's
-      # default of 360 minutes. That makes the exposure worse here, not better:
-      # `Lint` is a required context on `pull_request` and `merge_group` alike,
-      # so a stalled save would hold the shared serial queue for up to six
-      # hours before reporting `cancelled`. ⛔ Adding a ceiling is NOT the fix
-      # and is not attempted here — it would change what this gate rejects. The
-      # bound below is on the bookkeeping step alone.
+      # ⚠️ This job's ceiling is `timeout-minutes: 25`, DERIVED at the job
+      # header above from this job's own run distribution (objectui#7270).
+      # Until that line existed the ceiling was GitHub's 360-minute default,
+      # and `Lint` is a required context on `pull_request` and `merge_group`
+      # alike, so a stalled save would have held the shared serial queue for up
+      # to six hours before reporting `cancelled`. The bound below is still on
+      # the bookkeeping step alone, and both numbers are needed: 25 minutes is
+      # the backstop, 5 minutes is what a healthy save is allowed.
       #
       # The restore half is deliberately left UNBOUNDED: a restore stall fails
       # BEFORE any verdict exists — a gate that did not run, which is honest —

.github/workflows/performance-budget.yml

@@ -110,6 +110,45 @@ jobs:
   bundle-analysis:
     name: Bundle Analysis
     runs-on: ubuntu-latest
+
+    # ── The job ceiling, DERIVED FOR THIS JOB (objectui#7270) ────────────
+    # Before this line the only backstop was GitHub's 360-minute default, on a
+    # job this repository treats as a required context (objectui#6245). Two
+    # incidents put a number on that exposure: objectui#5304 (`apt-get` hang)
+    # and objectui#6577 (cache-save stall), both of which turned a transient
+    # fault into a `cancelled` check — a gate that reports nothing.
+    #
+    # ⚠️ Measured while deriving this, and worth knowing before reusing the
+    # `Lint` argument here: this workflow subscribes only `push` and
+    # `pull_request`, NOT `merge_group`, and 300 sampled runs contain zero
+    # `merge_group` runs. So this job cannot hold the merge queue the way
+    # `Lint` can; what six hours costs here is the pull request and the runner.
+    # The ceiling is still derived, because that exposure is real on its own.
+    #
+    # ⛔ NOT inherited from `ci.yml`'s 10/15/20/30/40 — objectui#7048 fences
+    # exactly that. The arithmetic below is this job's own.
+    #
+    #   - Population: the last 300 successful runs of this workflow, window
+    #     2026-09-03T11:40Z .. 2026-09-06T04:10Z, n=300 `Bundle Analysis` jobs.
+    #     Per-job wall clock from the Actions jobs endpoint (`completed_at`
+    #     minus `started_at`), never the run's total.
+    #   - min 57s / median 3m35s / p95 4m37s / max 5m23s.
+    #   - Broad rather than bimodal: 72 runs under 2m and 228 at or above it,
+    #     spread continuously across the range (Turbo cache hit versus miss),
+    #     with no isolated tail.
+    #   - By event: pull_request n=191 (median 3m37s, max 5m23s), push n=109
+    #     (median 3m31s, max 4m47s). No material difference.
+    #   - max/p95 = 1.17 — no fat tail, so the multiplier is not widened.
+    #
+    # Ceiling = the smallest round number that is both >= 3x max (16.2min) and
+    # >= max + 15min (20.4min) => 25; the second leg is the one that binds.
+    # That is ~4.6x the slowest run ever observed here.
+    #
+    # ⛔ Raising this toward 360 is the ruled-out non-fix (objectui#6577,
+    # objectui#7048): the hang just runs longer and the gate still reports
+    # `cancelled`.
+    timeout-minutes: 25
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v7
@@ -152,12 +191,14 @@ jobs:
       # pull request (objectui#6577). `actions/cache`'s own `save-always`
       # deprecation text points at this same split.
       #
-      # ⚠️ This job declares NO `timeout-minutes`, so its ceiling is GitHub's
-      # default of 360 minutes, and `Bundle Analysis` is a required context
-      # (objectui#6245). A stalled save would therefore hold a required check
-      # open for up to six hours before reporting `cancelled`. ⛔ Adding a
-      # ceiling is NOT the fix and is not attempted here — it would change what
-      # this gate rejects. The bound below is on the bookkeeping step alone.
+      # ⚠️ This job's ceiling is `timeout-minutes: 25`, DERIVED at the job
+      # header above from this job's own run distribution (objectui#7270).
+      # Until that line existed the ceiling was GitHub's 360-minute default on
+      # a required context (objectui#6245), so a stalled save would have held
+      # that check open for up to six hours before reporting `cancelled`. The
+      # bound below is still on the bookkeeping step alone, and both numbers
+      # are needed: 25 minutes is the backstop, 5 minutes is what a healthy
+      # save is allowed.
       #
       # The restore half is deliberately left UNBOUNDED: a restore stall fails
       # BEFORE any verdict exists — a gate that did not run, which is honest —

scripts/__tests__/workflow-cache-save-bound.test.ts

@@ -40,9 +40,11 @@ import { parse as parseYaml } from 'yaml';
  *
  * Fold any split back into one `actions/cache` step and nothing goes red. The
  * cache still works, every run still passes, and the next transient upload stall
- * ejects the next green pull request — or, in the three jobs here that declare
- * no `timeout-minutes` at all, holds a required context open against GitHub's
- * 360-minute default. That is this repository's recurring "looks like
+ * ejects the next green pull request. Until objectui#7270 three of these jobs
+ * declared no `timeout-minutes` at all, so the same stall held a required
+ * context open against GitHub's 360-minute default instead; those three now
+ * carry ceilings derived from their own run distributions, and the last pin in
+ * this file is what keeps them. That is this repository's recurring "looks like
  * enforcement, isn't" class (objectui#3009, #3181, #3494).
  *
  * ## Deliberately NOT asserted
@@ -281,4 +283,68 @@ describe('every workflow cache save is bounded and non-fatal (objectui#7048)', (
       ).toBeLessThanOrEqual(max);
     }
   });
+
+  it('bounds the three jobs objectui#7270 found running under the 360-minute default', () => {
+    // Presence and bound in one assertion, because the two ways this can be
+    // undone need the same fix site and neither is visible in a run.
+    //
+    //   - Delete the key and the job silently returns to GitHub's 360-minute
+    //     default. Nothing goes red: a healthy job never approaches any
+    //     ceiling, so the only symptom is the next transient hang holding a
+    //     required context for six hours and then reporting `cancelled` — a
+    //     verdict the merge queue cannot tell from `failure` (objectui#5304,
+    //     objectui#6577 are the two times this repository has paid it).
+    //   - Raise the number and objectui#7048's fence is crossed in the other
+    //     direction: a larger ceiling buys a longer hang and still ends in
+    //     `cancelled`.
+    //
+    // Each number is DERIVED FOR ITS OWN JOB from that job's measured run
+    // distribution, and the derivation is written beside the key in the
+    // workflow — window, sample size, min/median/p95/max, and the rule.
+    // objectui#7048 fences copying `ci.yml`'s 10/15/20/30/40, so the three
+    // values here are not a shared constant and two of them being equal is a
+    // coincidence of the arithmetic. Pinning them means a future change to one
+    // has to move the derivation beside it in the same commit.
+    const derived: Record<string, number> = {
+      'lint.yml::lint': 25,
+      'performance-budget.yml::bundle-analysis': 25,
+      'changeset-release.yml::release': 40,
+    };
+
+    const missing: string[] = [];
+    const raised: string[] = [];
+
+    for (const [key, ceiling] of Object.entries(derived)) {
+      const [file, jobKey] = key.split('::');
+      const entry = allJobs.find((e) => e.file === file && e.jobKey === jobKey);
+      expect(entry, `${file} must still define a \`${jobKey}:\` job`).toBeDefined();
+
+      const declared = entry!.job['timeout-minutes'];
+      if (typeof declared !== 'number') {
+        missing.push(`${file} :: job \`${jobKey}\``);
+      } else if (declared > ceiling) {
+        raised.push(`${file} :: job \`${jobKey}\` declares ${declared}, derived ${ceiling}`);
+      }
+    }
+
+    expect(
+      missing,
+      'these jobs declare no job-level `timeout-minutes`, so their only backstop is GitHub\'s ' +
+        '360-minute default again:\n' +
+        missing.map((o) => `  - ${o}`).join('\n') +
+        '\n\nTwo of them are required contexts and one is the publish lane. Restore the key ' +
+        'TOGETHER WITH the derivation comment beside it — a ceiling with no recorded provenance ' +
+        'is the next reader\'s excuse to guess at it (objectui#7270).',
+    ).toEqual([]);
+
+    expect(
+      raised,
+      'these job ceilings are above the value derived for them:\n' +
+        raised.map((o) => `  - ${o}`).join('\n') +
+        '\n\nRaising a ceiling does not fix a hang — it buys a longer one and the gate still ' +
+        'reports `cancelled` (objectui#6577, objectui#7048). If a job genuinely got slower, ' +
+        're-derive it from a fresh distribution, rewrite the comment beside the key, and move ' +
+        'this pin in the same commit.',
+    ).toEqual([]);
+  });
 });