From 1935279f3eb41dc9fe6a35c153998d07f1a86dc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 15:45:11 +0000 Subject: [PATCH 1/4] Cap rust-cache restore at 5 min in CI On 2026-10-26 the yarn-classic job of merge-queue head #1719 sat in its `Cache cargo` (Swatinem/rust-cache) restore for 14+ minutes. The job normally takes 2-3 minutes, and the 12 entries queued behind it were blocked until the run was cancelled by hand. Without that, the job would have run to its 40-minute timeout and then evicted #1719. A healthy restore takes under 30 s, and a cache miss only costs a cold build, so the restore is best-effort. Give every rust-cache step in ci.yml a 5-minute step timeout and continue-on-error. A hung restore now fails only that step, and the job builds without the cache instead of stalling the queue. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LZ8V19imL1WHdq6hZdoums --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d37b32a34..0149d30cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,14 @@ jobs: # shared-key by profile + runner image (not job or package-manager # version). rust-cache also hashes the toolchain, Cargo manifests # and Rust environment. Check/coverage/release keep separate caches. + # A cache restore is an optimisation, so cap it and carry on cold. + # Healthy restores take under 30 s. On 2026-10-26 one hung for 14+ + # min and held merge-queue head #1719 (and 12 entries behind it) + # until it was cancelled; uncapped it would have run to the 40-min + # job timeout. Later `Cache cargo` steps carry the same two keys. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: save-if: ${{ github.ref == 'refs/heads/main' }} @@ -150,6 +157,8 @@ jobs: - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -393,6 +402,8 @@ jobs: # Swatinem/rust-cache, main-only saves: see the first `Cache cargo` # step in this file. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-${{ matrix.os }} save-if: ${{ github.ref == 'refs/heads/main' }} @@ -569,6 +580,8 @@ jobs: # Swatinem/rust-cache, main-only saves: see the first `Cache cargo` # step in this file. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: save-if: ${{ github.ref == 'refs/heads/main' }} @@ -633,6 +646,8 @@ jobs: # The shards share one cache key and shard 1 alone writes it. # rust-cache keeps only dependencies, which every shard builds. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: save-if: ${{ github.ref == 'refs/heads/main' && matrix.shard == 1 }} @@ -1055,6 +1070,8 @@ jobs: - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: e2e-${{ matrix.os }} save-if: ${{ github.ref == 'refs/heads/main' }} @@ -1121,6 +1138,8 @@ jobs: - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: check-windows-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2281,6 +2300,8 @@ jobs: run: scripts/rustup-retry.sh toolchain install - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2326,6 +2347,8 @@ jobs: run: scripts/rustup-retry.sh toolchain install - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-${{ matrix.os }} save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2561,6 +2584,8 @@ jobs: docker pull rust:1.56-slim - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2669,6 +2694,8 @@ jobs: - name: Cache cargo if: steps.gate.outputs.run == 'true' uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} From 05d96a93deeedaa465249fb37922dca9e46393c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 15:47:43 +0000 Subject: [PATCH 2/4] Allow step-level continue-on-error in CI contract The fail-fast contract test rejected any `continue-on-error` in ci.yml. Its concern is a job-level key, which would let ci-ok pass despite a failed job. The capped `Cache cargo` steps set it at step level, which leaves the job's conclusion to its other steps, so narrow the check to job-level keys (four-space indent). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LZ8V19imL1WHdq6hZdoums --- scripts/merge-queue-fail-fast.py | 2 +- scripts/tests/test_merge_queue_fail_fast.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/merge-queue-fail-fast.py b/scripts/merge-queue-fail-fast.py index 452208d22..b3d6be1cf 100644 --- a/scripts/merge-queue-fail-fast.py +++ b/scripts/merge-queue-fail-fast.py @@ -7,7 +7,7 @@ ~45 minutes the Gradle e2e legs take, burning macOS and Windows runners on a run that could no longer pass. -No CI job sets `continue-on-error`, so one failed or timed-out job means +No CI job sets a job-level `continue-on-error`, so one failed or timed-out job means `ci-ok` will fail. Cancelling the run then changes only *when* it fails: `ci-ok` is `if: always()`, so it still runs on the cancelled run, sees the failed and cancelled `needs`, and reports failure within seconds, and the diff --git a/scripts/tests/test_merge_queue_fail_fast.py b/scripts/tests/test_merge_queue_fail_fast.py index 7efefb9f3..3633a3b83 100644 --- a/scripts/tests/test_merge_queue_fail_fast.py +++ b/scripts/tests/test_merge_queue_fail_fast.py @@ -115,7 +115,9 @@ class CiWorkflowContract(unittest.TestCase): def test_no_job_tolerates_failure(self): # A job-level continue-on-error would let ci-ok pass despite a failed # job, and this script would then cancel a run that could still land. - self.assertIsNone(re.search(r"^\s*continue-on-error:", self.text, re.M)) + # Step-level ones (8+ spaces, e.g. the capped `Cache cargo` restores) + # leave the job's conclusion to its other steps, so they are fine. + self.assertIsNone(re.search(r"^ continue-on-error:", self.text, re.M)) def test_gate_runs_on_a_cancelled_run(self): gate = self.text[self.text.index("\n ci-ok:"):] From d2398dcf9f31376bbaba37c24f6f8accafed186e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 17:15:23 +0000 Subject: [PATCH 3/4] Restrict continue-on-error to capped restores Review follow-ups: - The contract test allowed continue-on-error on any step, so a test or build step tolerating failure would slip past it. Allow it only on rust-cache / actions/cache/restore steps that also carry a timeout, and require every such restore in ci.yml to be capped. - Cap the macOS vexctl actions/cache/restore the same way. If that restore times out on main, skip the save step, which would otherwise run with an empty key and fail the job. - Correct the comment: the hang was in yarn-classic-matrix, and an uncapped restore runs to that job's own timeout. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LZ8V19imL1WHdq6hZdoums --- .github/workflows/ci.yml | 14 ++++++++++---- scripts/tests/test_merge_queue_fail_fast.py | 19 +++++++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0149d30cd..83aaf9594 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,10 +118,12 @@ jobs: # version). rust-cache also hashes the toolchain, Cargo manifests # and Rust environment. Check/coverage/release keep separate caches. # A cache restore is an optimisation, so cap it and carry on cold. - # Healthy restores take under 30 s. On 2026-10-26 one hung for 14+ - # min and held merge-queue head #1719 (and 12 entries behind it) - # until it was cancelled; uncapped it would have run to the 40-min - # job timeout. Later `Cache cargo` steps carry the same two keys. + # Healthy restores take under 30 s on every runner OS. On 2026-10-26 + # one hung for 14+ min in yarn-classic-matrix and held merge-queue + # head #1719 (and 12 entries behind it) until it was cancelled; + # uncapped it runs to the job's own timeout. Every cache restore in + # this file carries the same two keys (enforced by + # scripts/tests/test_merge_queue_fail_fast.py). uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 timeout-minutes: 5 continue-on-error: true @@ -442,6 +444,9 @@ jobs: id: vexctl-cache if: needs.release-readiness.outputs.reuse != 'true' && runner.os == 'macOS' uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + # Capped like `Cache cargo`: a miss only means compiling vexctl. + timeout-minutes: 5 + continue-on-error: true with: path: ${{ runner.temp }}/vexctl-bin key: vexctl-${{ env.VEXCTL_VERSION }}-go${{ steps.go.outputs.go-version }}-${{ runner.os }}-${{ runner.arch }} @@ -515,6 +520,7 @@ jobs: && runner.os == 'macOS' && github.ref == 'refs/heads/main' && steps.vexctl-cache.outputs.cache-hit != 'true' + && steps.vexctl-cache.outcome == 'success' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/vexctl-bin diff --git a/scripts/tests/test_merge_queue_fail_fast.py b/scripts/tests/test_merge_queue_fail_fast.py index 3633a3b83..1b7a6f64b 100644 --- a/scripts/tests/test_merge_queue_fail_fast.py +++ b/scripts/tests/test_merge_queue_fail_fast.py @@ -115,9 +115,24 @@ class CiWorkflowContract(unittest.TestCase): def test_no_job_tolerates_failure(self): # A job-level continue-on-error would let ci-ok pass despite a failed # job, and this script would then cancel a run that could still land. - # Step-level ones (8+ spaces, e.g. the capped `Cache cargo` restores) - # leave the job's conclusion to its other steps, so they are fine. + # A step-level one hides that step's failure the same way, so only + # the capped cache restores may carry it: a miss there just means + # building cold. self.assertIsNone(re.search(r"^ continue-on-error:", self.text, re.M)) + steps = re.split(r"(?m)^ - ", self.text) + tolerant = [s for s in steps if re.search(r"(?m)^\s*continue-on-error:", s)] + self.assertTrue(tolerant) + for step in tolerant: + self.assertRegex(step, r"(?m)^ uses: (Swatinem/rust-cache|actions/cache/restore)@", step[:80]) + self.assertRegex(step, r"(?m)^ timeout-minutes: \d+$", step[:80]) + self.assertRegex(step, r"(?m)^ continue-on-error: true$", step[:80]) + + def test_every_cache_restore_is_capped(self): + # An uncapped restore that hangs holds the queue head until the job + # times out (#1730). + for step in re.split(r"(?m)^ - ", self.text): + if re.search(r"(?m)^ uses: (Swatinem/rust-cache|actions/cache/restore)@", step): + self.assertRegex(step, r"(?m)^ timeout-minutes: \d+$", step[:80]) def test_gate_runs_on_a_cancelled_run(self): gate = self.text[self.text.index("\n ci-ok:"):] From c33588fc1e3f1a0cb9ed15e35a0ddc14f8cbf9b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 17:51:14 +0000 Subject: [PATCH 4/4] Reword hosted-e2e continue-on-error invariant The invariant said the job has no continue-on-error, but its capped Cache cargo restore now carries one. Say what still holds: none on the job or its e2e steps. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LZ8V19imL1WHdq6hZdoums --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83aaf9594..78ed5da7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2629,7 +2629,8 @@ jobs: # The kill switch gates the STEPS, never the job. # * NO `needs:` — an upstream failure would skip this job, same wedge. # * NO matrix and NO rename — the check name must stay `hosted-e2e`. - # * NO `continue-on-error` — a bypass must be visible, not invisible. + # * NO `continue-on-error` on the job or its e2e steps — a bypass must be + # visible, not invisible. Only the capped `Cache cargo` restore has it. # The job ALWAYS runs and ALWAYS reaches success or failure. # # ESCAPE HATCH — when production is down and this is blocking merges: