diff --git a/.github/workflows/_hugo.yml b/.github/workflows/_hugo.yml index fe0f2f6f3..7eab2dbbc 100644 --- a/.github/workflows/_hugo.yml +++ b/.github/workflows/_hugo.yml @@ -6,6 +6,35 @@ on: env: HUGO_ENVIRONMENT: production + # Stalled-fetch guard (actions/checkout#2441). Measured across the last 40 + # publish runs on 2026-08-20: 137 of 170 checkout steps finished in 12-19s, + # but 13 hung with ZERO fetch output until the job cap killed them. The bad + # durations cluster on 599s and 899s - which ARE the 10- and 15-minute + # timeout-minutes values, so those steps never finished slowly, they never + # finished at all. Evidence: run 32407671265 logs `git fetch` at 19:15:42 + # and the very next line is "The operation was canceled" at 19:30:41. + # It is NOT a size problem - a healthy checkout of this repo is 12s, and + # run 32414801788 hung for 10 min on an already-shallow `--depth=1` fetch. + # + # These two make git abort a transfer that has moved <1 KB/s for 30s, + # turning the silent hang into an error - and actions/checkout already + # retries a FAILED fetch 3 times (src/git-command-manager.ts wraps fetch + # in retryHelper). So a stall costs ~30s and self-heals instead of + # burning the job. A real transfer never trips it; even an awful link + # beats 1 KB/s. + # + # Passed as GIT_CONFIG_* env, NOT `git config --global`: checkout + # overrides HOME before it runs git ("Temporarily overriding HOME=..." in + # its own log), so a global config set by an earlier step is invisible to + # it. Reusable workflows do not inherit the caller's env, and neither do + # sibling workflow files - every workflow that checks out carries its own + # copy, pointing back here. + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: http.lowSpeedLimit + GIT_CONFIG_VALUE_0: '1000' + GIT_CONFIG_KEY_1: http.lowSpeedTime + GIT_CONFIG_VALUE_1: '30' + jobs: build: @@ -34,7 +63,15 @@ jobs: # right commit date in 0.02s with zero blobs fetched. # # This is a MITIGATION, not a cure - it cuts what we ask for, it - # cannot fix an upstream stall. Keep the re-run rule. + # cannot fix an upstream stall. The stall itself is handled by the + # GIT_CONFIG_* low-speed guard in this file's `env:` block, which + # makes a hung fetch fail fast so checkout retries it; the manual + # re-run rule stays as the last resort. + # + # fetch-depth: 0 is NOT reducible to a shallow fetch here - + # enableGitInfo (config/_default/hugo.toml) needs the commit + # history to resolve .Lastmod. Every OTHER job in this repo + # already checks out at the default --depth=1. fetch-depth: 0 filter: blob:none diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml index 7192ac0cd..2385e634f 100644 --- a/.github/workflows/link-check.yml +++ b/.github/workflows/link-check.yml @@ -42,6 +42,19 @@ defaults: run: shell: bash +env: + # Stalled-fetch guard (actions/checkout#2441): abort a fetch that has moved + # <1 KB/s for 30s, so checkout's built-in 3x retry can recover instead of + # the step hanging silently until the job cap kills it. Measurements and + # full rationale live in .github/workflows/_hugo.yml. GIT_CONFIG_* env + # rather than `git config --global`, because checkout overrides HOME before + # it runs git - a global config from an earlier step never reaches it. + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: http.lowSpeedLimit + GIT_CONFIG_VALUE_0: '1000' + GIT_CONFIG_KEY_1: http.lowSpeedTime + GIT_CONFIG_VALUE_1: '30' + jobs: link_check: name: Broken Internal Links diff --git a/.github/workflows/process-issue-zip.yml b/.github/workflows/process-issue-zip.yml index 36b82d753..2783021e0 100644 --- a/.github/workflows/process-issue-zip.yml +++ b/.github/workflows/process-issue-zip.yml @@ -4,6 +4,19 @@ on: issue_comment: types: [created] +env: + # Stalled-fetch guard (actions/checkout#2441): abort a fetch that has moved + # <1 KB/s for 30s, so checkout's built-in 3x retry can recover instead of + # the step hanging silently until the job cap kills it. Measurements and + # full rationale live in .github/workflows/_hugo.yml. GIT_CONFIG_* env + # rather than `git config --global`, because checkout overrides HOME before + # it runs git - a global config from an earlier step never reaches it. + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: http.lowSpeedLimit + GIT_CONFIG_VALUE_0: '1000' + GIT_CONFIG_KEY_1: http.lowSpeedTime + GIT_CONFIG_VALUE_1: '30' + jobs: handle-upload: if: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3e3544082..c1094aeca 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,6 +26,19 @@ defaults: run: shell: bash +env: + # Stalled-fetch guard (actions/checkout#2441): abort a fetch that has moved + # <1 KB/s for 30s, so checkout's built-in 3x retry can recover instead of + # the step hanging silently until the job cap kills it. Measurements and + # full rationale live in .github/workflows/_hugo.yml. GIT_CONFIG_* env + # rather than `git config --global`, because checkout overrides HOME before + # it runs git - a global config from an earlier step never reaches it. + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: http.lowSpeedLimit + GIT_CONFIG_VALUE_0: '1000' + GIT_CONFIG_KEY_1: http.lowSpeedTime + GIT_CONFIG_VALUE_1: '30' + # Sync fan-out gate (both jobs): "Sync articles" completes every 10 min # during the day, but most runs commit nothing - deploying + testing an # unchanged tree ~84x/day is pure waste. For workflow_run events, run only diff --git a/.github/workflows/sync-and-publish.yml b/.github/workflows/sync-and-publish.yml index 20affedd5..4d2538691 100644 --- a/.github/workflows/sync-and-publish.yml +++ b/.github/workflows/sync-and-publish.yml @@ -22,6 +22,19 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Stalled-fetch guard (actions/checkout#2441): abort a fetch that has moved + # <1 KB/s for 30s, so checkout's built-in 3x retry can recover instead of + # the step hanging silently until the job cap kills it. Measurements and + # full rationale live in .github/workflows/_hugo.yml. GIT_CONFIG_* env + # rather than `git config --global`, because checkout overrides HOME before + # it runs git - a global config from an earlier step never reaches it. + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: http.lowSpeedLimit + GIT_CONFIG_VALUE_0: '1000' + GIT_CONFIG_KEY_1: http.lowSpeedTime + GIT_CONFIG_VALUE_1: '30' + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: run_sync: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd35bdb9d..5800d9fdb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,6 +48,19 @@ concurrency: group: screenshots-${{ github.ref }} cancel-in-progress: true +env: + # Stalled-fetch guard (actions/checkout#2441): abort a fetch that has moved + # <1 KB/s for 30s, so checkout's built-in 3x retry can recover instead of + # the step hanging silently until the job cap kills it. Measurements and + # full rationale live in .github/workflows/_hugo.yml. GIT_CONFIG_* env + # rather than `git config --global`, because checkout overrides HOME before + # it runs git - a global config from an earlier step never reaches it. + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: http.lowSpeedLimit + GIT_CONFIG_VALUE_0: '1000' + GIT_CONFIG_KEY_1: http.lowSpeedTime + GIT_CONFIG_VALUE_1: '30' + jobs: screenshots: name: Screenshot Tests diff --git a/.okf/build/ci-gates.md b/.okf/build/ci-gates.md index 846d40f73..e27506bb7 100644 --- a/.okf/build/ci-gates.md +++ b/.okf/build/ci-gates.md @@ -64,9 +64,46 @@ above: since **2026-05-19** `actions/checkout` has been stalling silently on described upstream as silent stalls of 15-25 minutes killed by `timeout-minutes`. That is our signature exactly. +**The duration histogram decides slow-vs-stalled — you do not have to guess** +(2026-08-20, 170 checkout steps across the last 40 `publish.yml` runs): + +| Checkout duration | Steps | +|---|---| +| < 30s | 137 | +| 30-120s | 13 | +| 2-5 min | 7 | +| 5-8 min | 3 | +| > 8 min | 10 | + +A healthy checkout of this repo is **12-19s**. The bad tail is not a slow +transfer smeared across a range — it piles up on **599s and 899s**, which ARE +the 10- and 15-minute `timeout-minutes` values. A step that ends exactly at +the cap never finished; it hung. And it is **not size**: run 32414801788 hung +for 10 minutes on an already-shallow `--depth=1` fetch, and run 32407671265 +logged `git fetch` at 19:15:42 with the next line +`The operation was canceled` at 19:30:41 — 15 minutes of zero output. + Consequences for how to react: -* **Re-run; do not raise the cap.** A timeout cannot rescue a step that never +* **The stall now self-heals — a low-speed guard turns the hang into an + error** (2026-08-20). Every workflow that checks out sets, at workflow + level, `GIT_CONFIG_COUNT=2` / `http.lowSpeedLimit=1000` / + `http.lowSpeedTime=30`. Git aborts a transfer that has moved <1 KB/s for + 30s, and `actions/checkout` **already retries a FAILED fetch 3 times** + (`src/git-command-manager.ts` wraps `fetch` in `retryHelper`, 3 attempts) — + it just never got the chance, because a hang is not a failure. A stall now + costs ~30s instead of the whole job. + Two details that make or break this: it must be **`GIT_CONFIG_*` env, not + `git config --global`** (checkout logs "Temporarily overriding HOME=..." + before it runs git, so an earlier step's global config is invisible), and + **reusable workflows do not inherit the caller's env** — `_hugo.yml` needs + its own copy, as does every sibling workflow file. + Verified both directions before shipping: at an absurd 100 MB/s floor a + healthy fetch aborts with `curl 28 Operation too slow` (exit 128 — an error + `retryHelper` catches), and at the shipped 1 KB/s floor a real fetch of + master completes normally. A config you can only read is not a guard. +* **If it still stalls: re-run; do not raise the cap.** A timeout cannot + rescue a step that never progresses — raising it only makes each failure cost longer. Confirm first by pulling the job log (`gh api repos///actions/jobs//logs`) and looking for @@ -91,6 +128,11 @@ Consequences for how to react: skipped. Measured: bare blobless clone **4.7 MB in 1.1s** vs the 1.70 GiB pack, and `git log -1 -- content/blog//index.md` still returned the correct date in **0.02s with zero blobs fetched**, so GitInfo is unaffected. + **"Just shallow-clone it" is not an available lever** — every job other than + `_hugo.yml / build` already checks out at the default `--depth=1`, and they + stall anyway (run 32414801788, 10 min on `--depth=1`). The one job that + cannot go shallow is the one that must not: `enableGitInfo` in + `config/_default/hugo.toml` needs commit history to resolve `.Lastmod`. * **The deeper fix is the content weight**, not the checkout flags: 625 MB of images in git is the floor every job pays. Moving them to LFS or CDN-only would be a separate, larger decision. diff --git a/.okf/log.md b/.okf/log.md index aef816873..398c8f0c7 100644 --- a/.okf/log.md +++ b/.okf/log.md @@ -1276,3 +1276,43 @@ data, so any before/after read spanning 2026-08-13 to 08-20 compares a polluted before against a clean after. Date-bound every keyEvents query, or read the underlying event names rather than the aggregate. A fix that changes only future collection is not a fix to the series you are about to analyse. + +## 2026-08-20 - The stalled checkout is a hang, not a slow clone - make it fail fast + +Asked to "optimize the slow checkout" on a 7-minute `Asset Pipeline` job. The +optimisation was the wrong frame, and the histogram said so before any code +changed: across 170 checkout steps in the last 40 `publish.yml` runs, **137 +finished in 12-19s** and the bad tail piled up on **599s and 899s** - which +are exactly the 10- and 15-minute `timeout-minutes` values. A duration that +lands precisely on the cap did not run slowly; it never ran. Evidence: run +32407671265 logs `git fetch` at 19:15:42 and `The operation was canceled` at +19:30:41, with **zero output in between**. + +That reframes every "make the clone smaller" lever as beside the point. In +particular the obvious one was already spent: **every job except +`_hugo.yml / build` was already at the default `--depth=1`**, and run +32414801788 hung 10 minutes on a shallow fetch. The one job that needs +`fetch-depth: 0` needs it for `enableGitInfo` -> `.Lastmod`, so it cannot go +shallow either. Nothing left to shrink. + +The fix works with the grain of the tool instead of around it: `actions/checkout` +**already retries a failed fetch three times** (`src/git-command-manager.ts` +wraps `fetch` in `retryHelper`) - it just never fires, because a hang is not a +failure. So make the hang a failure. `http.lowSpeedLimit=1000` + +`http.lowSpeedTime=30` abort a transfer that stops moving, and checkout's own +retry recovers. No new action, no dependency, no per-step retry scaffolding. + +Two mechanics that would have silently no-op'd this: + +* It must be **`GIT_CONFIG_*` env, not `git config --global`.** Checkout logs + "Temporarily overriding HOME=..." before it runs git, so a global config set + by an earlier step never reaches it. The env form survives the HOME override. +* **Reusable workflows do not inherit the caller's env**, and neither do + sibling workflow files. Six workflows check out; six copies of the block. + +The habit worth keeping is the last step, not the first: a config you can only +*read back* is not a guard. Verified both directions against the real remote - +at an absurd 100 MB/s floor a healthy fetch aborts with `curl 28 Operation too +slow` (exit 128, an error `retryHelper` catches), and at the shipped 1 KB/s +floor a real fetch of master completes clean. Reading the value back would have +"passed" either way, including if the knob were inert.