From 10b62885fa144cb364004d615f716a1040381a62 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:07:08 +1000 Subject: [PATCH 1/3] CI: add timeout and retry to apt-get install steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bare `sudo apt-get install -y` with no `timeout-minutes` and no retry let a stalled Ubuntu mirror hang CI for hours — two 358-minute hangs on 2026-08-18 across two edition repos ten minutes apart, and four more hangs on 2026-08-19 each cleared only by cancel-and-rerun. Every apt-installing step now carries a step-level `timeout-minutes` and runs up to three attempts with a per-attempt `timeout`, so a stalled fetch fails fast and a transient mirror outage is ridden out rather than burning the job. Package lists are unchanged. Tracked in QuantEcon/project-translation#43. --- .github/workflows/ci.yml | 20 +++++++++----------- .github/workflows/publish.yml | 20 +++++++++----------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdbd367..f7e5f43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,18 +26,16 @@ jobs: # and does not upgrade. pip install "jax==0.11.0" - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0c7667a..0037c98 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,18 +31,16 @@ jobs: # and does not upgrade. pip install "jax==0.11.0" - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list From 798d6254738dcc21c027abf7990e4c638be9a6a7 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:40:37 +1000 Subject: [PATCH 2/3] CI: harden the cache.yml graphviz step too; two attempts with wider per-attempt timeouts (4m/14m) Addresses Copilot review on QuantEcon/lecture-python.zh-cn#263: the cache.yml graphviz step was missed by a name match, and three 3m/9m attempts could never let a slow-but-progressing install finish. Now two attempts at 4m (graphviz) / 14m (texlive) inside the unchanged 10/30-minute step budgets. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/publish.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7e5f43..44c9bac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0037c98..bc8d6b0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -36,9 +36,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions From 61a3b3994589b3df8aad44ef78c31b095e6f4cd5 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:43:55 +1000 Subject: [PATCH 3/3] CI: run timeout inside sudo, per apt command, so the kill reaches apt-get Addresses Copilot review on QuantEcon/lecture-intro.zh-cn#301: with timeout wrapping sudo, a timed-out attempt terminated sudo and the bash -c shell but could orphan apt-get holding the dpkg lock, making the retry block instead of retry. timeout is now the direct parent of each apt-get (inside sudo, -k 30s escalation): update 1m/2m, install 3m/12m for graphviz/texlive. Step budgets unchanged. --- .github/workflows/ci.yml | 5 +++-- .github/workflows/publish.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44c9bac..a8ff22a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,11 +28,12 @@ jobs: - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bc8d6b0..f05f137 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,11 +33,12 @@ jobs: - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done