From ccd810bb73f767386f094b50f266cacc1cb57beb Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Wed, 19 Aug 2026 19:07:30 +0000
Subject: [PATCH 1/9] CI: install ccache from the staged bundle and bound every
 apt call

ccache-setup ran a raw apt-get update, so jobs whose ghcr .deb bundle had
already installed cleanly still reached the mirror. On 2026-08-19 that step
stalled on archive.ubuntu.com for 9-43 min and took out 11 jobs. The .deb is
already in /var/cache/apt/archives - install-apt-deps stages the whole bundle
- so install it with --no-download first and only fall back to the mirror.
ccache was missing from the 22.04-minimal and linuxkm bundles; add it.

Neither the fallback in install-apt-deps nor any of the raw apt sites had a
timeout, so a wedged mirror hung instead of failing and the retry loops never
fired. ci-deps-image already solved this for itself (Acquire timeouts plus
`timeout`); apply the same to the consumers, and to the linuxkm producer job
that was still missing it. Also recover from a dpkg interrupted by a kill,
and bound the bundle pull.
---
 .github/actions/ccache-setup/action.yml       | 34 +++++++++++++--
 .github/actions/install-apt-deps/action.yml   | 18 ++++++--
 .../ci-deps/packages-ubuntu-22.04-minimal.txt |  1 +
 .github/scripts/zephyr-4.x/zephyr-test.sh     |  8 +++-
 .github/workflows/ci-deps-image.yml           | 14 ++++---
 .github/workflows/cross-library.yml           | 42 +++++++++++++++----
 .github/workflows/falcon-interop.yml          | 42 +++++++++++++++++--
 .github/workflows/sbom.yml                    | 21 +++++++++-
 8 files changed, 154 insertions(+), 26 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index f68734209fa..7e11c1f1159 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -42,9 +42,37 @@ runs:
         if command -v ccache >/dev/null 2>&1; then
           echo "ccache already installed: $(ccache --version | head -1)"
         elif [ "${{ runner.os }}" = "Linux" ]; then
-          sudo apt-get update -q
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            --no-install-recommends ccache
+          export DEBIAN_FRONTEND=noninteractive
+          # install-apt-deps stages the WHOLE ghcr bundle into
+          # /var/cache/apt/archives, and ccache is in the -minimal/-full
+          # lists, so in a job that ran it first the .deb is already on disk.
+          # Take it offline (--no-download): no apt-get update, nothing to
+          # stall on. Every other path here reaches the mirror, which is what
+          # used to hang these jobs for 10-40 min after the bundle had
+          # already installed cleanly.
+          if sudo apt-get install -y --no-install-recommends \
+               --no-download ccache; then
+            echo "ccache installed offline from the staged .deb bundle"
+          else
+            # Same defence in depth as install-apt-deps: Acquire timeouts drop
+            # a stalled connection, `timeout` hard-kills a wedged apt-get, and
+            # only then does the retry loop get a non-zero exit to act on.
+            APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                      -o Acquire::https::Timeout=30)
+            ok=""
+            for i in 1 2 3; do
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
+                   --no-install-recommends ccache; then
+                ok=1
+                break
+              fi
+              echo "::warning::ccache apt install failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
+          fi
         elif [ "${{ runner.os }}" = "macOS" ]; then
           brew install ccache
         else
diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index b8cb68b4e4a..c279dca0e75 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -50,7 +50,7 @@ runs:
         # PRs read the public upstream image too rather than a nonexistent
         # ghcr.io/<fork>/wolfssl-ci-debs.
         IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}"
-        if ! docker pull -q "$IMG" >/dev/null 2>&1; then
+        if ! timeout -k 10 300 docker pull -q "$IMG" >/dev/null 2>&1; then
           echo "::notice::ghcr bundle $IMG unavailable; using apt"
           exit 0
         fi
@@ -85,9 +85,21 @@ runs:
           NO_REC="--no-install-recommends"
         fi
 
+        # A wedged mirror hangs apt rather than failing it, so the retry loop
+        # below never fired and the job burned its whole budget instead.
+        # Defend in depth: apt drops a stalled connection after 30s and retries
+        # it (Acquire timeouts), `timeout` hard-kills an apt-get that wedged
+        # anyway, then the loop re-runs - re-reading apt-mirrors.txt, so a
+        # retry can land on a different mirror.
+        APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                  -o Acquire::https::Timeout=30)
+
         for i in $(seq 1 $RETRIES); do
-          if sudo apt-get update -q && \
-             sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
+          # A previous attempt killed mid-unpack leaves dpkg needing this.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+               $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
           if [ "$i" -eq "$RETRIES" ]; then
diff --git a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt
index c32e3ccb9cf..6fdbf8cfbba 100644
--- a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt
+++ b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt
@@ -4,6 +4,7 @@
 autoconf
 automake
 build-essential
+ccache
 crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf
diff --git a/.github/scripts/zephyr-4.x/zephyr-test.sh b/.github/scripts/zephyr-4.x/zephyr-test.sh
index c225277cacc..66a2930d64c 100755
--- a/.github/scripts/zephyr-4.x/zephyr-test.sh
+++ b/.github/scripts/zephyr-4.x/zephyr-test.sh
@@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..."
 west zephyr-export
 
 echo "==> [container] Installing host packages (newlib, python3-venv)..."
-sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true
+# `|| true` keeps this best-effort, but without a timeout a wedged mirror
+# stalls here silently until the job budget runs out.
+APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \
+  && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \
+       python3-venv libnewlib-dev >/dev/null 2>&1 \
+  || echo "==> [container] host package install skipped (apt unavailable)"
 python3 -m venv .venv
 source .venv/bin/activate
 pip3 install west
diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml
index d91ac4fb877..2fa1b1130b1 100644
--- a/.github/workflows/ci-deps-image.yml
+++ b/.github/workflows/ci-deps-image.yml
@@ -183,21 +183,25 @@ jobs:
           set -euo pipefail
           K="${{ steps.check.outputs.kernel }}"
           # linuxkm.yml installs only the headers; the membrowse linuxkm targets
-          # also need the build toolchain. Bundle the union - each consumer
-          # installs its own subset offline.
-          PKGS=(build-essential autoconf automake libtool "linux-headers-$K")
+          # also need the build toolchain, and ccache-setup installs ccache
+          # offline from whatever this bundle staged. Bundle the union - each
+          # consumer installs its own subset offline.
+          PKGS=(build-essential autoconf automake libtool ccache
+                "linux-headers-$K")
           echo "Packages: ${PKGS[*]}"
           export DEBIAN_FRONTEND=noninteractive
           rm -rf debs && mkdir -p debs
           sudo apt-get clean
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
           retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
-          retry sudo apt-get update -q
+          retry sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only
           # on success, so a transient mirror error keeps the last good bundle
           # rather than publishing a partial one - which the kernel-label skip
           # would then pin in place until the kernel next changes (~monthly).
-          retry sudo apt-get install -y --download-only "${PKGS[@]}"
+          retry sudo timeout -k 10 600 apt-get "${APT_OPTS[@]}" install -y \
+            --download-only "${PKGS[@]}"
           sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
           test -n "$(ls debs/*.deb 2>/dev/null)"  # headers are never preinstalled
diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml
index af25dd52cf1..3d0463dfc78 100644
--- a/.github/workflows/cross-library.yml
+++ b/.github/workflows/cross-library.yml
@@ -64,10 +64,23 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          for i in 1 2 3; do
+            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/3)"
+            sleep $((5 * i))
+          done
 
       # Building only needs the commit under test, not history. The break check
       # that needs history runs in the compile job, not here.
@@ -129,10 +142,23 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          for i in 1 2 3; do
+            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/3)"
+            sleep $((5 * i))
+          done
 
       # This job does not build wolfSSL, but the latest leg still checks out
       # wolfSSL history because check-break.sh scans commit messages here. The
diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml
index ad6bc8f0137..a6b1e6ffe46 100644
--- a/.github/workflows/falcon-interop.yml
+++ b/.github/workflows/falcon-interop.yml
@@ -91,8 +91,25 @@ jobs:
     steps:
       - name: Install build tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y ninja-build
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2 3; do
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            echo "::error::apt-get failed after 3 attempts"
+            return 1
+          }
+          apt_retry ninja-build
 
       # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which
       # would delete an untracked oqs-install/ placed in the workspace by the
@@ -161,8 +178,25 @@ jobs:
     steps:
       - name: Install build tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y autoconf automake libtool
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2 3; do
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            echo "::error::apt-get failed after 3 attempts"
+            return 1
+          }
+          apt_retry autoconf automake libtool
 
       - name: Checkout wolfSSL
         uses: actions/checkout@v5
diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml
index 4e2b3fcd950..98adbe5ffad 100644
--- a/.github/workflows/sbom.yml
+++ b/.github/workflows/sbom.yml
@@ -822,7 +822,24 @@ jobs:
 
       - name: Install build deps + SBOM validators
         run: |
-          sudo apt-get update
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2 3; do
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            echo "::error::apt-get failed after 3 attempts"
+            return 1
+          }
           # bison + autotools-dev are required by strace's ./bootstrap.
           # gcc-multilib + g++-multilib give strace's --enable-mpers=check
           # the 32-bit/x32 compilers it needs - without them mpers is
@@ -830,7 +847,7 @@ jobs:
           # syscalls, diverging from what bomsh's devcontainer produces.
           # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3
           # stage.
-          sudo apt-get install -y build-essential autoconf automake libtool \
+          apt_retry build-essential autoconf automake libtool \
             bison autotools-dev gcc-multilib g++-multilib \
             python3 python3-pip git
           python3 -m pip install --user --upgrade pip

From 16d9378f808cab1ff549c526f203a7053d7b3db8 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Thu, 20 Aug 2026 09:21:33 +0000
Subject: [PATCH 2/9] CI: keep the apt retry loops POSIX and inside the job
 budget

cross-library's containers are bare images with no bash, so Actions runs the
step under sh; the bash array APT_OPTS=(...) was a syntax error and took out
all six cross-library builds. Use a plain word-splitting variable.

The retry budget also has to fit the caller's timeout-minutes - install-apt-deps
is used by jobs with 4 minutes - or the last attempt is cut off before it can
report. Two attempts at 60s/300s instead of three at 120s/900s. The Acquire
timeouts are what actually detect a wedge, so the outer bound only backstops
apt wedging outside its own I/O loop, and apt resumes from archives/partial/.

Also run dpkg --configure -a before the offline ccache probe, not only in the
fallback loop: an interrupted dpkg would otherwise push it to the mirror.
---
 .github/actions/ccache-setup/action.yml     | 11 ++---
 .github/actions/install-apt-deps/action.yml | 21 ++++++---
 .github/workflows/ci-deps-image.yml         |  8 ++--
 .github/workflows/cross-library.yml         | 50 ++++++++++++---------
 .github/workflows/falcon-interop.yml        | 26 ++++++-----
 .github/workflows/sbom.yml                  | 13 +++---
 6 files changed, 76 insertions(+), 53 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index 7e11c1f1159..31cea8d4686 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -50,6 +50,7 @@ runs:
           # stall on. Every other path here reaches the mirror, which is what
           # used to hang these jobs for 10-40 min after the bundle had
           # already installed cleanly.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
           if sudo apt-get install -y --no-install-recommends \
                --no-download ccache; then
             echo "ccache installed offline from the staged .deb bundle"
@@ -60,16 +61,16 @@ runs:
             APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                       -o Acquire::https::Timeout=30)
             ok=""
-            for i in 1 2 3; do
+            for i in 1 2; do
               sudo dpkg --configure -a >/dev/null 2>&1 || true
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 180 apt-get "${APT_OPTS[@]}" install -y \
                    --no-install-recommends ccache; then
                 ok=1
                 break
               fi
-              echo "::warning::ccache apt install failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::ccache apt install failed (attempt $i/2)"
+              sleep 5
             done
             [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
           fi
diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index c279dca0e75..f61e81febf3 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -5,9 +5,12 @@ inputs:
     description: 'Space-separated list of apt packages to install'
     required: true
   retries:
-    description: 'Number of retry attempts'
+    description: >
+      Number of retry attempts. Keep attempts x (60s update + 300s install)
+      within the calling job's timeout-minutes, or the last attempt is cut
+      off before it can report.
     required: false
-    default: '3'
+    default: '2'
   retry-delay:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
@@ -88,17 +91,21 @@ runs:
         # A wedged mirror hangs apt rather than failing it, so the retry loop
         # below never fired and the job burned its whole budget instead.
         # Defend in depth: apt drops a stalled connection after 30s and retries
-        # it (Acquire timeouts), `timeout` hard-kills an apt-get that wedged
-        # anyway, then the loop re-runs - re-reading apt-mirrors.txt, so a
-        # retry can land on a different mirror.
+        # it (Acquire timeouts - this is what actually detects a wedge, in
+        # ~90s), `timeout` hard-kills an apt-get that wedged outside its own
+        # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a
+        # retry can land on a different mirror. The timeouts stay tight so
+        # every attempt fits the caller's timeout-minutes (as low as 4 min);
+        # apt resumes from archives/partial/, so a killed transfer is not
+        # restarted from scratch.
         APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                   -o Acquire::https::Timeout=30)
 
         for i in $(seq 1 $RETRIES); do
           # A previous attempt killed mid-unpack leaves dpkg needing this.
           sudo dpkg --configure -a >/dev/null 2>&1 || true
-          if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-             sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+          if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
                $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml
index 2fa1b1130b1..76f1a65489d 100644
--- a/.github/workflows/ci-deps-image.yml
+++ b/.github/workflows/ci-deps-image.yml
@@ -193,14 +193,16 @@ jobs:
           rm -rf debs && mkdir -p debs
           sudo apt-get clean
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
-          retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
-          retry sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q
+          # 2 attempts, not 5: this job's timeout-minutes is 20, and an
+          # attempt cut off mid-flight reports nothing.
+          retry() { local i; for i in 1 2; do "$@" && return 0; sleep 5; done; "$@"; }
+          retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only
           # on success, so a transient mirror error keeps the last good bundle
           # rather than publishing a partial one - which the kernel-label skip
           # would then pin in place until the kernel next changes (~monthly).
-          retry sudo timeout -k 10 600 apt-get "${APT_OPTS[@]}" install -y \
+          retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
             --download-only "${PKGS[@]}"
           sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml
index 3d0463dfc78..34bb4da8c41 100644
--- a/.github/workflows/cross-library.yml
+++ b/.github/workflows/cross-library.yml
@@ -64,22 +64,27 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
-          # drop a stalled connection, `timeout` hard-kills apt-get if it
-          # wedges anyway, and the loop then retries against a fresh mirror.
-          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
-                    -o Acquire::https::Timeout=30)
-          for i in 1 2 3; do
-            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
                  --no-install-recommends \
                  build-essential autoconf automake libtool pkg-config \
                  git ca-certificates ${{ inputs.apt_packages }}; then
               break
             fi
-            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
-            echo "::warning::apt-get failed (attempt $i/3)"
-            sleep $((5 * i))
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
           done
 
       # Building only needs the commit under test, not history. The break check
@@ -142,22 +147,27 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
-          # drop a stalled connection, `timeout` hard-kills apt-get if it
-          # wedges anyway, and the loop then retries against a fresh mirror.
-          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
-                    -o Acquire::https::Timeout=30)
-          for i in 1 2 3; do
-            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
                  --no-install-recommends \
                  build-essential autoconf automake libtool pkg-config \
                  git ca-certificates ${{ inputs.apt_packages }}; then
               break
             fi
-            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
-            echo "::warning::apt-get failed (attempt $i/3)"
-            sleep $((5 * i))
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
           done
 
       # This job does not build wolfSSL, but the latest leg still checks out
diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml
index a6b1e6ffe46..1cf613e2ff4 100644
--- a/.github/workflows/falcon-interop.yml
+++ b/.github/workflows/falcon-interop.yml
@@ -94,19 +94,20 @@ jobs:
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
           # drop a stalled connection, `timeout` hard-kills apt-get if it
           # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                     -o Acquire::https::Timeout=30)
           apt_retry() {
             local i
-            for i in 1 2 3; do
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
               fi
-              echo "::warning::apt-get failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
             done
-            echo "::error::apt-get failed after 3 attempts"
+            echo "::error::apt-get failed after 2 attempts"
             return 1
           }
           apt_retry ninja-build
@@ -181,19 +182,20 @@ jobs:
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
           # drop a stalled connection, `timeout` hard-kills apt-get if it
           # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                     -o Acquire::https::Timeout=30)
           apt_retry() {
             local i
-            for i in 1 2 3; do
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
               fi
-              echo "::warning::apt-get failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
             done
-            echo "::error::apt-get failed after 3 attempts"
+            echo "::error::apt-get failed after 2 attempts"
             return 1
           }
           apt_retry autoconf automake libtool
diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml
index 98adbe5ffad..8a6389a6c91 100644
--- a/.github/workflows/sbom.yml
+++ b/.github/workflows/sbom.yml
@@ -825,19 +825,20 @@ jobs:
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
           # drop a stalled connection, `timeout` hard-kills apt-get if it
           # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                     -o Acquire::https::Timeout=30)
           apt_retry() {
             local i
-            for i in 1 2 3; do
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
               fi
-              echo "::warning::apt-get failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
             done
-            echo "::error::apt-get failed after 3 attempts"
+            echo "::error::apt-get failed after 2 attempts"
             return 1
           }
           # bison + autotools-dev are required by strace's ./bootstrap.

From e1a2dd18628528226de0ee435b1b2e11aaf9d1d3 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Fri, 21 Aug 2026 02:48:32 +0000
Subject: [PATCH 3/9] Address review: budget the apt loop and keep
 DEBIAN_FRONTEND across sudo

The per-attempt deadlines did not fit callers with a four minute
timeout-minutes, so a wedged mirror cancelled the job before the loop
could report it. Spend a configurable budget across the attempts instead
and set one that fits on the short jobs.

sudo resets the environment, so an exported DEBIAN_FRONTEND never
reached apt-get and a debconf prompt could block until the timeout.
---
 .github/actions/ccache-setup/action.yml     | 12 +++--
 .github/actions/install-apt-deps/action.yml | 57 +++++++++++++++------
 .github/workflows/check-source-text.yml     |  3 ++
 .github/workflows/cyrus-sasl.yml            |  3 ++
 .github/workflows/ipmitool.yml              |  3 ++
 .github/workflows/jwt-cpp.yml               |  3 ++
 .github/workflows/linuxkm.yml               |  3 ++
 .github/workflows/mosquitto.yml             |  3 ++
 .github/workflows/pam-ipmi.yml              |  3 ++
 .github/workflows/rng-tools.yml             |  3 ++
 10 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index 31cea8d4686..a39b93567d3 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -51,8 +51,8 @@ runs:
           # used to hang these jobs for 10-40 min after the bundle had
           # already installed cleanly.
           sudo dpkg --configure -a >/dev/null 2>&1 || true
-          if sudo apt-get install -y --no-install-recommends \
-               --no-download ccache; then
+          if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+               --no-install-recommends --no-download ccache; then
             echo "ccache installed offline from the staged .deb bundle"
           else
             # Same defence in depth as install-apt-deps: Acquire timeouts drop
@@ -63,9 +63,11 @@ runs:
             ok=""
             for i in 1 2; do
               sudo dpkg --configure -a >/dev/null 2>&1 || true
-              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 180 apt-get "${APT_OPTS[@]}" install -y \
-                   --no-install-recommends ccache; then
+              if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \
+                     apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 180 \
+                     apt-get "${APT_OPTS[@]}" install -y \
+                     --no-install-recommends ccache; then
                 ok=1
                 break
               fi
diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index f61e81febf3..03417164832 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -5,12 +5,24 @@ inputs:
     description: 'Space-separated list of apt packages to install'
     required: true
   retries:
-    description: >
-      Number of retry attempts. Keep attempts x (60s update + 300s install)
-      within the calling job's timeout-minutes, or the last attempt is cut
-      off before it can report.
+    description: 'Number of retry attempts.'
     required: false
     default: '2'
+  budget-seconds:
+    description: >
+      Wall-clock the whole retry loop may spend. Split across the attempts,
+      so a wedged mirror is reported by this action instead of the job being
+      cancelled around it. Must fit inside the caller's timeout-minutes
+      alongside pull-timeout; the default suits jobs of 15 minutes or more,
+      short jobs have to lower it.
+    required: false
+    default: '600'
+  pull-timeout:
+    description: >
+      Deadline for the ghcr bundle pull. Counts against the same
+      timeout-minutes as budget-seconds.
+    required: false
+    default: '300'
   retry-delay:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
@@ -53,7 +65,7 @@ runs:
         # PRs read the public upstream image too rather than a nonexistent
         # ghcr.io/<fork>/wolfssl-ci-debs.
         IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}"
-        if ! timeout -k 10 300 docker pull -q "$IMG" >/dev/null 2>&1; then
+        if ! timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" >/dev/null 2>&1; then
           echo "::notice::ghcr bundle $IMG unavailable; using apt"
           exit 0
         fi
@@ -80,9 +92,9 @@ runs:
       if: steps.ghcr.outputs.satisfied != 'true'
       shell: bash
       run: |
-        export DEBIAN_FRONTEND=noninteractive
         RETRIES=${{ inputs.retries }}
         DELAY=${{ inputs.retry-delay }}
+        BUDGET=${{ inputs.budget-seconds }}
         NO_REC=""
         if [ "${{ inputs.no-install-recommends }}" = "true" ]; then
           NO_REC="--no-install-recommends"
@@ -94,23 +106,38 @@ runs:
         # it (Acquire timeouts - this is what actually detects a wedge, in
         # ~90s), `timeout` hard-kills an apt-get that wedged outside its own
         # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a
-        # retry can land on a different mirror. The timeouts stay tight so
-        # every attempt fits the caller's timeout-minutes (as low as 4 min);
-        # apt resumes from archives/partial/, so a killed transfer is not
-        # restarted from scratch.
+        # retry can land on a different mirror. apt resumes from
+        # archives/partial/, so a killed transfer is not restarted from
+        # scratch.
         APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                   -o Acquire::https::Timeout=30)
 
+        # Spend budget-seconds over the attempts rather than a fixed
+        # per-attempt deadline: a caller with a short timeout-minutes would
+        # otherwise be cancelled mid-attempt, before the loop could report
+        # the failure. update gets a sixth of an attempt, install the rest,
+        # with floors so a small budget still leaves apt time to work.
+        PER=$((BUDGET / RETRIES))
+        UPD=$((PER / 6))
+        [ "$UPD" -ge 20 ] || UPD=20
+        INS=$((PER - UPD))
+        [ "$INS" -ge 40 ] || INS=40
+        DEADLINE=$(($(date +%s) + BUDGET))
+
+        # sudo resets the environment, so DEBIAN_FRONTEND has to ride along
+        # on each privileged command rather than being exported once.
         for i in $(seq 1 $RETRIES); do
           # A previous attempt killed mid-unpack leaves dpkg needing this.
           sudo dpkg --configure -a >/dev/null 2>&1 || true
-          if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
-             sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
-               $NO_REC ${{ inputs.packages }}; then
+          if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \
+                 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \
+                 apt-get "${APT_OPTS[@]}" install -y \
+                 $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
-          if [ "$i" -eq "$RETRIES" ]; then
-            echo "::error::apt-get failed after $RETRIES attempts"
+          if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then
+            echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s"
             exit 1
           fi
           echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."
diff --git a/.github/workflows/check-source-text.yml b/.github/workflows/check-source-text.yml
index cf3efb95da2..7f9524563b8 100644
--- a/.github/workflows/check-source-text.yml
+++ b/.github/workflows/check-source-text.yml
@@ -45,6 +45,9 @@ jobs:
       - name: Install shellcheck
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: shellcheck python3-yaml
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/cyrus-sasl.yml b/.github/workflows/cyrus-sasl.yml
index 5c1ad746e0a..c4a5bc3c97e 100644
--- a/.github/workflows/cyrus-sasl.yml
+++ b/.github/workflows/cyrus-sasl.yml
@@ -66,6 +66,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: krb5-kdc krb5-otp libkrb5-dev libsocket-wrapper libnss-wrapper krb5-admin-server libdb5.3-dev
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/ipmitool.yml b/.github/workflows/ipmitool.yml
index b5df5fbe124..6db463a4e63 100644
--- a/.github/workflows/ipmitool.yml
+++ b/.github/workflows/ipmitool.yml
@@ -63,6 +63,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libreadline-dev
           ghcr-debs-tag: ubuntu-24.04-full
       - name: Download lib
diff --git a/.github/workflows/jwt-cpp.yml b/.github/workflows/jwt-cpp.yml
index e3263f3f996..df9a56247f0 100644
--- a/.github/workflows/jwt-cpp.yml
+++ b/.github/workflows/jwt-cpp.yml
@@ -67,6 +67,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libgtest-dev
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/linuxkm.yml b/.github/workflows/linuxkm.yml
index 8c404a4b0c8..a7194f2065d 100644
--- a/.github/workflows/linuxkm.yml
+++ b/.github/workflows/linuxkm.yml
@@ -36,6 +36,9 @@ jobs:
       - name: Install linux-headers
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: linux-headers-$(uname -r)
           ghcr-debs-tag: ubuntu-24.04-linuxkm
 
diff --git a/.github/workflows/mosquitto.yml b/.github/workflows/mosquitto.yml
index 49b515d274b..dbea1074939 100644
--- a/.github/workflows/mosquitto.yml
+++ b/.github/workflows/mosquitto.yml
@@ -78,6 +78,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: build-essential libev-dev libssl-dev automake python3-docutils libcunit1 libcunit1-doc libcunit1-dev pkg-config make python3-psutil
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/pam-ipmi.yml b/.github/workflows/pam-ipmi.yml
index de907d05aff..faa19001a5a 100644
--- a/.github/workflows/pam-ipmi.yml
+++ b/.github/workflows/pam-ipmi.yml
@@ -64,6 +64,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libpam-dev ninja-build meson
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/rng-tools.yml b/.github/workflows/rng-tools.yml
index 1027ab80fc6..f14877d0124 100644
--- a/.github/workflows/rng-tools.yml
+++ b/.github/workflows/rng-tools.yml
@@ -65,6 +65,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libcurl4-openssl-dev libjansson-dev libp11-dev librtlsdr-dev libcap-dev
           ghcr-debs-tag: ubuntu-24.04-full
 

From 3c8cdaecea77a84f770f94986a42c69b8ae26c74 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Mon, 24 Aug 2026 12:11:46 +0000
Subject: [PATCH 4/9] CI: recover an interrupted dpkg in the open-coded apt
 retry loops

install-apt-deps and ccache-setup run dpkg --configure -a before each
attempt; the five loops open-coded in workflow files did not. timeout
signals the whole process group, so a kill during unpack leaves a dpkg
journal and the next apt-get aborts in debSystem::Lock(), making the
retry useless.
---
 .github/workflows/cross-library.yml  | 6 ++++++
 .github/workflows/falcon-interop.yml | 6 ++++++
 .github/workflows/sbom.yml           | 3 +++
 3 files changed, 15 insertions(+)

diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml
index 34bb4da8c41..d8556d89475 100644
--- a/.github/workflows/cross-library.yml
+++ b/.github/workflows/cross-library.yml
@@ -75,6 +75,9 @@ jobs:
           # resumes from archives/partial/, so a killed transfer is not lost.
           APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
           for i in 1 2; do
+            # An attempt killed mid-unpack leaves dpkg needing this, or the
+            # next apt-get aborts instantly in debSystem::Lock().
+            dpkg --configure -a >/dev/null 2>&1 || true
             if timeout -k 10 60 apt-get $APT_OPTS update -q && \
                timeout -k 10 300 apt-get $APT_OPTS install -y \
                  --no-install-recommends \
@@ -158,6 +161,9 @@ jobs:
           # resumes from archives/partial/, so a killed transfer is not lost.
           APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
           for i in 1 2; do
+            # An attempt killed mid-unpack leaves dpkg needing this, or the
+            # next apt-get aborts instantly in debSystem::Lock().
+            dpkg --configure -a >/dev/null 2>&1 || true
             if timeout -k 10 60 apt-get $APT_OPTS update -q && \
                timeout -k 10 300 apt-get $APT_OPTS install -y \
                  --no-install-recommends \
diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml
index 1cf613e2ff4..7832578f803 100644
--- a/.github/workflows/falcon-interop.yml
+++ b/.github/workflows/falcon-interop.yml
@@ -100,6 +100,9 @@ jobs:
           apt_retry() {
             local i
             for i in 1 2; do
+              # An attempt killed mid-unpack leaves dpkg needing this, or the
+              # next apt-get aborts instantly in debSystem::Lock().
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
               if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
                  sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
@@ -188,6 +191,9 @@ jobs:
           apt_retry() {
             local i
             for i in 1 2; do
+              # An attempt killed mid-unpack leaves dpkg needing this, or the
+              # next apt-get aborts instantly in debSystem::Lock().
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
               if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
                  sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml
index 8a6389a6c91..4861aee5671 100644
--- a/.github/workflows/sbom.yml
+++ b/.github/workflows/sbom.yml
@@ -831,6 +831,9 @@ jobs:
           apt_retry() {
             local i
             for i in 1 2; do
+              # An attempt killed mid-unpack leaves dpkg needing this, or the
+              # next apt-get aborts instantly in debSystem::Lock().
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
               if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
                  sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0

From e2866ad15653318061ab2733eb077a8ee7c559b9 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Mon, 24 Aug 2026 12:11:46 +0000
Subject: [PATCH 5/9] CI: bound the ccache-setup apt fallback

This loop has no budget input and chains onto install-apt-deps in the
same job, several of which are capped at 10 minutes. Drop the install
deadline to 90s (ccache is ~700 KB) and stop sleeping after the final
failed attempt: worst case 510s -> 250s.
---
 .github/actions/ccache-setup/action.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index a39b93567d3..306b6e1d90a 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -58,6 +58,9 @@ runs:
             # Same defence in depth as install-apt-deps: Acquire timeouts drop
             # a stalled connection, `timeout` hard-kills a wedged apt-get, and
             # only then does the retry loop get a non-zero exit to act on.
+            # 60s+90s x2: ccache is a ~700 KB package, and this loop has no
+            # budget input of its own, so it has to stay small enough to chain
+            # after install-apt-deps inside a 10-minute job.
             APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                       -o Acquire::https::Timeout=30)
             ok=""
@@ -65,14 +68,16 @@ runs:
               sudo dpkg --configure -a >/dev/null 2>&1 || true
               if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \
                      apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 180 \
+                 sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 90 \
                      apt-get "${APT_OPTS[@]}" install -y \
                      --no-install-recommends ccache; then
                 ok=1
                 break
               fi
               echo "::warning::ccache apt install failed (attempt $i/2)"
-              sleep 5
+              # No sleep after the last attempt - this loop is not budgeted by
+              # the caller and chains onto install-apt-deps in the same job.
+              [ "$i" -eq 2 ] || sleep 5
             done
             [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
           fi

From bba979c79dfd3f096b23db8b7209834f7862bb33 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Mon, 24 Aug 2026 12:11:46 +0000
Subject: [PATCH 6/9] CI: drop the apt budget overrides from three jobs with no
 timeout-minutes

build_ipmitool, jwt-cpp's and pam-ipmi's build_pam-ipmi declare no
timeout-minutes - the 4 in each file belongs to the sibling
build_wolfssl, which does not call this action. The comment justifying
the tighter budget was false and the override only cost resilience.
---
 .github/workflows/ipmitool.yml | 3 ---
 .github/workflows/jwt-cpp.yml  | 3 ---
 .github/workflows/pam-ipmi.yml | 3 ---
 3 files changed, 9 deletions(-)

diff --git a/.github/workflows/ipmitool.yml b/.github/workflows/ipmitool.yml
index 6db463a4e63..b5df5fbe124 100644
--- a/.github/workflows/ipmitool.yml
+++ b/.github/workflows/ipmitool.yml
@@ -63,9 +63,6 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
-          # Fit the loop inside this job's timeout-minutes.
-          budget-seconds: '120'
-          pull-timeout: '60'
           packages: libreadline-dev
           ghcr-debs-tag: ubuntu-24.04-full
       - name: Download lib
diff --git a/.github/workflows/jwt-cpp.yml b/.github/workflows/jwt-cpp.yml
index df9a56247f0..e3263f3f996 100644
--- a/.github/workflows/jwt-cpp.yml
+++ b/.github/workflows/jwt-cpp.yml
@@ -67,9 +67,6 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
-          # Fit the loop inside this job's timeout-minutes.
-          budget-seconds: '120'
-          pull-timeout: '60'
           packages: libgtest-dev
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/pam-ipmi.yml b/.github/workflows/pam-ipmi.yml
index faa19001a5a..de907d05aff 100644
--- a/.github/workflows/pam-ipmi.yml
+++ b/.github/workflows/pam-ipmi.yml
@@ -64,9 +64,6 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
-          # Fit the loop inside this job's timeout-minutes.
-          budget-seconds: '120'
-          pull-timeout: '60'
           packages: libpam-dev ninja-build meson
           ghcr-debs-tag: ubuntu-24.04-full
 

From 88e40d52df8047f9d815f76d53ac576972b346db Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Mon, 24 Aug 2026 12:11:46 +0000
Subject: [PATCH 7/9] CI: give pq-all an apt budget that matches its job

pq-all is the only caller whose packages are not a subset of its
bundle: crossbuild-essential-* appear only in the 22.04 lists, and the
offline install is all-or-nothing, so it always takes the apt path -
the one the 45-minute timeout was widened for. The default budget caps
each install attempt at 250s and hard-fails a merely slow download.

Also state the real budget-seconds arithmetic: the loop overshoots by
retry-delay plus the SIGKILL grace, the floors make small values inert,
and the defaults need ~16 minutes rather than the 15 claimed.
---
 .github/actions/install-apt-deps/action.yml | 12 +++++++-----
 .github/workflows/pq-all.yml                |  7 +++++++
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index 03417164832..865d910c910 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -10,11 +10,13 @@ inputs:
     default: '2'
   budget-seconds:
     description: >
-      Wall-clock the whole retry loop may spend. Split across the attempts,
-      so a wedged mirror is reported by this action instead of the job being
-      cancelled around it. Must fit inside the caller's timeout-minutes
-      alongside pull-timeout; the default suits jobs of 15 minutes or more,
-      short jobs have to lower it.
+      Nominal wall-clock for the whole retry loop, split across the attempts
+      as per-command deadlines, so a wedged mirror is reported by this action
+      instead of the job being cancelled around it. The loop overshoots it by
+      retry-delay plus 10s of SIGKILL grace, and the per-command floors make
+      values below retries*60 inert. This, plus pull-timeout, plus any
+      ccache-setup in the same job, has to fit the caller's timeout-minutes -
+      the defaults need ~16 minutes.
     required: false
     default: '600'
   pull-timeout:
diff --git a/.github/workflows/pq-all.yml b/.github/workflows/pq-all.yml
index 71cfa85b3f5..c1b4e06625a 100644
--- a/.github/workflows/pq-all.yml
+++ b/.github/workflows/pq-all.yml
@@ -51,6 +51,13 @@ jobs:
         with:
           packages: autoconf automake libtool build-essential bubblewrap crossbuild-essential-arm64 crossbuild-essential-armhf
           ghcr-debs-tag: ubuntu-24.04-minimal
+          # crossbuild-essential-* are not in the -minimal bundle (only the
+          # 22.04 lists carry them), and the offline install is all-or-nothing,
+          # so this caller always takes the apt path - see the timeout-minutes
+          # note above. The default budget would cap each install attempt at
+          # 250s and hard-fail a download that merely runs slow; 1200 gives it
+          # 500s an attempt and still reports inside the 45-minute job.
+          budget-seconds: '1200'
 
       # ccache via the cross-platform composite; the script passes the
       # compiler to configure as CC="ccache gcc" (or a per-config "cc").

From b9eabfbaf52371642a91f195eac03f8c53a40ee3 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Mon, 24 Aug 2026 12:12:01 +0000
Subject: [PATCH 8/9] CI: make the linuxkm bundle retry helper do the 2
 attempts it documents

The trailing "$@" after the loop made it three, so a wedged mirror
could spend 19m20s of the 20-minute job and be cancelled instead of
reporting - the thing the 5-to-2 cut was for.
---
 .github/workflows/ci-deps-image.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml
index 76f1a65489d..740b14ae402 100644
--- a/.github/workflows/ci-deps-image.yml
+++ b/.github/workflows/ci-deps-image.yml
@@ -194,8 +194,10 @@ jobs:
           sudo apt-get clean
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
           # 2 attempts, not 5: this job's timeout-minutes is 20, and an
-          # attempt cut off mid-flight reports nothing.
-          retry() { local i; for i in 1 2; do "$@" && return 0; sleep 5; done; "$@"; }
+          # attempt cut off mid-flight reports nothing. The explicit return
+          # keeps the real exit code (124 from timeout) and stops a bare
+          # `sleep` from making an exhausted retry look like success.
+          retry() { local i rc=0; for i in 1 2; do "$@" && return 0; rc=$?; [ "$i" -eq 2 ] || sleep 5; done; return "$rc"; }
           retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only

From e8166219b57b18b295ab8fbe00400b629281362c Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Mon, 24 Aug 2026 12:12:01 +0000
Subject: [PATCH 9/9] CI: key the linuxkm bundle rebuild on the package set,
 not just the kernel

The gate compared only the published image's kernel label with uname -r,
so a package added to the bundle never shipped until the kernel next
rolled (~monthly) - and workflow_dispatch could not force it either.
The bundle published on 2026-08-14 still carries neither the toolchain
nor ccache for that reason, so fips-dev-no-post's offline ccache probe
fails on every run and reaches the mirror.

Label the image with a hash of the package set and compare that too.
The list now lives in the check step and is passed to the download, so
the gate and the closure cannot drift apart.
---
 .github/workflows/ci-deps-image.yml | 41 +++++++++++++++++++----------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml
index 740b14ae402..b28e653d0ac 100644
--- a/.github/workflows/ci-deps-image.yml
+++ b/.github/workflows/ci-deps-image.yml
@@ -33,8 +33,9 @@ on:
     - cron: '0 2 * * 6'
     # Daily - the kernel-tracking -linuxkm bundle only. linux-headers-$(uname
     # -r) pins to the runner's running kernel (changes ~monthly); the linuxkm
-    # job rebuilds solely when uname -r differs from the published bundle, a
-    # cheap no-op otherwise. A mismatch mid-rollout just falls back to apt.
+    # job rebuilds when uname -r or its package list differs from the
+    # published bundle, a cheap no-op otherwise. A mismatch mid-rollout just
+    # falls back to apt.
     - cron: '0 3 * * *'
   workflow_dispatch:
 
@@ -141,9 +142,9 @@ jobs:
   # Kernel-tracking bundle for the linux kernel-module builds (linuxkm.yml and
   # the membrowse linuxkm targets). linux-headers-$(uname -r) pins to the
   # runner's running kernel, so this runs daily but rebuilds only when the
-  # kernel changed since the published bundle (the image carries the kernel as
-  # a label). A mismatch - e.g. during a gradual runner-image rollout - just
-  # makes install-apt-deps fall back to apt.
+  # kernel or the package list changed since the published bundle (the image
+  # carries both as labels). A mismatch - e.g. during a gradual runner-image
+  # rollout - just makes install-apt-deps fall back to apt.
   linuxkm:
     name: build ubuntu-24.04-linuxkm
     if: github.repository_owner == 'wolfssl'
@@ -154,7 +155,7 @@ jobs:
         shell: bash
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
 
-      - name: Decide whether the published bundle already matches this kernel
+      - name: Decide whether the published bundle is still current
         id: check
         shell: bash
         run: |
@@ -162,14 +163,25 @@ jobs:
           OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
           IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm"
           K=$(uname -r)
+          # The package set is defined here, not in the download step below, so
+          # the rebuild gate and the download cannot drift apart. Keyed on the
+          # kernel alone the gate would pin the published package set in place
+          # until the kernel next rolls (~monthly), so a package added here
+          # would never ship. The pkgset label closes that.
+          PKGS="build-essential autoconf automake libtool ccache"
+          PKGSET=$(printf '%s\n' "$PKGS" "linux-headers-$K" | sha256sum | cut -c1-12)
           echo "kernel=$K" >> "$GITHUB_OUTPUT"
-          echo "runner kernel: $K"
+          echo "pkgs=$PKGS" >> "$GITHUB_OUTPUT"
+          echo "pkgset=$PKGSET" >> "$GITHUB_OUTPUT"
+          echo "runner kernel: $K, package set: $PKGSET"
           have=""
+          havepkgset=""
           if docker pull -q "$IMG" >/dev/null 2>&1; then
             have=$(docker inspect --format '{{ index .Config.Labels "kernel" }}' "$IMG" 2>/dev/null || true)
+            havepkgset=$(docker inspect --format '{{ index .Config.Labels "pkgset" }}' "$IMG" 2>/dev/null || true)
           fi
-          echo "published bundle kernel: ${have:-<none>}"
-          if [ "$have" = "$K" ]; then
+          echo "published bundle kernel: ${have:-<none>}, package set: ${havepkgset:-<none>}"
+          if [ "$have" = "$K" ] && [ "$havepkgset" = "$PKGSET" ]; then
             echo "rebuild=false" >> "$GITHUB_OUTPUT"
             echo "Bundle already current for $K; nothing to do."
           else
@@ -185,9 +197,9 @@ jobs:
           # linuxkm.yml installs only the headers; the membrowse linuxkm targets
           # also need the build toolchain, and ccache-setup installs ccache
           # offline from whatever this bundle staged. Bundle the union - each
-          # consumer installs its own subset offline.
-          PKGS=(build-essential autoconf automake libtool ccache
-                "linux-headers-$K")
+          # consumer installs its own subset offline. The list comes from the
+          # check step so it stays in step with the pkgset the gate compares.
+          read -r -a PKGS <<< "${{ steps.check.outputs.pkgs }} linux-headers-$K"
           echo "Packages: ${PKGS[*]}"
           export DEBIAN_FRONTEND=noninteractive
           rm -rf debs && mkdir -p debs
@@ -210,7 +222,7 @@ jobs:
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
           test -n "$(ls debs/*.deb 2>/dev/null)"  # headers are never preinstalled
 
-      - name: Build and push bundle (labelled with the kernel)
+      - name: Build and push bundle (labelled with the kernel and package set)
         if: steps.check.outputs.rebuild == 'true'
         shell: bash
         run: |
@@ -218,7 +230,8 @@ jobs:
           K="${{ steps.check.outputs.kernel }}"
           OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
           IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm"
-          printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\n' "$K" > Dockerfile.debs
+          printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\nLABEL pkgset=%s\n' \
+            "$K" "${{ steps.check.outputs.pkgset }}" > Dockerfile.debs
           docker build -f Dockerfile.debs -t "$IMG" .
           docker push "$IMG"
           echo "Pushed $IMG (kernel $K)"