CI: bound apt calls with timeouts and install ccache from staged bundle

.github/actions/ccache-setup/action.yml

@@ -42,9 +42,45 @@ runs:
         if command -v ccache >/dev/null 2>&1; then
           echo "ccache already installed: $(ccache --version | head -1)"
         elif [ "${{ runner.os }}" = "Linux" ]; then
-          sudo apt-get update -q
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            --no-install-recommends ccache
+          export DEBIAN_FRONTEND=noninteractive
+          # install-apt-deps stages the WHOLE ghcr bundle into
+          # /var/cache/apt/archives, and ccache is in the -minimal/-full
+          # lists, so in a job that ran it first the .deb is already on disk.
+          # Take it offline (--no-download): no apt-get update, nothing to
+          # stall on. Every other path here reaches the mirror, which is what
+          # used to hang these jobs for 10-40 min after the bundle had
+          # already installed cleanly.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+               --no-install-recommends --no-download ccache; then
+            echo "ccache installed offline from the staged .deb bundle"
+          else
+            # Same defence in depth as install-apt-deps: Acquire timeouts drop
+            # a stalled connection, `timeout` hard-kills a wedged apt-get, and
+            # only then does the retry loop get a non-zero exit to act on.
+            # 60s+90s x2: ccache is a ~700 KB package, and this loop has no
+            # budget input of its own, so it has to stay small enough to chain
+            # after install-apt-deps inside a 10-minute job.
+            APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                      -o Acquire::https::Timeout=30)
+            ok=""
+            for i in 1 2; do
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
+              if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \
+                     apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 90 \
+                     apt-get "${APT_OPTS[@]}" install -y \
+                     --no-install-recommends ccache; then
+                ok=1
+                break
+              fi
+              echo "::warning::ccache apt install failed (attempt $i/2)"
+              # No sleep after the last attempt - this loop is not budgeted by
+              # the caller and chains onto install-apt-deps in the same job.
+              [ "$i" -eq 2 ] || sleep 5
+            done
+            [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
+          fi
         elif [ "${{ runner.os }}" = "macOS" ]; then
           brew install ccache
         else

.github/actions/install-apt-deps/action.yml

@@ -5,9 +5,26 @@ inputs:
     description: 'Space-separated list of apt packages to install'
     required: true
   retries:
-    description: 'Number of retry attempts'
+    description: 'Number of retry attempts.'
     required: false
-    default: '3'
+    default: '2'
+  budget-seconds:
+    description: >
+      Nominal wall-clock for the whole retry loop, split across the attempts
+      as per-command deadlines, so a wedged mirror is reported by this action
+      instead of the job being cancelled around it. The loop overshoots it by
+      retry-delay plus 10s of SIGKILL grace, and the per-command floors make
+      values below retries*60 inert. This, plus pull-timeout, plus any
+      ccache-setup in the same job, has to fit the caller's timeout-minutes -
+      the defaults need ~16 minutes.
+    required: false
+    default: '600'
+  pull-timeout:
+    description: >
+      Deadline for the ghcr bundle pull. Counts against the same
+      timeout-minutes as budget-seconds.
+    required: false
+    default: '300'
   retry-delay:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
@@ -50,7 +67,7 @@ runs:
         # PRs read the public upstream image too rather than a nonexistent
         # ghcr.io/<fork>/wolfssl-ci-debs.
         IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}"
-        if ! docker pull -q "$IMG" >/dev/null 2>&1; then
+        if ! timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" >/dev/null 2>&1; then
           echo "::notice::ghcr bundle $IMG unavailable; using apt"
           exit 0
         fi
@@ -77,21 +94,52 @@ runs:
       if: steps.ghcr.outputs.satisfied != 'true'
       shell: bash
       run: |
-        export DEBIAN_FRONTEND=noninteractive
         RETRIES=${{ inputs.retries }}
         DELAY=${{ inputs.retry-delay }}
+        BUDGET=${{ inputs.budget-seconds }}
         NO_REC=""
         if [ "${{ inputs.no-install-recommends }}" = "true" ]; then
           NO_REC="--no-install-recommends"
         fi
 
+        # A wedged mirror hangs apt rather than failing it, so the retry loop
+        # below never fired and the job burned its whole budget instead.
+        # Defend in depth: apt drops a stalled connection after 30s and retries
+        # it (Acquire timeouts - this is what actually detects a wedge, in
+        # ~90s), `timeout` hard-kills an apt-get that wedged outside its own
+        # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a
+        # retry can land on a different mirror. apt resumes from
+        # archives/partial/, so a killed transfer is not restarted from
+        # scratch.
+        APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                  -o Acquire::https::Timeout=30)
+
+        # Spend budget-seconds over the attempts rather than a fixed
+        # per-attempt deadline: a caller with a short timeout-minutes would
+        # otherwise be cancelled mid-attempt, before the loop could report
+        # the failure. update gets a sixth of an attempt, install the rest,
+        # with floors so a small budget still leaves apt time to work.
+        PER=$((BUDGET / RETRIES))
+        UPD=$((PER / 6))
+        [ "$UPD" -ge 20 ] || UPD=20
+        INS=$((PER - UPD))
+        [ "$INS" -ge 40 ] || INS=40
+        DEADLINE=$(($(date +%s) + BUDGET))
+
+        # sudo resets the environment, so DEBIAN_FRONTEND has to ride along
+        # on each privileged command rather than being exported once.
         for i in $(seq 1 $RETRIES); do
-          if sudo apt-get update -q && \
-             sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
+          # A previous attempt killed mid-unpack leaves dpkg needing this.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \
+                 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \
+                 apt-get "${APT_OPTS[@]}" install -y \
+                 $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
-          if [ "$i" -eq "$RETRIES" ]; then
-            echo "::error::apt-get failed after $RETRIES attempts"
+          if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then
+            echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s"
             exit 1
           fi
           echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."

.github/ci-deps/packages-ubuntu-22.04-minimal.txt

@@ -4,6 +4,7 @@
 autoconf
 automake
 build-essential
+ccache
 crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf

.github/scripts/zephyr-4.x/zephyr-test.sh

@@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..."
 west zephyr-export
 
 echo "==> [container] Installing host packages (newlib, python3-venv)..."
-sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true
+# `|| true` keeps this best-effort, but without a timeout a wedged mirror
+# stalls here silently until the job budget runs out.
+APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \
+  && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \
+       python3-venv libnewlib-dev >/dev/null 2>&1 \
+  || echo "==> [container] host package install skipped (apt unavailable)"
 python3 -m venv .venv
 source .venv/bin/activate
 pip3 install west

.github/workflows/check-source-text.yml

@@ -45,6 +45,9 @@ jobs:
       - name: Install shellcheck
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: shellcheck python3-yaml
           ghcr-debs-tag: ubuntu-24.04-full
 

.github/workflows/ci-deps-image.yml

@@ -33,8 +33,9 @@ on:
     - cron: '0 2 * * 6'
     # Daily - the kernel-tracking -linuxkm bundle only. linux-headers-$(uname
     # -r) pins to the runner's running kernel (changes ~monthly); the linuxkm
-    # job rebuilds solely when uname -r differs from the published bundle, a
-    # cheap no-op otherwise. A mismatch mid-rollout just falls back to apt.
+    # job rebuilds when uname -r or its package list differs from the
+    # published bundle, a cheap no-op otherwise. A mismatch mid-rollout just
+    # falls back to apt.
     - cron: '0 3 * * *'
   workflow_dispatch:
 
@@ -141,9 +142,9 @@ jobs:
   # Kernel-tracking bundle for the linux kernel-module builds (linuxkm.yml and
   # the membrowse linuxkm targets). linux-headers-$(uname -r) pins to the
   # runner's running kernel, so this runs daily but rebuilds only when the
-  # kernel changed since the published bundle (the image carries the kernel as
-  # a label). A mismatch - e.g. during a gradual runner-image rollout - just
-  # makes install-apt-deps fall back to apt.
+  # kernel or the package list changed since the published bundle (the image
+  # carries both as labels). A mismatch - e.g. during a gradual runner-image
+  # rollout - just makes install-apt-deps fall back to apt.
   linuxkm:
     name: build ubuntu-24.04-linuxkm
     if: github.repository_owner == 'wolfssl'
@@ -154,22 +155,33 @@ jobs:
         shell: bash
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
 
-      - name: Decide whether the published bundle already matches this kernel
+      - name: Decide whether the published bundle is still current
         id: check
         shell: bash
         run: |
           set -uo pipefail
           OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
           IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm"
           K=$(uname -r)
+          # The package set is defined here, not in the download step below, so
+          # the rebuild gate and the download cannot drift apart. Keyed on the
+          # kernel alone the gate would pin the published package set in place
+          # until the kernel next rolls (~monthly), so a package added here
+          # would never ship. The pkgset label closes that.
+          PKGS="build-essential autoconf automake libtool ccache"
+          PKGSET=$(printf '%s\n' "$PKGS" "linux-headers-$K" | sha256sum | cut -c1-12)
           echo "kernel=$K" >> "$GITHUB_OUTPUT"
-          echo "runner kernel: $K"
+          echo "pkgs=$PKGS" >> "$GITHUB_OUTPUT"
+          echo "pkgset=$PKGSET" >> "$GITHUB_OUTPUT"
+          echo "runner kernel: $K, package set: $PKGSET"
           have=""
+          havepkgset=""
           if docker pull -q "$IMG" >/dev/null 2>&1; then
             have=$(docker inspect --format '{{ index .Config.Labels "kernel" }}' "$IMG" 2>/dev/null || true)
+            havepkgset=$(docker inspect --format '{{ index .Config.Labels "pkgset" }}' "$IMG" 2>/dev/null || true)
           fi
-          echo "published bundle kernel: ${have:-<none>}"
-          if [ "$have" = "$K" ]; then
+          echo "published bundle kernel: ${have:-<none>}, package set: ${havepkgset:-<none>}"
+          if [ "$have" = "$K" ] && [ "$havepkgset" = "$PKGSET" ]; then
             echo "rebuild=false" >> "$GITHUB_OUTPUT"
             echo "Bundle already current for $K; nothing to do."
           else
@@ -183,34 +195,43 @@ jobs:
           set -euo pipefail
           K="${{ steps.check.outputs.kernel }}"
           # linuxkm.yml installs only the headers; the membrowse linuxkm targets
-          # also need the build toolchain. Bundle the union - each consumer
-          # installs its own subset offline.
-          PKGS=(build-essential autoconf automake libtool "linux-headers-$K")
+          # also need the build toolchain, and ccache-setup installs ccache
+          # offline from whatever this bundle staged. Bundle the union - each
+          # consumer installs its own subset offline. The list comes from the
+          # check step so it stays in step with the pkgset the gate compares.
+          read -r -a PKGS <<< "${{ steps.check.outputs.pkgs }} linux-headers-$K"
           echo "Packages: ${PKGS[*]}"
           export DEBIAN_FRONTEND=noninteractive
           rm -rf debs && mkdir -p debs
           sudo apt-get clean
-          retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
-          retry sudo apt-get update -q
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+          # 2 attempts, not 5: this job's timeout-minutes is 20, and an
+          # attempt cut off mid-flight reports nothing. The explicit return
+          # keeps the real exit code (124 from timeout) and stops a bare
+          # `sleep` from making an exhausted retry look like success.
+          retry() { local i rc=0; for i in 1 2; do "$@" && return 0; rc=$?; [ "$i" -eq 2 ] || sleep 5; done; return "$rc"; }
+          retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only
           # on success, so a transient mirror error keeps the last good bundle
           # rather than publishing a partial one - which the kernel-label skip
           # would then pin in place until the kernel next changes (~monthly).
-          retry sudo apt-get install -y --download-only "${PKGS[@]}"
+          retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
+            --download-only "${PKGS[@]}"
           sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
           test -n "$(ls debs/*.deb 2>/dev/null)"  # headers are never preinstalled
 
-      - name: Build and push bundle (labelled with the kernel)
+      - name: Build and push bundle (labelled with the kernel and package set)
         if: steps.check.outputs.rebuild == 'true'
         shell: bash
         run: |
           set -euo pipefail
           K="${{ steps.check.outputs.kernel }}"
           OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
           IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm"
-          printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\n' "$K" > Dockerfile.debs
+          printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\nLABEL pkgset=%s\n' \
+            "$K" "${{ steps.check.outputs.pkgset }}" > Dockerfile.debs
           docker build -f Dockerfile.debs -t "$IMG" .
           docker push "$IMG"
           echo "Pushed $IMG (kernel $K)"

.github/workflows/cross-library.yml

@@ -64,10 +64,31 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            # An attempt killed mid-unpack leaves dpkg needing this, or the
+            # next apt-get aborts instantly in debSystem::Lock().
+            dpkg --configure -a >/dev/null 2>&1 || true
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
+          done
 
       # Building only needs the commit under test, not history. The break check
       # that needs history runs in the compile job, not here.
@@ -129,10 +150,31 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            # An attempt killed mid-unpack leaves dpkg needing this, or the
+            # next apt-get aborts instantly in debSystem::Lock().
+            dpkg --configure -a >/dev/null 2>&1 || true
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
+          done
 
       # This job does not build wolfSSL, but the latest leg still checks out
       # wolfSSL history because check-break.sh scans commit messages here. The

.github/workflows/cyrus-sasl.yml

@@ -66,6 +66,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: krb5-kdc krb5-otp libkrb5-dev libsocket-wrapper libnss-wrapper krb5-admin-server libdb5.3-dev
           ghcr-debs-tag: ubuntu-24.04-full