diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml index f68734209fa..306b6e1d90a 100644 --- a/.github/actions/ccache-setup/action.yml +++ b/.github/actions/ccache-setup/action.yml @@ -42,9 +42,45 @@ runs: if command -v ccache >/dev/null 2>&1; then echo "ccache already installed: $(ccache --version | head -1)" elif [ "${{ runner.os }}" = "Linux" ]; then - sudo apt-get update -q - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - --no-install-recommends ccache + export DEBIAN_FRONTEND=noninteractive + # install-apt-deps stages the WHOLE ghcr bundle into + # /var/cache/apt/archives, and ccache is in the -minimal/-full + # lists, so in a job that ran it first the .deb is already on disk. + # Take it offline (--no-download): no apt-get update, nothing to + # stall on. Every other path here reaches the mirror, which is what + # used to hang these jobs for 10-40 min after the bundle had + # already installed cleanly. + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + --no-install-recommends --no-download ccache; then + echo "ccache installed offline from the staged .deb bundle" + else + # Same defence in depth as install-apt-deps: Acquire timeouts drop + # a stalled connection, `timeout` hard-kills a wedged apt-get, and + # only then does the retry loop get a non-zero exit to act on. + # 60s+90s x2: ccache is a ~700 KB package, and this loop has no + # budget input of its own, so it has to stay small enough to chain + # after install-apt-deps inside a 10-minute job. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + ok="" + for i in 1 2; do + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \ + apt-get "${APT_OPTS[@]}" update -q && \ + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 90 \ + apt-get "${APT_OPTS[@]}" install -y \ + --no-install-recommends ccache; then + ok=1 + break + fi + echo "::warning::ccache apt install failed (attempt $i/2)" + # No sleep after the last attempt - this loop is not budgeted by + # the caller and chains onto install-apt-deps in the same job. + [ "$i" -eq 2 ] || sleep 5 + done + [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; } + fi elif [ "${{ runner.os }}" = "macOS" ]; then brew install ccache else diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml index b8cb68b4e4a..865d910c910 100644 --- a/.github/actions/install-apt-deps/action.yml +++ b/.github/actions/install-apt-deps/action.yml @@ -5,9 +5,26 @@ inputs: description: 'Space-separated list of apt packages to install' required: true retries: - description: 'Number of retry attempts' + description: 'Number of retry attempts.' required: false - default: '3' + default: '2' + budget-seconds: + description: > + Nominal wall-clock for the whole retry loop, split across the attempts + as per-command deadlines, so a wedged mirror is reported by this action + instead of the job being cancelled around it. The loop overshoots it by + retry-delay plus 10s of SIGKILL grace, and the per-command floors make + values below retries*60 inert. This, plus pull-timeout, plus any + ccache-setup in the same job, has to fit the caller's timeout-minutes - + the defaults need ~16 minutes. + required: false + default: '600' + pull-timeout: + description: > + Deadline for the ghcr bundle pull. Counts against the same + timeout-minutes as budget-seconds. + required: false + default: '300' retry-delay: description: 'Initial delay between retries (seconds, doubles each attempt)' required: false @@ -50,7 +67,7 @@ runs: # PRs read the public upstream image too rather than a nonexistent # ghcr.io//wolfssl-ci-debs. IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}" - if ! docker pull -q "$IMG" >/dev/null 2>&1; then + if ! timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" >/dev/null 2>&1; then echo "::notice::ghcr bundle $IMG unavailable; using apt" exit 0 fi @@ -77,21 +94,52 @@ runs: if: steps.ghcr.outputs.satisfied != 'true' shell: bash run: | - export DEBIAN_FRONTEND=noninteractive RETRIES=${{ inputs.retries }} DELAY=${{ inputs.retry-delay }} + BUDGET=${{ inputs.budget-seconds }} NO_REC="" if [ "${{ inputs.no-install-recommends }}" = "true" ]; then NO_REC="--no-install-recommends" fi + # A wedged mirror hangs apt rather than failing it, so the retry loop + # below never fired and the job burned its whole budget instead. + # Defend in depth: apt drops a stalled connection after 30s and retries + # it (Acquire timeouts - this is what actually detects a wedge, in + # ~90s), `timeout` hard-kills an apt-get that wedged outside its own + # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a + # retry can land on a different mirror. apt resumes from + # archives/partial/, so a killed transfer is not restarted from + # scratch. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + + # Spend budget-seconds over the attempts rather than a fixed + # per-attempt deadline: a caller with a short timeout-minutes would + # otherwise be cancelled mid-attempt, before the loop could report + # the failure. update gets a sixth of an attempt, install the rest, + # with floors so a small budget still leaves apt time to work. + PER=$((BUDGET / RETRIES)) + UPD=$((PER / 6)) + [ "$UPD" -ge 20 ] || UPD=20 + INS=$((PER - UPD)) + [ "$INS" -ge 40 ] || INS=40 + DEADLINE=$(($(date +%s) + BUDGET)) + + # sudo resets the environment, so DEBIAN_FRONTEND has to ride along + # on each privileged command rather than being exported once. for i in $(seq 1 $RETRIES); do - if sudo apt-get update -q && \ - sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then + # A previous attempt killed mid-unpack leaves dpkg needing this. + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \ + apt-get "${APT_OPTS[@]}" update -q && \ + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \ + apt-get "${APT_OPTS[@]}" install -y \ + $NO_REC ${{ inputs.packages }}; then exit 0 fi - if [ "$i" -eq "$RETRIES" ]; then - echo "::error::apt-get failed after $RETRIES attempts" + if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then + echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s" exit 1 fi echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..." diff --git a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt index c32e3ccb9cf..6fdbf8cfbba 100644 --- a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt +++ b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt @@ -4,6 +4,7 @@ autoconf automake build-essential +ccache crossbuild-essential-arm64 crossbuild-essential-armel crossbuild-essential-armhf diff --git a/.github/scripts/zephyr-4.x/zephyr-test.sh b/.github/scripts/zephyr-4.x/zephyr-test.sh index c225277cacc..66a2930d64c 100755 --- a/.github/scripts/zephyr-4.x/zephyr-test.sh +++ b/.github/scripts/zephyr-4.x/zephyr-test.sh @@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..." west zephyr-export echo "==> [container] Installing host packages (newlib, python3-venv)..." -sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true +# `|| true` keeps this best-effort, but without a timeout a wedged mirror +# stalls here silently until the job budget runs out. +APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) +sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \ + && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \ + python3-venv libnewlib-dev >/dev/null 2>&1 \ + || echo "==> [container] host package install skipped (apt unavailable)" python3 -m venv .venv source .venv/bin/activate pip3 install west diff --git a/.github/workflows/check-source-text.yml b/.github/workflows/check-source-text.yml index cf3efb95da2..7f9524563b8 100644 --- a/.github/workflows/check-source-text.yml +++ b/.github/workflows/check-source-text.yml @@ -45,6 +45,9 @@ jobs: - name: Install shellcheck uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: shellcheck python3-yaml ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml index d91ac4fb877..b28e653d0ac 100644 --- a/.github/workflows/ci-deps-image.yml +++ b/.github/workflows/ci-deps-image.yml @@ -33,8 +33,9 @@ on: - cron: '0 2 * * 6' # Daily - the kernel-tracking -linuxkm bundle only. linux-headers-$(uname # -r) pins to the runner's running kernel (changes ~monthly); the linuxkm - # job rebuilds solely when uname -r differs from the published bundle, a - # cheap no-op otherwise. A mismatch mid-rollout just falls back to apt. + # job rebuilds when uname -r or its package list differs from the + # published bundle, a cheap no-op otherwise. A mismatch mid-rollout just + # falls back to apt. - cron: '0 3 * * *' workflow_dispatch: @@ -141,9 +142,9 @@ jobs: # Kernel-tracking bundle for the linux kernel-module builds (linuxkm.yml and # the membrowse linuxkm targets). linux-headers-$(uname -r) pins to the # runner's running kernel, so this runs daily but rebuilds only when the - # kernel changed since the published bundle (the image carries the kernel as - # a label). A mismatch - e.g. during a gradual runner-image rollout - just - # makes install-apt-deps fall back to apt. + # kernel or the package list changed since the published bundle (the image + # carries both as labels). A mismatch - e.g. during a gradual runner-image + # rollout - just makes install-apt-deps fall back to apt. linuxkm: name: build ubuntu-24.04-linuxkm if: github.repository_owner == 'wolfssl' @@ -154,7 +155,7 @@ jobs: shell: bash run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - name: Decide whether the published bundle already matches this kernel + - name: Decide whether the published bundle is still current id: check shell: bash run: | @@ -162,14 +163,25 @@ jobs: OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm" K=$(uname -r) + # The package set is defined here, not in the download step below, so + # the rebuild gate and the download cannot drift apart. Keyed on the + # kernel alone the gate would pin the published package set in place + # until the kernel next rolls (~monthly), so a package added here + # would never ship. The pkgset label closes that. + PKGS="build-essential autoconf automake libtool ccache" + PKGSET=$(printf '%s\n' "$PKGS" "linux-headers-$K" | sha256sum | cut -c1-12) echo "kernel=$K" >> "$GITHUB_OUTPUT" - echo "runner kernel: $K" + echo "pkgs=$PKGS" >> "$GITHUB_OUTPUT" + echo "pkgset=$PKGSET" >> "$GITHUB_OUTPUT" + echo "runner kernel: $K, package set: $PKGSET" have="" + havepkgset="" if docker pull -q "$IMG" >/dev/null 2>&1; then have=$(docker inspect --format '{{ index .Config.Labels "kernel" }}' "$IMG" 2>/dev/null || true) + havepkgset=$(docker inspect --format '{{ index .Config.Labels "pkgset" }}' "$IMG" 2>/dev/null || true) fi - echo "published bundle kernel: ${have:-}" - if [ "$have" = "$K" ]; then + echo "published bundle kernel: ${have:-}, package set: ${havepkgset:-}" + if [ "$have" = "$K" ] && [ "$havepkgset" = "$PKGSET" ]; then echo "rebuild=false" >> "$GITHUB_OUTPUT" echo "Bundle already current for $K; nothing to do." else @@ -183,26 +195,34 @@ jobs: set -euo pipefail K="${{ steps.check.outputs.kernel }}" # linuxkm.yml installs only the headers; the membrowse linuxkm targets - # also need the build toolchain. Bundle the union - each consumer - # installs its own subset offline. - PKGS=(build-essential autoconf automake libtool "linux-headers-$K") + # also need the build toolchain, and ccache-setup installs ccache + # offline from whatever this bundle staged. Bundle the union - each + # consumer installs its own subset offline. The list comes from the + # check step so it stays in step with the pkgset the gate compares. + read -r -a PKGS <<< "${{ steps.check.outputs.pkgs }} linux-headers-$K" echo "Packages: ${PKGS[*]}" export DEBIAN_FRONTEND=noninteractive rm -rf debs && mkdir -p debs sudo apt-get clean - retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; } - retry sudo apt-get update -q + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) + # 2 attempts, not 5: this job's timeout-minutes is 20, and an + # attempt cut off mid-flight reports nothing. The explicit return + # keeps the real exit code (124 from timeout) and stops a bare + # `sleep` from making an exhausted retry look like success. + retry() { local i rc=0; for i in 1 2; do "$@" && return 0; rc=$?; [ "$i" -eq 2 ] || sleep 5; done; return "$rc"; } + retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q # The whole set is required and this bundle is small, so resolve it as # one closure and let any download failure fail the job. We push only # on success, so a transient mirror error keeps the last good bundle # rather than publishing a partial one - which the kernel-label skip # would then pin in place until the kernel next changes (~monthly). - retry sudo apt-get install -y --download-only "${PKGS[@]}" + retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \ + --download-only "${PKGS[@]}" sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files" test -n "$(ls debs/*.deb 2>/dev/null)" # headers are never preinstalled - - name: Build and push bundle (labelled with the kernel) + - name: Build and push bundle (labelled with the kernel and package set) if: steps.check.outputs.rebuild == 'true' shell: bash run: | @@ -210,7 +230,8 @@ jobs: K="${{ steps.check.outputs.kernel }}" OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm" - printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\n' "$K" > Dockerfile.debs + printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\nLABEL pkgset=%s\n' \ + "$K" "${{ steps.check.outputs.pkgset }}" > Dockerfile.debs docker build -f Dockerfile.debs -t "$IMG" . docker push "$IMG" echo "Pushed $IMG (kernel $K)" diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml index af25dd52cf1..d8556d89475 100644 --- a/.github/workflows/cross-library.yml +++ b/.github/workflows/cross-library.yml @@ -64,10 +64,31 @@ jobs: run: | set -eux export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }} + # These containers are bare images with no bash, so this step runs + # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it + # word-splits. + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection (and are what actually detects a wedge), + # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, + # and the loop then retries - re-reading the mirror list. Two + # attempts at 60s+300s fit inside this job's timeout-minutes; apt + # resumes from archives/partial/, so a killed transfer is not lost. + APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" + for i in 1 2; do + # An attempt killed mid-unpack leaves dpkg needing this, or the + # next apt-get aborts instantly in debSystem::Lock(). + dpkg --configure -a >/dev/null 2>&1 || true + if timeout -k 10 60 apt-get $APT_OPTS update -q && \ + timeout -k 10 300 apt-get $APT_OPTS install -y \ + --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }}; then + break + fi + test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done # Building only needs the commit under test, not history. The break check # that needs history runs in the compile job, not here. @@ -129,10 +150,31 @@ jobs: run: | set -eux export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }} + # These containers are bare images with no bash, so this step runs + # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it + # word-splits. + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection (and are what actually detects a wedge), + # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, + # and the loop then retries - re-reading the mirror list. Two + # attempts at 60s+300s fit inside this job's timeout-minutes; apt + # resumes from archives/partial/, so a killed transfer is not lost. + APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" + for i in 1 2; do + # An attempt killed mid-unpack leaves dpkg needing this, or the + # next apt-get aborts instantly in debSystem::Lock(). + dpkg --configure -a >/dev/null 2>&1 || true + if timeout -k 10 60 apt-get $APT_OPTS update -q && \ + timeout -k 10 300 apt-get $APT_OPTS install -y \ + --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }}; then + break + fi + test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done # This job does not build wolfSSL, but the latest leg still checks out # wolfSSL history because check-break.sh scans commit messages here. The diff --git a/.github/workflows/cyrus-sasl.yml b/.github/workflows/cyrus-sasl.yml index 5c1ad746e0a..c4a5bc3c97e 100644 --- a/.github/workflows/cyrus-sasl.yml +++ b/.github/workflows/cyrus-sasl.yml @@ -66,6 +66,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: krb5-kdc krb5-otp libkrb5-dev libsocket-wrapper libnss-wrapper krb5-admin-server libdb5.3-dev ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml index ad6bc8f0137..7832578f803 100644 --- a/.github/workflows/falcon-interop.yml +++ b/.github/workflows/falcon-interop.yml @@ -91,8 +91,29 @@ jobs: steps: - name: Install build tools run: | - sudo apt-get update - sudo apt-get install -y ninja-build + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + # An attempt killed mid-unpack leaves dpkg needing this, or the + # next apt-get aborts instantly in debSystem::Lock(). + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } + apt_retry ninja-build # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which # would delete an untracked oqs-install/ placed in the workspace by the @@ -161,8 +182,29 @@ jobs: steps: - name: Install build tools run: | - sudo apt-get update - sudo apt-get install -y autoconf automake libtool + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + # An attempt killed mid-unpack leaves dpkg needing this, or the + # next apt-get aborts instantly in debSystem::Lock(). + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } + apt_retry autoconf automake libtool - name: Checkout wolfSSL uses: actions/checkout@v5 diff --git a/.github/workflows/linuxkm.yml b/.github/workflows/linuxkm.yml index 8c404a4b0c8..a7194f2065d 100644 --- a/.github/workflows/linuxkm.yml +++ b/.github/workflows/linuxkm.yml @@ -36,6 +36,9 @@ jobs: - name: Install linux-headers uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: linux-headers-$(uname -r) ghcr-debs-tag: ubuntu-24.04-linuxkm diff --git a/.github/workflows/mosquitto.yml b/.github/workflows/mosquitto.yml index 49b515d274b..dbea1074939 100644 --- a/.github/workflows/mosquitto.yml +++ b/.github/workflows/mosquitto.yml @@ -78,6 +78,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: build-essential libev-dev libssl-dev automake python3-docutils libcunit1 libcunit1-doc libcunit1-dev pkg-config make python3-psutil ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/pq-all.yml b/.github/workflows/pq-all.yml index 71cfa85b3f5..c1b4e06625a 100644 --- a/.github/workflows/pq-all.yml +++ b/.github/workflows/pq-all.yml @@ -51,6 +51,13 @@ jobs: with: packages: autoconf automake libtool build-essential bubblewrap crossbuild-essential-arm64 crossbuild-essential-armhf ghcr-debs-tag: ubuntu-24.04-minimal + # crossbuild-essential-* are not in the -minimal bundle (only the + # 22.04 lists carry them), and the offline install is all-or-nothing, + # so this caller always takes the apt path - see the timeout-minutes + # note above. The default budget would cap each install attempt at + # 250s and hard-fail a download that merely runs slow; 1200 gives it + # 500s an attempt and still reports inside the 45-minute job. + budget-seconds: '1200' # ccache via the cross-platform composite; the script passes the # compiler to configure as CC="ccache gcc" (or a per-config "cc"). diff --git a/.github/workflows/rng-tools.yml b/.github/workflows/rng-tools.yml index 1027ab80fc6..f14877d0124 100644 --- a/.github/workflows/rng-tools.yml +++ b/.github/workflows/rng-tools.yml @@ -65,6 +65,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: + # Fit the loop inside this job's timeout-minutes. + budget-seconds: '120' + pull-timeout: '60' packages: libcurl4-openssl-dev libjansson-dev libp11-dev librtlsdr-dev libcap-dev ghcr-debs-tag: ubuntu-24.04-full diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 4e2b3fcd950..4861aee5671 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -822,7 +822,28 @@ jobs: - name: Install build deps + SBOM validators run: | - sudo apt-get update + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + # An attempt killed mid-unpack leaves dpkg needing this, or the + # next apt-get aborts instantly in debSystem::Lock(). + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } # bison + autotools-dev are required by strace's ./bootstrap. # gcc-multilib + g++-multilib give strace's --enable-mpers=check # the 32-bit/x32 compilers it needs - without them mpers is @@ -830,7 +851,7 @@ jobs: # syscalls, diverging from what bomsh's devcontainer produces. # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3 # stage. - sudo apt-get install -y build-essential autoconf automake libtool \ + apt_retry build-essential autoconf automake libtool \ bison autotools-dev gcc-multilib g++-multilib \ python3 python3-pip git python3 -m pip install --user --upgrade pip