CI: Address occasional ccache install hangs

.github/actions/ccache-setup/action.yml

@@ -23,6 +23,16 @@ inputs:
     description: 'Per-job ccache max size (passed to ccache -M).'
     required: false
     default: '500M'
+  ghcr-debs-tag:
+    description: >
+      Bundle to install ccache from offline (see install-apt-deps).
+      'auto' resolves to the runner's ubuntu-<version>-minimal bundle,
+      which already carries ccache and which most callers pull earlier in
+      the same job, making the second pull free. 'none' skips the bundle
+      and uses apt - for callers whose job pulls a different bundle,
+      where fetching this one costs more than it saves.
+    required: false
+    default: 'auto'
   read-only:
     description: >
       When 'true', restore the cache but do NOT save it (no post-job
@@ -36,26 +46,73 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - name: Install ccache
+    - name: Check for ccache
+      id: check
       shell: bash
       run: |
         if command -v ccache >/dev/null 2>&1; then
           echo "ccache already installed: $(ccache --version | head -1)"
-        elif [ "${{ runner.os }}" = "Linux" ]; then
-          sudo apt-get update -q
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            --no-install-recommends ccache
-        elif [ "${{ runner.os }}" = "macOS" ]; then
-          brew install ccache
+          echo "present=true" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+        echo "present=false" >> "$GITHUB_OUTPUT"
+        case "${{ runner.os }}" in
+          Linux|macOS) ;;
+          *) echo "::error::ccache install not supported on ${{ runner.os }}"
+             exit 1 ;;
+        esac
+        tag=""
+        if [ "${{ runner.os }}" = "Linux" ]; then
+          tag="${{ inputs.ghcr-debs-tag }}"
+          if [ "$tag" = "auto" ]; then
+            # os-release rather than lsb_release: no package needed.
+            . /etc/os-release
+            tag="ubuntu-$VERSION_ID-minimal"
+          elif [ "$tag" = "none" ]; then
+            tag=""
+          fi
+        fi
+        echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
+    # Offline from the ghcr bundle when it has ccache, apt otherwise -
+    # bounded either way. See install-apt-deps. ccache only makes the
+    # build faster, so the apt path gives up early and the job carries on
+    # without it rather than failing; the caps below are per attempt, so
+    # the wall is roughly their sum.
+    - name: Install ccache (Linux)
+      if: steps.check.outputs.present != 'true' && runner.os == 'Linux'
+      uses: ./.github/actions/install-apt-deps
+      with:
+        packages: ccache
+        no-install-recommends: 'true'
+        ghcr-debs-tag: ${{ steps.check.outputs.tag }}
+        apt-budget: '120'
+        apt-update-timeout: '120'
+        apt-install-timeout: '120'
+        optional: 'true'
+
+    - name: Install ccache (macOS)
+      if: steps.check.outputs.present != 'true' && runner.os == 'macOS'
+      shell: bash
+      run: brew install ccache || true
+
+    # Everything below needs a working ccache, so settle that once here
+    # rather than assuming the install worked.
+    - name: Confirm ccache is usable
+      id: have
+      shell: bash
+      run: |
+        if ccache --version >/dev/null 2>&1; then
+          echo "ccache=true" >> "$GITHUB_OUTPUT"
         else
-          echo "::error::ccache install not supported on ${{ runner.os }}"
-          exit 1
+          echo "ccache=false" >> "$GITHUB_OUTPUT"
+          echo "::warning::ccache unavailable; building without it"
         fi
 
     # read-only=false (default): restore + post-job save (the run_id in the
     # key never hits, so it always saves its contribution).
     - name: Restore + save ccache
-      if: inputs.read-only != 'true'
+      if: inputs.read-only != 'true' && steps.have.outputs.ccache == 'true'
       uses: actions/cache@v5
       with:
         path: ~/.ccache
@@ -68,7 +125,7 @@ runs:
           ccache-${{ inputs.workflow-id }}-${{ runner.os }}-${{ runner.arch }}-
     # read-only=true: restore the shared cache but never upload (PR runs).
     - name: Restore ccache (read-only)
-      if: inputs.read-only == 'true'
+      if: inputs.read-only == 'true' && steps.have.outputs.ccache == 'true'
       uses: actions/cache/restore@v5
       with:
         path: ~/.ccache
@@ -81,6 +138,7 @@ runs:
           ccache-${{ inputs.workflow-id }}-${{ runner.os }}-${{ runner.arch }}-
 
     - name: Configure ccache and PATH
+      if: steps.have.outputs.ccache == 'true'
       shell: bash
       run: |
         # Set CCACHE_DIR for the commands below too, not only via
@@ -118,10 +176,11 @@ runs:
     # (read-only is false on schedule), and PR/push runs are unaffected -
     # they keep their warm hits. Cost: the scheduled jobs recompile fully.
     - name: Force fresh compiles on scheduled reseed
-      if: github.event_name == 'schedule'
+      if: github.event_name == 'schedule' && steps.have.outputs.ccache == 'true'
       shell: bash
       run: echo "CCACHE_RECACHE=1" >> "$GITHUB_ENV"
 
     - name: Show ccache stats (initial)
+      if: steps.have.outputs.ccache == 'true'
       shell: bash
       run: ccache -s

.github/actions/install-apt-deps/action.yml

@@ -12,6 +12,33 @@ inputs:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
     default: '5'
+  apt-budget:
+    description: >
+      Wall-clock budget (seconds) for retrying the apt install.
+    required: false
+    default: '600'
+  apt-update-timeout:
+    description: >
+      Per-attempt cap (seconds) on apt-get update. The default covers a
+      full failover of every index file at the Acquire timeout below, so
+      a slow-but-recovering mirror is not cut short. Callers that can do
+      without the packages may prefer a smaller value (see 'optional').
+    required: false
+    default: '240'
+  apt-install-timeout:
+    description: >
+      Per-attempt cap (seconds) on apt-get install. The default suits the
+      large bundles; callers installing one small package should pass a
+      much smaller value.
+    required: false
+    default: '600'
+  optional:
+    description: >
+      When 'true', giving up warns instead of failing the step, so the
+      job continues without the packages. The caller is then responsible
+      for checking whether they actually arrived.
+    required: false
+    default: 'false'
   no-install-recommends:
     description: 'Pass --no-install-recommends to apt-get install'
     required: false
@@ -85,13 +112,34 @@ runs:
           NO_REC="--no-install-recommends"
         fi
 
+        # A stalled mirror connection returns no error, so the loop below
+        # cannot see it - apt just waits out its timeout once per index
+        # file, and the runner fetches around twenty of them. Keep that
+        # per-connection timeout short so failing over to the next mirror
+        # is cheap, cap each apt-get so a wedged one is killed, and stop
+        # retrying once the budget is gone.
+        APT_OPTS=(-o Acquire::Retries=2 -o Acquire::http::Timeout=10
+                  -o Acquire::https::Timeout=10)
+        # The budget stops further retries, but an attempt already under
+        # way is never interrupted, so the wall is the two caps below
+        # plus the budget - not the budget alone.
+        deadline=$((SECONDS + ${{ inputs.apt-budget }}))
+
         for i in $(seq 1 $RETRIES); do
-          if sudo apt-get update -q && \
-             sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
+          if sudo timeout -k 10 ${{ inputs.apt-update-timeout }} \
+               apt-get "${APT_OPTS[@]}" update -q && \
+             sudo timeout -k 10 ${{ inputs.apt-install-timeout }} \
+               apt-get "${APT_OPTS[@]}" install -y $NO_REC \
+               ${{ inputs.packages }}; then
             exit 0
           fi
-          if [ "$i" -eq "$RETRIES" ]; then
-            echo "::error::apt-get failed after $RETRIES attempts"
+          if [ "$i" -eq "$RETRIES" ] || [ "$SECONDS" -ge "$deadline" ]; then
+            if [ "${{ inputs.optional }}" = "true" ]; then
+              echo "::warning::apt-get failed after $i attempt(s);" \
+                   "continuing without ${{ inputs.packages }}"
+              exit 0
+            fi
+            echo "::error::apt-get failed after $i attempt(s)"
             exit 1
           fi
           echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."

.github/ci-deps/packages-ubuntu-22.04-minimal.txt

@@ -4,6 +4,7 @@
 autoconf
 automake
 build-essential
+ccache
 crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf

.github/workflows/fips-dev-no-post.yml

@@ -1,3 +1,3 @@
 name: FIPS dev-no-POST tests
 
 # --enable-fips=dev-no-post builds with FIPS-like settings but none of the fips
@@ -69,6 +69,9 @@
           workflow-id: fips-dev-no-post
           read-only: ${{ github.event_name == 'pull_request' }}
           max-size: 500M
+          # This job pulls the linuxkm bundle, not -minimal; fetching the
+          # latter just for ccache would cost more than the apt path.
+          ghcr-debs-tag: none
 
       - name: Prepare target kernel for module builds
         run: |