From 81aceb232c0cee6b248da1f7d69a3037b791b1c3 Mon Sep 17 00:00:00 2001 From: Yakira Date: Tue, 31 Mar 2026 20:38:55 +0100 Subject: [PATCH 1/2] ci: skip unchanged images on PR builds with path-based filtering The load-matrix job now filters the build matrix by comparing PR changed files against each image's declared source paths. Single-image PRs only build the affected image; flake/common/workflow changes still trigger a full rebuild of all images. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/image-matrix.json | 49 +++++++++++++++--- .github/workflows/build.yml | 58 +++++++++++++++++++++- .github/workflows/cleanup-stale-images.yml | 2 +- .github/workflows/pre-release.yml | 2 +- .github/workflows/sbom-quality-gate.yml | 9 +++- 5 files changed, 109 insertions(+), 11 deletions(-) diff --git a/.github/image-matrix.json b/.github/image-matrix.json index 209e0bd..45c5020 100644 --- a/.github/image-matrix.json +++ b/.github/image-matrix.json @@ -1,9 +1,44 @@ [ - { "name": "postgres", "package": "postgres-image", "sbomify_component_id": "M8rixM6mMEPe" }, - { "name": "redis", "package": "redis-image", "sbomify_component_id": "ABBCcw2YiYrG" }, - { "name": "minio", "package": "minio-image", "sbomify_component_id": "PLACEHOLDER_MINIO" }, - { "name": "sbomify-app", "package": "sbomify-app-image", "sbomify_component_id": "VP42I4XQgpDE" }, - { "name": "sbomify-keycloak", "package": "sbomify-keycloak-image", "sbomify_component_id": "N4agQD8pvej8" }, - { "name": "sbomify-caddy-dev", "package": "sbomify-caddy-dev-image", "sbomify_component_id": "zYDq6NtrOBuo" }, - { "name": "sbomify-minio-init", "package": "sbomify-minio-init-image", "sbomify_component_id": "PLACEHOLDER_SBOMIFY_MINIO_INIT" } + { + "name": "postgres", + "package": "postgres-image", + "sbomify_component_id": "M8rixM6mMEPe", + "paths": ["common/images/postgres.nix"] + }, + { + "name": "redis", + "package": "redis-image", + "sbomify_component_id": "ABBCcw2YiYrG", + "paths": ["common/images/redis.nix"] + }, + { + "name": "minio", + "package": "minio-image", + "sbomify_component_id": "PLACEHOLDER_MINIO", + "paths": ["common/images/minio.nix"] + }, + { + "name": "sbomify-app", + "package": "sbomify-app-image", + "sbomify_component_id": "VP42I4XQgpDE", + "paths": ["apps/sbomify/images/sbomify-app.nix", "apps/sbomify/deployments/build-support/"] + }, + { + "name": "sbomify-keycloak", + "package": "sbomify-keycloak-image", + "sbomify_component_id": "N4agQD8pvej8", + "paths": ["apps/sbomify/images/sbomify-keycloak.nix"] + }, + { + "name": "sbomify-caddy-dev", + "package": "sbomify-caddy-dev-image", + "sbomify_component_id": "zYDq6NtrOBuo", + "paths": ["apps/sbomify/images/sbomify-caddy-dev.nix", "apps/sbomify/deployments/compose/Caddyfile.dev"] + }, + { + "name": "sbomify-minio-init", + "package": "sbomify-minio-init-image", + "sbomify_component_id": "PLACEHOLDER_SBOMIFY_MINIO_INIT", + "paths": ["apps/sbomify/images/sbomify-minio-init.nix"] + } ] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d07b2a3..824696f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,10 +36,66 @@ jobs: steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + run: | + FULL_MATRIX=$(cat .github/image-matrix.json) + + # On workflow_dispatch, build everything + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "matrix=$(echo "$FULL_MATRIX" | jq -c '[.[] | del(.paths)]')" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Get changed files from the PR + CHANGED=$(gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ + --paginate --jq '.[].filename') + + # Check for paths that should trigger a full rebuild + REBUILD_ALL=false + while IFS= read -r file; do + case "$file" in + flake.nix|flake.lock) REBUILD_ALL=true ;; + common/lib/*|common/pkgs/*|common/images/*) REBUILD_ALL=true ;; + .github/workflows/build.yml|.github/image-matrix.json) REBUILD_ALL=true ;; + esac + done <<< "$CHANGED" + + if [ "$REBUILD_ALL" = true ]; then + echo "Rebuild-all path changed; building all images" + echo "matrix=$(echo "$FULL_MATRIX" | jq -c '[.[] | del(.paths)]')" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Filter to only images whose paths were touched + CHANGED_JSON=$(echo "$CHANGED" | jq -R -s 'split("\n") | map(select(. != ""))') + FILTERED=$(echo "$FULL_MATRIX" | jq -c --argjson changed "$CHANGED_JSON" ' + [.[] | select( + .paths as $paths | + any($changed[]; . as $file | + any($paths[]; . as $prefix | $file | startswith($prefix)) + ) + ) | del(.paths)] + ') + + if [ "$FILTERED" = "[]" ]; then + echo "No image paths changed; nothing to build" + else + echo "Building filtered images: $(echo "$FILTERED" | jq -r '[.[].name] | join(", ")')" + fi + echo "matrix=$FILTERED" >> "$GITHUB_OUTPUT" + + - name: Write filtered matrix file + run: echo '${{ steps.set.outputs.matrix }}' > filtered-matrix.json + - name: Upload filtered matrix + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: filtered-matrix + path: filtered-matrix.json build: needs: load-matrix + if: ${{ needs.load-matrix.outputs.matrix != '[]' }} runs-on: blacksmith-4vcpu-ubuntu-2404 defaults: run: diff --git a/.github/workflows/cleanup-stale-images.yml b/.github/workflows/cleanup-stale-images.yml index ce6ce82..497a257 100644 --- a/.github/workflows/cleanup-stale-images.yml +++ b/.github/workflows/cleanup-stale-images.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(jq -c '[.[] | del(.paths)]' .github/image-matrix.json)" >> "$GITHUB_OUTPUT" weekly-sweep: needs: load-matrix diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 1a64a25..3fc83e7 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(jq -c '[.[] | del(.paths)]' .github/image-matrix.json)" >> "$GITHUB_OUTPUT" resolve-pr: runs-on: blacksmith-2vcpu-ubuntu-2404 diff --git a/.github/workflows/sbom-quality-gate.yml b/.github/workflows/sbom-quality-gate.yml index 5e57214..20b3e38 100644 --- a/.github/workflows/sbom-quality-gate.yml +++ b/.github/workflows/sbom-quality-gate.yml @@ -35,8 +35,14 @@ jobs: head_sha7: ${{ steps.sha.outputs.head_sha7 }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Download filtered matrix from build run + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: filtered-matrix + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(jq -c . filtered-matrix.json)" >> "$GITHUB_OUTPUT" - id: sha run: | HEAD_SHA="${{ github.event.workflow_run.head_sha }}" @@ -44,6 +50,7 @@ jobs: score: needs: load-matrix + if: ${{ needs.load-matrix.outputs.matrix != '[]' }} runs-on: blacksmith-2vcpu-ubuntu-2404 defaults: run: From f1677a407d2504f6565f3bdc311b3b9f3eaf0e5e Mon Sep 17 00:00:00 2001 From: Yakira Date: Tue, 31 Mar 2026 22:17:46 +0100 Subject: [PATCH 2/2] refactor: extract PR matrix filtering into testable script Move the inline changed-file detection and matrix filtering logic from build.yml into common/lib/scripts/filter-build-matrix with ShellSpec tests, following the existing script+test pattern. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/build.yml | 45 +---- common/lib/scripts/filter-build-matrix | 86 +++++++++ common/lib/tests/filter_build_matrix_spec.sh | 184 +++++++++++++++++++ 3 files changed, 277 insertions(+), 38 deletions(-) create mode 100755 common/lib/scripts/filter-build-matrix create mode 100644 common/lib/tests/filter_build_matrix_spec.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 824696f..54fe58a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,51 +39,20 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - FULL_MATRIX=$(cat .github/image-matrix.json) - # On workflow_dispatch, build everything if [ "${{ github.event_name }}" != "pull_request" ]; then - echo "matrix=$(echo "$FULL_MATRIX" | jq -c '[.[] | del(.paths)]')" >> "$GITHUB_OUTPUT" + echo "matrix=$(jq -c '[.[] | del(.paths)]' .github/image-matrix.json)" >> "$GITHUB_OUTPUT" exit 0 fi # Get changed files from the PR - CHANGED=$(gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ - --paginate --jq '.[].filename') - - # Check for paths that should trigger a full rebuild - REBUILD_ALL=false - while IFS= read -r file; do - case "$file" in - flake.nix|flake.lock) REBUILD_ALL=true ;; - common/lib/*|common/pkgs/*|common/images/*) REBUILD_ALL=true ;; - .github/workflows/build.yml|.github/image-matrix.json) REBUILD_ALL=true ;; - esac - done <<< "$CHANGED" - - if [ "$REBUILD_ALL" = true ]; then - echo "Rebuild-all path changed; building all images" - echo "matrix=$(echo "$FULL_MATRIX" | jq -c '[.[] | del(.paths)]')" >> "$GITHUB_OUTPUT" - exit 0 - fi + CHANGED_FILE="$(mktemp)" + gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ + --paginate --jq '.[].filename' > "$CHANGED_FILE" - # Filter to only images whose paths were touched - CHANGED_JSON=$(echo "$CHANGED" | jq -R -s 'split("\n") | map(select(. != ""))') - FILTERED=$(echo "$FULL_MATRIX" | jq -c --argjson changed "$CHANGED_JSON" ' - [.[] | select( - .paths as $paths | - any($changed[]; . as $file | - any($paths[]; . as $prefix | $file | startswith($prefix)) - ) - ) | del(.paths)] - ') - - if [ "$FILTERED" = "[]" ]; then - echo "No image paths changed; nothing to build" - else - echo "Building filtered images: $(echo "$FILTERED" | jq -r '[.[].name] | join(", ")')" - fi - echo "matrix=$FILTERED" >> "$GITHUB_OUTPUT" + MATRIX=$(common/lib/scripts/filter-build-matrix .github/image-matrix.json "$CHANGED_FILE") + rm -f "$CHANGED_FILE" + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - name: Write filtered matrix file run: echo '${{ steps.set.outputs.matrix }}' > filtered-matrix.json diff --git a/common/lib/scripts/filter-build-matrix b/common/lib/scripts/filter-build-matrix new file mode 100755 index 0000000..9fc8115 --- /dev/null +++ b/common/lib/scripts/filter-build-matrix @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Filter the image build matrix to only images whose paths were changed. +# +# Usage: filter-build-matrix [--jq-cmd CMD] +# +# Arguments: +# matrix.json Path to the full image matrix JSON (array of objects with .name and .paths) +# changed-files Path to a file with one changed filepath per line +# +# The script checks changed files against two categories: +# 1. "Rebuild-all" paths — changes here trigger a full rebuild of every image: +# flake.nix, flake.lock, common/lib/*, common/pkgs/*, common/images/*, +# .github/workflows/build.yml, .github/image-matrix.json +# 2. Per-image paths — each matrix entry's .paths[] are matched as prefixes +# +# Output (stdout): filtered matrix JSON (array, .paths stripped). Empty array if nothing matched. +# Exit code: 0 always (empty array is valid — the caller decides what to do). + +jq_cmd="jq" + +while [ $# -gt 0 ]; do + case "$1" in + --jq-cmd) jq_cmd="$2"; shift 2 ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +if [ $# -lt 2 ]; then + echo "Usage: filter-build-matrix [--jq-cmd CMD] " >&2 + exit 1 +fi + +matrix_file="$1" +changed_file="$2" + +if [ ! -f "$matrix_file" ]; then + echo "Matrix file not found: $matrix_file" >&2 + exit 1 +fi + +if [ ! -f "$changed_file" ]; then + echo "Changed-files file not found: $changed_file" >&2 + exit 1 +fi + +FULL_MATRIX=$(<"$matrix_file") +CHANGED=$(<"$changed_file") + +# Check for paths that should trigger a full rebuild +REBUILD_ALL=false +while IFS= read -r file; do + [ -z "$file" ] && continue + case "$file" in + flake.nix|flake.lock) REBUILD_ALL=true ;; + common/lib/*|common/pkgs/*|common/images/*) REBUILD_ALL=true ;; + .github/workflows/build.yml|.github/image-matrix.json) REBUILD_ALL=true ;; + esac +done <<< "$CHANGED" + +if [ "$REBUILD_ALL" = true ]; then + echo "Rebuild-all path changed; building all images" >&2 + echo "$FULL_MATRIX" | "$jq_cmd" -c '[.[] | del(.paths)]' + exit 0 +fi + +# Filter to only images whose paths were touched +CHANGED_JSON=$(echo "$CHANGED" | "$jq_cmd" -R -s 'split("\n") | map(select(. != ""))') +FILTERED=$(echo "$FULL_MATRIX" | "$jq_cmd" -c --argjson changed "$CHANGED_JSON" ' + [.[] | select( + .paths as $paths | + any($changed[]; . as $file | + any($paths[]; . as $prefix | $file | startswith($prefix)) + ) + ) | del(.paths)] +') + +if [ "$FILTERED" = "[]" ]; then + echo "No image paths changed; nothing to build" >&2 +else + echo "Building filtered images: $(echo "$FILTERED" | "$jq_cmd" -r '[.[].name] | join(", ")')" >&2 +fi + +echo "$FILTERED" diff --git a/common/lib/tests/filter_build_matrix_spec.sh b/common/lib/tests/filter_build_matrix_spec.sh new file mode 100644 index 0000000..7ec6e4f --- /dev/null +++ b/common/lib/tests/filter_build_matrix_spec.sh @@ -0,0 +1,184 @@ +# shellcheck shell=sh +Describe "common/lib/scripts/filter-build-matrix" + setup() { + MATRIX_FILE="$(mktemp)" + CHANGED_FILE="$(mktemp)" + # Use app-level paths for per-image filtering tests (common/images/* triggers rebuild-all) + cat > "$MATRIX_FILE" <<'JSON' +[ + {"name":"sbomify-app","package":"sbomify-app-image","paths":["apps/sbomify/images/sbomify-app.nix","apps/sbomify/deployments/build-support/"]}, + {"name":"sbomify-keycloak","package":"sbomify-keycloak-image","paths":["apps/sbomify/images/sbomify-keycloak.nix"]}, + {"name":"postgres","package":"postgres-image","paths":["common/images/postgres.nix"]} +] +JSON + } + cleanup() { + rm -f "$MATRIX_FILE" "$CHANGED_FILE" + } + Before "setup" + After "cleanup" + + Describe "argument validation" + It "fails when no arguments are given" + When run common/lib/scripts/filter-build-matrix + The status should be failure + The stderr should include "Usage" + End + + It "fails when only one argument is given" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" + The status should be failure + The stderr should include "Usage" + End + + It "fails when matrix file does not exist" + When run common/lib/scripts/filter-build-matrix /nonexistent/matrix.json "$CHANGED_FILE" + The status should be failure + The stderr should include "not found" + End + + It "fails when changed-files file does not exist" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" /nonexistent/changed.txt + The status should be failure + The stderr should include "not found" + End + End + + Describe "rebuild-all triggers" + It "rebuilds all when flake.nix is changed" + echo "flake.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The output should include '"sbomify-app"' + The output should include '"sbomify-keycloak"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when flake.lock is changed" + echo "flake.lock" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when common/lib/ file is changed" + echo "common/lib/scripts/build-and-push" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The output should include '"sbomify-app"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when common/pkgs/ file is changed" + echo "common/pkgs/something.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when common/images/ file is changed" + echo "common/images/base.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when build.yml is changed" + echo ".github/workflows/build.yml" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when image-matrix.json is changed" + echo ".github/image-matrix.json" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "strips .paths from output on rebuild-all" + echo "flake.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should not include '"paths"' + The stderr should include "Rebuild-all" + End + End + + Describe "per-image path filtering" + It "returns only the image whose path matched" + echo "apps/sbomify/images/sbomify-app.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"sbomify-app"' + The output should not include '"sbomify-keycloak"' + The output should not include '"postgres"' + The stderr should include "Building filtered" + End + + It "returns multiple images when multiple paths match" + printf "apps/sbomify/images/sbomify-app.nix\napps/sbomify/images/sbomify-keycloak.nix\n" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"sbomify-app"' + The output should include '"sbomify-keycloak"' + The output should not include '"postgres"' + The stderr should include "Building filtered" + End + + It "matches prefix paths (directory-level paths)" + echo "apps/sbomify/deployments/build-support/Dockerfile" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"sbomify-app"' + The output should not include '"postgres"' + The stderr should include "Building filtered" + End + + It "strips .paths from filtered output" + echo "apps/sbomify/images/sbomify-app.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should not include '"paths"' + The stderr should include "Building filtered" + End + End + + Describe "no matches" + It "returns empty array when no paths match" + echo "README.md" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should equal "[]" + The stderr should include "No image paths changed" + End + + It "returns empty array for empty changed-files" + : > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should equal "[]" + The stderr should include "No image paths changed" + End + End + + Describe "rebuild-all takes precedence" + It "rebuilds all even when a specific app path also matches" + printf "flake.nix\napps/sbomify/images/sbomify-app.nix\n" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The output should include '"sbomify-app"' + The output should include '"sbomify-keycloak"' + The stderr should include "Rebuild-all" + End + End +End