diff --git a/.github/image-matrix.json b/.github/image-matrix.json index 209e0bd..45c5020 100644 --- a/.github/image-matrix.json +++ b/.github/image-matrix.json @@ -1,9 +1,44 @@ [ - { "name": "postgres", "package": "postgres-image", "sbomify_component_id": "M8rixM6mMEPe" }, - { "name": "redis", "package": "redis-image", "sbomify_component_id": "ABBCcw2YiYrG" }, - { "name": "minio", "package": "minio-image", "sbomify_component_id": "PLACEHOLDER_MINIO" }, - { "name": "sbomify-app", "package": "sbomify-app-image", "sbomify_component_id": "VP42I4XQgpDE" }, - { "name": "sbomify-keycloak", "package": "sbomify-keycloak-image", "sbomify_component_id": "N4agQD8pvej8" }, - { "name": "sbomify-caddy-dev", "package": "sbomify-caddy-dev-image", "sbomify_component_id": "zYDq6NtrOBuo" }, - { "name": "sbomify-minio-init", "package": "sbomify-minio-init-image", "sbomify_component_id": "PLACEHOLDER_SBOMIFY_MINIO_INIT" } + { + "name": "postgres", + "package": "postgres-image", + "sbomify_component_id": "M8rixM6mMEPe", + "paths": ["common/images/postgres.nix"] + }, + { + "name": "redis", + "package": "redis-image", + "sbomify_component_id": "ABBCcw2YiYrG", + "paths": ["common/images/redis.nix"] + }, + { + "name": "minio", + "package": "minio-image", + "sbomify_component_id": "PLACEHOLDER_MINIO", + "paths": ["common/images/minio.nix"] + }, + { + "name": "sbomify-app", + "package": "sbomify-app-image", + "sbomify_component_id": "VP42I4XQgpDE", + "paths": ["apps/sbomify/images/sbomify-app.nix", "apps/sbomify/deployments/build-support/"] + }, + { + "name": "sbomify-keycloak", + "package": "sbomify-keycloak-image", + "sbomify_component_id": "N4agQD8pvej8", + "paths": ["apps/sbomify/images/sbomify-keycloak.nix"] + }, + { + "name": "sbomify-caddy-dev", + "package": "sbomify-caddy-dev-image", + "sbomify_component_id": "zYDq6NtrOBuo", + "paths": ["apps/sbomify/images/sbomify-caddy-dev.nix", "apps/sbomify/deployments/compose/Caddyfile.dev"] + }, + { + "name": "sbomify-minio-init", + "package": "sbomify-minio-init-image", + "sbomify_component_id": "PLACEHOLDER_SBOMIFY_MINIO_INIT", + "paths": ["apps/sbomify/images/sbomify-minio-init.nix"] + } ] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d07b2a3..54fe58a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,10 +36,35 @@ jobs: steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + run: | + # On workflow_dispatch, build everything + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "matrix=$(jq -c '[.[] | del(.paths)]' .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Get changed files from the PR + CHANGED_FILE="$(mktemp)" + gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ + --paginate --jq '.[].filename' > "$CHANGED_FILE" + + MATRIX=$(common/lib/scripts/filter-build-matrix .github/image-matrix.json "$CHANGED_FILE") + rm -f "$CHANGED_FILE" + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + - name: Write filtered matrix file + run: echo '${{ steps.set.outputs.matrix }}' > filtered-matrix.json + - name: Upload filtered matrix + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: filtered-matrix + path: filtered-matrix.json build: needs: load-matrix + if: ${{ needs.load-matrix.outputs.matrix != '[]' }} runs-on: blacksmith-4vcpu-ubuntu-2404 defaults: run: diff --git a/.github/workflows/cleanup-stale-images.yml b/.github/workflows/cleanup-stale-images.yml index ce6ce82..497a257 100644 --- a/.github/workflows/cleanup-stale-images.yml +++ b/.github/workflows/cleanup-stale-images.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(jq -c '[.[] | del(.paths)]' .github/image-matrix.json)" >> "$GITHUB_OUTPUT" weekly-sweep: needs: load-matrix diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 1a64a25..3fc83e7 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(jq -c '[.[] | del(.paths)]' .github/image-matrix.json)" >> "$GITHUB_OUTPUT" resolve-pr: runs-on: blacksmith-2vcpu-ubuntu-2404 diff --git a/.github/workflows/sbom-quality-gate.yml b/.github/workflows/sbom-quality-gate.yml index 5e57214..20b3e38 100644 --- a/.github/workflows/sbom-quality-gate.yml +++ b/.github/workflows/sbom-quality-gate.yml @@ -35,8 +35,14 @@ jobs: head_sha7: ${{ steps.sha.outputs.head_sha7 }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Download filtered matrix from build run + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: filtered-matrix + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} - id: set - run: echo "matrix=$(jq -c . .github/image-matrix.json)" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(jq -c . filtered-matrix.json)" >> "$GITHUB_OUTPUT" - id: sha run: | HEAD_SHA="${{ github.event.workflow_run.head_sha }}" @@ -44,6 +50,7 @@ jobs: score: needs: load-matrix + if: ${{ needs.load-matrix.outputs.matrix != '[]' }} runs-on: blacksmith-2vcpu-ubuntu-2404 defaults: run: diff --git a/common/lib/scripts/filter-build-matrix b/common/lib/scripts/filter-build-matrix new file mode 100755 index 0000000..9fc8115 --- /dev/null +++ b/common/lib/scripts/filter-build-matrix @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Filter the image build matrix to only images whose paths were changed. +# +# Usage: filter-build-matrix [--jq-cmd CMD] +# +# Arguments: +# matrix.json Path to the full image matrix JSON (array of objects with .name and .paths) +# changed-files Path to a file with one changed filepath per line +# +# The script checks changed files against two categories: +# 1. "Rebuild-all" paths — changes here trigger a full rebuild of every image: +# flake.nix, flake.lock, common/lib/*, common/pkgs/*, common/images/*, +# .github/workflows/build.yml, .github/image-matrix.json +# 2. Per-image paths — each matrix entry's .paths[] are matched as prefixes +# +# Output (stdout): filtered matrix JSON (array, .paths stripped). Empty array if nothing matched. +# Exit code: 0 always (empty array is valid — the caller decides what to do). + +jq_cmd="jq" + +while [ $# -gt 0 ]; do + case "$1" in + --jq-cmd) jq_cmd="$2"; shift 2 ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +if [ $# -lt 2 ]; then + echo "Usage: filter-build-matrix [--jq-cmd CMD] " >&2 + exit 1 +fi + +matrix_file="$1" +changed_file="$2" + +if [ ! -f "$matrix_file" ]; then + echo "Matrix file not found: $matrix_file" >&2 + exit 1 +fi + +if [ ! -f "$changed_file" ]; then + echo "Changed-files file not found: $changed_file" >&2 + exit 1 +fi + +FULL_MATRIX=$(<"$matrix_file") +CHANGED=$(<"$changed_file") + +# Check for paths that should trigger a full rebuild +REBUILD_ALL=false +while IFS= read -r file; do + [ -z "$file" ] && continue + case "$file" in + flake.nix|flake.lock) REBUILD_ALL=true ;; + common/lib/*|common/pkgs/*|common/images/*) REBUILD_ALL=true ;; + .github/workflows/build.yml|.github/image-matrix.json) REBUILD_ALL=true ;; + esac +done <<< "$CHANGED" + +if [ "$REBUILD_ALL" = true ]; then + echo "Rebuild-all path changed; building all images" >&2 + echo "$FULL_MATRIX" | "$jq_cmd" -c '[.[] | del(.paths)]' + exit 0 +fi + +# Filter to only images whose paths were touched +CHANGED_JSON=$(echo "$CHANGED" | "$jq_cmd" -R -s 'split("\n") | map(select(. != ""))') +FILTERED=$(echo "$FULL_MATRIX" | "$jq_cmd" -c --argjson changed "$CHANGED_JSON" ' + [.[] | select( + .paths as $paths | + any($changed[]; . as $file | + any($paths[]; . as $prefix | $file | startswith($prefix)) + ) + ) | del(.paths)] +') + +if [ "$FILTERED" = "[]" ]; then + echo "No image paths changed; nothing to build" >&2 +else + echo "Building filtered images: $(echo "$FILTERED" | "$jq_cmd" -r '[.[].name] | join(", ")')" >&2 +fi + +echo "$FILTERED" diff --git a/common/lib/tests/filter_build_matrix_spec.sh b/common/lib/tests/filter_build_matrix_spec.sh new file mode 100644 index 0000000..7ec6e4f --- /dev/null +++ b/common/lib/tests/filter_build_matrix_spec.sh @@ -0,0 +1,184 @@ +# shellcheck shell=sh +Describe "common/lib/scripts/filter-build-matrix" + setup() { + MATRIX_FILE="$(mktemp)" + CHANGED_FILE="$(mktemp)" + # Use app-level paths for per-image filtering tests (common/images/* triggers rebuild-all) + cat > "$MATRIX_FILE" <<'JSON' +[ + {"name":"sbomify-app","package":"sbomify-app-image","paths":["apps/sbomify/images/sbomify-app.nix","apps/sbomify/deployments/build-support/"]}, + {"name":"sbomify-keycloak","package":"sbomify-keycloak-image","paths":["apps/sbomify/images/sbomify-keycloak.nix"]}, + {"name":"postgres","package":"postgres-image","paths":["common/images/postgres.nix"]} +] +JSON + } + cleanup() { + rm -f "$MATRIX_FILE" "$CHANGED_FILE" + } + Before "setup" + After "cleanup" + + Describe "argument validation" + It "fails when no arguments are given" + When run common/lib/scripts/filter-build-matrix + The status should be failure + The stderr should include "Usage" + End + + It "fails when only one argument is given" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" + The status should be failure + The stderr should include "Usage" + End + + It "fails when matrix file does not exist" + When run common/lib/scripts/filter-build-matrix /nonexistent/matrix.json "$CHANGED_FILE" + The status should be failure + The stderr should include "not found" + End + + It "fails when changed-files file does not exist" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" /nonexistent/changed.txt + The status should be failure + The stderr should include "not found" + End + End + + Describe "rebuild-all triggers" + It "rebuilds all when flake.nix is changed" + echo "flake.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The output should include '"sbomify-app"' + The output should include '"sbomify-keycloak"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when flake.lock is changed" + echo "flake.lock" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when common/lib/ file is changed" + echo "common/lib/scripts/build-and-push" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The output should include '"sbomify-app"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when common/pkgs/ file is changed" + echo "common/pkgs/something.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when common/images/ file is changed" + echo "common/images/base.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when build.yml is changed" + echo ".github/workflows/build.yml" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "rebuilds all when image-matrix.json is changed" + echo ".github/image-matrix.json" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The stderr should include "Rebuild-all" + End + + It "strips .paths from output on rebuild-all" + echo "flake.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should not include '"paths"' + The stderr should include "Rebuild-all" + End + End + + Describe "per-image path filtering" + It "returns only the image whose path matched" + echo "apps/sbomify/images/sbomify-app.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"sbomify-app"' + The output should not include '"sbomify-keycloak"' + The output should not include '"postgres"' + The stderr should include "Building filtered" + End + + It "returns multiple images when multiple paths match" + printf "apps/sbomify/images/sbomify-app.nix\napps/sbomify/images/sbomify-keycloak.nix\n" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"sbomify-app"' + The output should include '"sbomify-keycloak"' + The output should not include '"postgres"' + The stderr should include "Building filtered" + End + + It "matches prefix paths (directory-level paths)" + echo "apps/sbomify/deployments/build-support/Dockerfile" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"sbomify-app"' + The output should not include '"postgres"' + The stderr should include "Building filtered" + End + + It "strips .paths from filtered output" + echo "apps/sbomify/images/sbomify-app.nix" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should not include '"paths"' + The stderr should include "Building filtered" + End + End + + Describe "no matches" + It "returns empty array when no paths match" + echo "README.md" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should equal "[]" + The stderr should include "No image paths changed" + End + + It "returns empty array for empty changed-files" + : > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should equal "[]" + The stderr should include "No image paths changed" + End + End + + Describe "rebuild-all takes precedence" + It "rebuilds all even when a specific app path also matches" + printf "flake.nix\napps/sbomify/images/sbomify-app.nix\n" > "$CHANGED_FILE" + When run common/lib/scripts/filter-build-matrix "$MATRIX_FILE" "$CHANGED_FILE" + The status should be success + The output should include '"postgres"' + The output should include '"sbomify-app"' + The output should include '"sbomify-keycloak"' + The stderr should include "Rebuild-all" + End + End +End