Skip to content
CI - Build & Test

fix(exchange): alarm on stale processing RI exchanges and let an operator resolve them #1261

Sign in to view logs

fix(exchange): alarm on stale processing RI exchanges and let an operator resolve them

fix(exchange): alarm on stale processing RI exchanges and let an operator resolve them #1261

Workflow file for this run

.github/workflows/ci.yml at 89c1363
# CI Workflow - Build, Test, Lint, Security
#
# This workflow runs on every pull request and push to main/develop branches.
# It performs comprehensive quality checks before allowing code to be merged.
#
# Required GitHub Secrets: None (all checks run without cloud credentials)
#
# Required GitHub Variables:
# - GO_VERSION: Go version to use (default: 1.26.9)
#
# Triggered by:
# - Pull requests to main/develop
# - Pushes to main/develop
# - Manual workflow dispatch
name: CI - Build & Test
permissions:
contents: read
on:
pull_request:
branches: [main, develop]
push:
branches: [main, develop]
workflow_dispatch:
env:
GO_VERSION: '1.26.9'
DOCKER_BUILDKIT: 1
COMPOSE_DOCKER_CLI_BUILD: 1
jobs:
# Go code linting
lint:
name: Lint Code
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Install golangci-lint
uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1
with:
version: v2.10.1
install-only: true
- name: Verify golangci-lint config against vendored schema
# `golangci-lint config verify` downloads its JSON schema from
# golangci-lint.run by default, with a hard-coded 2s HTTP timeout, and
# that fetch timing out failed this job at random (#977). The schema is
# vendored instead and passed through the (hidden) --schema flag, so
# the config is still validated on every run, with no network access.
#
# Vendored file: .github/golangci-lint/golangci.v<major>.<minor>.jsonschema.json
# source: https://golangci-lint.run/jsonschema/golangci.v2.10.jsonschema.json
# (byte-identical to jsonschema/golangci.next.jsonschema.json
# at golangci-lint tag v2.10.1, the copy this file came from)
# sha256: GOLANGCI_LINT_SCHEMA_SHA256 below
#
# When bumping `version:` above to a new minor, this step fails until
# the matching schema is vendored. To update: download the URL above
# with the new minor, save it under the new file name, delete the old
# one, and set GOLANGCI_LINT_SCHEMA_SHA256 to its sha256sum.
env:
GOLANGCI_LINT_SCHEMA_SHA256: '692e8da79dc1f110a4a4f3bb841b26ed44e461d3503678eaaae90ae8876e3403'
run: |
set -euo pipefail
version=$(golangci-lint version --short)
minor="${version#v}"
minor="${minor%.*}"
schema=".github/golangci-lint/golangci.v${minor}.jsonschema.json"
if [ ! -f "$schema" ]; then
echo "::error::no vendored schema for golangci-lint ${version}: expected ${schema} (see the update steps in ci.yml)"
exit 1
fi
echo "${GOLANGCI_LINT_SCHEMA_SHA256} ${schema}" | sha256sum -c -
# One check covers both modules: tests/e2e has no config of its own,
# and golangci-lint finds the root .golangci.yml from there too.
golangci-lint config verify --schema "file://${PWD}/${schema}"
- name: Run golangci-lint
run: |
set -uo pipefail
export GOLANGCI_LINT_CACHE="$HOME/.cache/golangci-lint"
status=0
for mod in . tests/e2e; do
echo "==> golangci-lint in $mod"
args=(run --timeout=10m)
# The standalone e2e module has only e2e-tagged test files.
if [ "$mod" = "tests/e2e" ]; then
args+=(--build-tags=e2e)
fi
if ! (cd "$mod" && golangci-lint "${args[@]}" ./...); then
status=1
fi
done
exit "$status"
- name: Run go vet
run: |
set -uo pipefail
status=0
for mod in . tests/e2e; do
echo "==> go vet in $mod"
args=(vet)
if [ "$mod" = "tests/e2e" ]; then
args+=(-tags=e2e)
fi
if ! (cd "$mod" && go "${args[@]}" ./...); then
status=1
fi
done
exit "$status"
- name: Install gocyclo
run: go install github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0
- name: Check cyclomatic complexity
run: |
echo "Checking for functions with cyclomatic complexity over 10..."
# Exclude _test.go files to stay consistent with .golangci.yml, which
# excludes gocyclo on test files (test helpers/table-driven tests are
# allowed higher complexity). Production code is still gated at >10.
COMPLEXITY_ISSUES=$(gocyclo -over 10 -ignore "_test\.go" . 2>&1 || true)
if [ -n "$COMPLEXITY_ISSUES" ]; then
echo "❌ Found functions with cyclomatic complexity over 10:"
echo "$COMPLEXITY_ISSUES"
echo ""
echo "⚠️ Please refactor these functions to reduce complexity."
echo "📖 Tip: Extract helper functions, use early returns, or simplify logic."
exit 1
fi
echo "✅ All functions have acceptable cyclomatic complexity (≤10)"
# GitHub Actions workflow linting
#
# Nothing else in this repo reads .github/workflows/ for defects. govulncheck
# and gosec are Go source scanners, trivy-config targets Terraform/Dockerfile/
# Kubernetes, and check-yaml only proves the YAML parses. That gap is why the
# rollback.yml and deploy-aws-lambda.yml expression injections (#1542, #1649),
# both reaching production cloud credentials, passed every CI run.
#
# Both linters are needed and neither substitutes for the other:
# - actionlint catches workflow-level defects and, via shellcheck, shell
# bugs inside run: blocks.
# - zizmor has a template-injection audit that names the injection itself.
# Measured against the pre-fix rollback.yml, actionlint exited 1 only on
# unrelated SC2086 noise and never flagged the injected heredoc at all;
# zizmor flagged that exact line high severity, high confidence. actionlint
# alone would not have caught the bug this job exists to prevent.
workflow-lint:
name: Lint Workflows
runs-on: ubuntu-latest
permissions:
contents: read
env:
# Pinned by digest, not only by tag. A tag is mutable, and a linter whose
# ruleset changes without a change in this repo turns main red on its own
# schedule -- the hadolint :latest failure in #1695.
ACTIONLINT_IMAGE: 'rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667'
ZIZMOR_VERSION: '1.29.0'
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Assert there are workflows to lint
run: |
set -euo pipefail
count=$(find .github/workflows -maxdepth 1 -type f \( -name '*.yml' -o -name '*.yaml' \) | wc -l | tr -d ' ')
# Defence in depth, not the only guard: both linters do exit 3 on an
# empty input set. This catches the case they cannot, where the path
# still resolves but the set silently shrinks, and it prints the count
# so a drop is visible in the log rather than inferred from silence.
if [ "$count" -eq 0 ]; then
echo "::error::no workflow files found under .github/workflows"
exit 1
fi
echo "Linting $count workflow files"
- name: Assert shellcheck is available to actionlint
# actionlint does not fail when shellcheck is missing from PATH: it
# silently skips every run: block and still exits 0. That silent skip is
# the failure mode this job exists to close, so assert the binary is
# present rather than trusting the image to keep bundling it.
run: |
set -euo pipefail
docker run --rm --entrypoint sh "$ACTIONLINT_IMAGE" -c '
command -v shellcheck >/dev/null || {
echo "::error::shellcheck is not present in the actionlint image; shell linting would be silently skipped"
exit 1
}
shellcheck --version | sed -n "1,3p"'
- name: Run actionlint
# No file arguments: actionlint discovers .github/workflows itself, so
# it also covers .yaml files and any workflow added later. Passing an
# explicit *.yml glob would silently skip a .yaml workflow that the
# count step above still counts.
run: |
set -euo pipefail
docker run --rm -v "$PWD:/repo" -w /repo "$ACTIONLINT_IMAGE" -color
- name: Run zizmor
# --offline on purpose: the online audits query the GitHub API for action
# metadata, so findings could change without a change in this repo and
# redden main, the same class of failure the hadolint digest pin fixed.
#
# Coverage note: zizmor reads the directory non-recursively, while
# actionlint walks it. A workflow under .github/workflows/sub/ would
# therefore reach actionlint but not zizmor. GitHub itself ignores
# workflows in subdirectories, so this is not a live hole, and the
# -maxdepth 1 count above fails loud if the set ever moves down a level.
#
# Two independent filters, and it matters which does what.
#
# --persona=pedantic rather than the default regular: regular hides
# three high-severity findings this repo actually had (workflow-level
# id-token: write in both sanity workflows, and an unpinned postgres
# service image). Those are fixed rather than filtered, so the stricter
# persona costs nothing today and gates more. auditor is the one level
# up and is documented as accepting false positives, so it is not used.
#
# --min-severity=medium is a threshold, not a suppression: no baseline
# file, no per-finding ignore, no only-new-issues. Every medium and
# high finding the pedantic persona surfaces fails this job, and the
# injection class this job exists to catch scores high. Below the line
# sit 106 findings, none above low: 66 template-injection on values
# #1649 already assessed as non-injectable (github.actor, github.sha
# and similar), plus undocumented-permissions, concurrency-limits and
# anonymous-definition. Clearing those means rewriting the deploy
# workflows, so they are left to a follow-up rather than silenced here.
run: |
set -euo pipefail
# `pipx run --spec` rather than `pipx install`: it pins the version in
# the same statement that invokes it and does not assume pipx's bin
# directory is on PATH.
pipx run --spec "zizmor==${ZIZMOR_VERSION}" zizmor \
--offline --persona=pedantic --min-severity=medium \
--color=always .github/workflows/
# Unit tests with race detection
unit-tests:
name: Unit Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Download dependencies
run: |
# Verify both platform-owned modules; libraries have their own CI.
set -e
for mod in . tests/e2e; do
echo "==> go mod download/verify in $mod"
(cd "$mod" && go mod download && go mod verify)
done
- name: Run unit tests (all modules)
run: |
# Each ./... walks one module. Check both platform-owned modules.
#
# A failing module must not abort the loop. Aborting would hide every
# later module's result behind the first failure, which is the same
# shape as the gosec bug in issue #1717. Guard each run, record the
# status, and fail at the end so every module is reported.
set -uo pipefail
status=0
for mod in . tests/e2e; do
tag=$(echo "$mod" | tr './' '--' | sed 's/^-/root/')
log="$RUNNER_TEMP/unit-${tag}.log"
# tests/e2e holds only //go:build e2e files, so `go test ./...`
# there matches no packages and exits 1 -- it cannot be run like the
# others. Its tests need the running stack and are executed by the
# e2e-tests job over docker compose; what this job can add is a
# type-check under that tag, which nothing else here does. Handled
# by name, not by a pattern, so the difference is visible in review.
if [ "$mod" = "tests/e2e" ]; then
echo "==> type-check $mod under -tags=e2e (tests run in the e2e-tests job)"
if ! (cd "$mod" && go vet -tags=e2e ./...) 2>&1 | tee "$log"; then
echo "::error::$mod failed to type-check under -tags=e2e"
status=1
fi
continue
fi
echo "==> unit tests in $mod"
# Profiles go to $RUNNER_TEMP, not the checkout: never leave working
# files in the repository root (repo coding guideline).
if ! (cd "$mod" && go test -v -race -short \
-coverprofile="$RUNNER_TEMP/coverage-${tag}.out" \
-covermode=atomic ./...) 2>&1 | tee "$log"; then
echo "::error::unit tests failed in $mod"
status=1
fi
# A module that runs zero tests is issue #1751 wearing a new
# costume: the loop would "cover" it while asserting nothing. Count
# top-level results (subtest lines are indented) and fail loudly.
ran=$(grep -cE '^--- (PASS|FAIL|SKIP)' "$log" || true)
echo "==> $mod ran $ran top-level test(s)"
if [ "$ran" -eq 0 ]; then
echo "::error::$mod ran zero tests -- it is in the loop but asserting nothing"
status=1
fi
done
# Merge the per-module profiles into the single file the steps below
# expect. A Go profile is one "mode:" header followed by block lines,
# so keep one header and concatenate the bodies.
shopt -s nullglob
profiles=("$RUNNER_TEMP"/coverage-*.out)
if [ "${#profiles[@]}" -eq 0 ]; then
echo "::error::no module produced a coverage profile" >&2
exit 1
fi
# Take the mode header from the first profile rather than hardcoding
# it: go test picks the default covermode itself (atomic whenever
# -race is on), so a literal here would silently mislabel the merge if
# the flags change.
merged="$RUNNER_TEMP/coverage.out"
head -n 1 "${profiles[0]}" > "$merged"
for p in "${profiles[@]}"; do
tail -n +2 "$p" >> "$merged"
done
echo "Merged ${#profiles[@]} coverage profile(s), $(( $(wc -l < "$merged") - 1 )) block(s)"
exit "$status"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
with:
files: ${{ runner.temp }}/coverage.out
flags: unittests
name: codecov-umbrella
fail_ci_if_error: false
- name: Generate coverage report
run: |
go tool cover -html="$RUNNER_TEMP/coverage.out" -o "$RUNNER_TEMP/coverage.html"
- name: Upload coverage artifacts
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: coverage-report
path: |
${{ runner.temp }}/coverage.out
${{ runner.temp }}/coverage.html
retention-days: 30
- name: Check coverage threshold
run: |
coverage=$(go tool cover -func="$RUNNER_TEMP/coverage.out" | grep total | awk '{print $3}' | sed 's/%//')
echo "Total coverage: ${coverage}%"
if (( $(echo "$coverage < 80" | bc -l) )); then
echo "::warning::Coverage is below 80% (current: ${coverage}%)"
fi
# Integration tests with real PostgreSQL
integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
permissions:
contents: read
services:
postgres:
# Pinned by digest for the same reason as the linter images below: a
# floating tag lets the service container change under an unchanged
# repo, which is how #1695 turned main red.
image: postgres:16-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685
env:
POSTGRES_DB: cudly_test
POSTGRES_USER: cudly_test
POSTGRES_PASSWORD: test_password # CI-only throwaway password — not used in any real environment
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Install golang-migrate
run: |
# -tags 'pgx5', matching the Dockerfile and the library import in
# internal/database/postgres/migrations: the postgres tag links
# lib/pq, which carries unfixable advisories (issue #1849). No
# @version suffix: installed as a package of this module, so the
# version and dependency set come from go.mod.
go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate
- name: Run database migrations
env:
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: cudly_test
DB_USER: cudly_test
DB_PASSWORD: test_password # CI-only throwaway password — not used in any real environment
run: |
if [ -d "internal/database/postgres/migrations" ]; then
# The password goes through PGPASSWORD, not the URL, so it needs no
# encoding whatever it contains (@ : / ? # % or spaces).
PGPASSWORD="$DB_PASSWORD" migrate -path internal/database/postgres/migrations \
-database "pgx5://${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" \
up
fi
- name: Run integration tests
env:
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: cudly_test
DB_USER: cudly_test
DB_PASSWORD: test_password # CI-only throwaway password — not used in any real environment
DB_SSL_MODE: disable
run: |
# Integration tests run in the root module; E2E gets its own tag.
set -uo pipefail
status=0
for mod in . tests/e2e; do
tag=$(echo "$mod" | tr './' '--' | sed 's/^-/root/')
log="$RUNNER_TEMP/integration-${tag}.log"
# See the unit job: tests/e2e has no package to test without its own
# tag, so it gets a type-check here too rather than a run.
if [ "$mod" = "tests/e2e" ]; then
echo "==> type-check $mod under -tags=e2e (tests run in the e2e-tests job)"
if ! (cd "$mod" && go vet -tags=e2e ./...) 2>&1 | tee "$log"; then
echo "::error::$mod failed to type-check under -tags=e2e"
status=1
fi
continue
fi
echo "==> integration tests in $mod"
if ! (cd "$mod" && go test -v -race -tags=integration -timeout=20m \
-coverprofile="$RUNNER_TEMP/coverage-integration-${tag}.out" \
./...) 2>&1 | tee "$log"; then
echo "::error::integration tests failed in $mod"
status=1
fi
ran=$(grep -cE '^--- (PASS|FAIL|SKIP)' "$log" || true)
echo "==> $mod ran $ran top-level test(s)"
if [ "$ran" -eq 0 ]; then
echo "::error::$mod ran zero tests -- it is in the loop but asserting nothing"
status=1
fi
done
# Merge per-module profiles for the upload step (see the unit job).
shopt -s nullglob
profiles=("$RUNNER_TEMP"/coverage-integration-*.out)
if [ "${#profiles[@]}" -eq 0 ]; then
echo "::error::no module produced an integration coverage profile" >&2
exit 1
fi
merged="$RUNNER_TEMP/coverage-integration.out"
head -n 1 "${profiles[0]}" > "$merged"
for p in "${profiles[@]}"; do
tail -n +2 "$p" >> "$merged"
done
echo "Merged ${#profiles[@]} integration coverage profile(s)"
exit "$status"
- name: Upload integration coverage
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: integration-coverage
path: ${{ runner.temp }}/coverage-integration.out
retention-days: 30
# Docker image build test
docker-build:
name: Build Docker Image
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build Docker image
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
push: false
# Export the built image into the local docker image store so the
# scan step below inspects the artifact this job just produced,
# rather than rebuilding one and hoping it is the same.
load: true
tags: cudly:${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
VERSION=${{ github.sha }}
- name: Test Docker image
run: |
docker build -t cudly:test .
docker run --rm cudly:test /app/cudly --version || true
docker run --rm cudly:test /app/cudly --help || true
# Nothing else in the pipeline looks at the artifact: govulncheck runs in
# source mode, and Trivy runs with scan-type fs and config, all against
# the repository. A vulnerable binary baked into the image was invisible
# by construction, which is how #1833 shipped twice (issue #1836). These
# steps live in this job rather than a new one so the image is scanned
# without being built a third time; docker-build is already in
# ci-success's needs, so the gate is wired.
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Install govulncheck
run: |
# Same pin as the security-scan job: a govulncheck release with new
# detection logic must not silently change this gate's verdict
# between PRs.
go install golang.org/x/vuln/cmd/govulncheck@v1.1.4
- name: Run image scan self-tests
# Asserts both directions of the verdict against recorded govulncheck
# output, so a scanner that can no longer fail cannot ship as coverage.
run: bash scripts/test-scan-shipped-image.sh
- name: Run govulncheck retry self-tests
run: bash scripts/test-govulncheck-retry.sh
- name: Scan the shipped image for Go advisories
run: bash scripts/scan-shipped-image.sh cudly:${{ github.sha }}
# Terraform validation for all environments
terraform-validate:
name: Validate Terraform (${{ matrix.cloud }})
runs-on: ubuntu-latest
permissions:
contents: read
strategy:
matrix:
cloud: [aws, gcp, azure]
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Setup Terraform
uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
with:
# Must satisfy `required_version = ">= 1.10.0"` declared by every
# terraform/environments/*/main.tf -- pinning below 1.10 makes
# `terraform init`/`validate` abort with "Unsupported Terraform
# Core version". Matches the pin in pre-commit.yml so both
# workflows resolve to the same binary.
terraform_version: 1.10.5
- name: Terraform Format Check
run: terraform fmt -check -recursive terraform/
- name: Test Azure federation identity guards
if: matrix.cloud == 'azure'
run: |
terraform -chdir=iac/federation/azure-target/terraform init -backend=false -lockfile=readonly -input=false
terraform -chdir=iac/federation/azure-target/terraform fmt -check -recursive
terraform -chdir=iac/federation/azure-target/terraform test
- name: Test AWS compute modules
if: matrix.cloud == 'aws'
run: |
for t in terraform/modules/compute/aws/*/tests; do
[ -d "$t" ] || continue
d=$(dirname "$t")
terraform -chdir="$d" init -backend=false -lockfile=readonly -input=false
terraform -chdir="$d" fmt -check -recursive
terraform -chdir="$d" test
done
# The post-deploy /health gate and the GCP uptime check must read the
# body's top-level "status", not just the 200 (#649).
- name: Test /health gate modules
if: matrix.cloud == 'gcp'
run: |
for d in terraform/modules/deployment-checks terraform/modules/monitoring/gcp; do
terraform -chdir="$d" init -backend=false -lockfile=readonly -input=false
terraform -chdir="$d" fmt -check -recursive
terraform -chdir="$d" test
done
- name: Test CI/CD trust (immutable ids)
# Skips a cloud until it has a tests dir. The lock file is gitignored,
# so no -lockfile=readonly.
run: |
d=terraform/environments/${{ matrix.cloud }}/ci-cd-permissions
if [ -d "$d/tests" ]; then
terraform -chdir="$d" init -backend=false -input=false
terraform -chdir="$d" test
fi
- name: Terraform Init
run: |
cd terraform/environments/${{ matrix.cloud }}
terraform init -backend=false
- name: Terraform Validate
run: |
cd terraform/environments/${{ matrix.cloud }}
terraform validate
- name: Validate environment tfvars files
run: |
cd terraform/environments/${{ matrix.cloud }}
for env in dev staging prod; do
# Check local tfvars if present
if [ -f "${env}.tfvars" ]; then
echo "Checking ${env}.tfvars syntax..."
terraform fmt -check "${env}.tfvars" || echo "Note: ${env}.tfvars may need formatting"
fi
# Check GitHub CI/CD tfvars
if [ -f "github-${env}.tfvars" ]; then
echo "Checking github-${env}.tfvars syntax..."
terraform fmt -check "github-${env}.tfvars" || echo "Note: github-${env}.tfvars may need formatting"
fi
done
# Security scanning
security-scan:
name: Security Scanning
runs-on: ubuntu-latest
permissions:
security-events: write
contents: read
steps:
- name: Checkout code
id: checkout
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Go
id: setup_go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Run govulncheck CVE scanner (all modules)
# always(): each scanner step below is independent evidence for the
# Security tab. Without this, one scanner failing (e.g. a live npm
# advisory) skips every scanner after it in the same job, silently
# disabling Go SAST and Terraform IaC coverage repo-wide. The job
# still fails overall if any scanner step here fails. Still requires
# checkout and Go setup to have actually succeeded -- an infra
# failure there must not be papered over as "scanner found nothing".
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success'
run: |
# Pinned (not @latest): a govulncheck release with new
# detection logic could silently change the gate's verdict
# between PRs without an intentional bump in this repo.
# Bumping is a deliberate review item, not a drift.
go install golang.org/x/vuln/cmd/govulncheck@v1.1.4
# Scan both platform-owned modules independently. The wrapper retries
# only transient vulnerability-DB fetch failures, never findings (#74).
set -e
for mod in . tests/e2e; do
echo "==> govulncheck in $mod"
(cd "$mod" && bash "$GITHUB_WORKSPACE/scripts/govulncheck-retry.sh" ./...)
done
- name: Run npm audit (frontend)
# always(): must not skip the scanners below it just because
# govulncheck failed, and its own failure must not skip gosec/Trivy.
# Still requires checkout to have succeeded (see govulncheck above).
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success'
run: |
if [ -f frontend/package.json ]; then
cd frontend && npm audit --audit-level=high
fi
- name: Run gosec Security Scanner
id: gosec
# always(): don't let an earlier scanner's failure (e.g. npm audit)
# skip Go SAST coverage. The job still fails if gosec itself fails.
# Still requires checkout and Go setup to have succeeded.
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success'
run: |
# Install pinned gosec using the job's existing setup-go.
# The securego/gosec Docker action bundles its own Go toolchain which
# cannot satisfy the module's go directive, causing a toolchain mismatch.
go install github.com/securego/gosec/v2/cmd/gosec@v2.28.0
# Collect SARIF for both platform-owned modules before uploading.
#
# gosec exits non-zero both for real findings and for a processing
# error. A bare `set -e` loop aborts at the first non-zero module,
# skipping the merge below entirely -- the upload step then fails
# on a missing file instead of surfacing the actual finding
# (issue #1717). Guarding the gosec call in `if ! ( ... )` exempts
# it from errexit so every module still gets scanned and merged;
# `status` records whether the job should still fail at the end.
set -e
status=0
for mod in . tests/e2e; do
tag=$(echo "$mod" | tr './' '--' | sed 's/^-/root/')
out="$RUNNER_TEMP/gosec-${tag}.sarif"
echo "==> gosec in $mod"
if ! (cd "$mod" && gosec -fmt sarif -out "$out" ./...); then
echo "::warning::gosec exited non-zero in $mod (findings or a scan error)"
status=1
fi
if [ ! -f "$out" ]; then
echo "::error::gosec produced no SARIF output for $mod"
status=1
continue
fi
# Code scanning rejects a SARIF file whose runs share a category
# (github.blog changelog 2025-07-21), so give each module's run a
# unique automationDetails.id before merging.
jq --arg id "gosec-${tag}/" '.runs |= map(.automationDetails = {id: $id})' \
"$out" > "$out.tmp" && mv "$out.tmp" "$out"
done
# Merge per-module SARIF runs into one file for the upload step.
# Only modules that actually produced a file are included; if
# gosec crashed before writing any of them, fail loud here rather
# than uploading an empty result silently.
shopt -s nullglob
sarif_files=("$RUNNER_TEMP"/gosec-*.sarif)
if [ "${#sarif_files[@]}" -eq 0 ]; then
echo "::error::no gosec SARIF output was produced by any module" >&2
exit 1
fi
# jq is preinstalled on the GitHub Ubuntu runner image (no new deps).
# Written to $RUNNER_TEMP, not the checkout root: never save working
# files in the repository root (repo coding guideline), and this
# file is purely a hand-off to the upload step below.
merged="$RUNNER_TEMP/gosec-results.sarif"
jq -s '{version: "2.1.0", "$schema": "https://json.schemastore.org/sarif-2.1.0.json", runs: [.[].runs[]]}' \
"${sarif_files[@]}" > "$merged"
echo "Merged $(jq '.runs | length' "$merged") SARIF runs"
exit "$status"
- name: Upload gosec results to GitHub Security
# Tolerate a missing SARIF only when the gosec step itself never ran
# (e.g. checkout/setup-go failed, so gosec's own `if:` above skipped
# it). If gosec ran and the file is still missing, fail loud instead
# of masking it as a skip.
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.gosec.outcome != 'skipped'
uses: github/codeql-action/upload-sarif@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
with:
sarif_file: ${{ runner.temp }}/gosec-results.sarif
- name: Run Trivy vulnerability scanner (filesystem)
id: trivy_fs
# always(): don't let an earlier scanner's failure skip Trivy fs
# coverage. This scan uses the default exit-code 0 (see the IaC
# scan comment below), so it does not gate the job on its own.
# Still requires checkout and Go setup to have succeeded.
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: 'fs'
scan-ref: '.'
format: 'sarif'
# $RUNNER_TEMP, not the checkout root (repo coding guideline: never
# save working files in the repository root).
output: '${{ runner.temp }}/trivy-results.sarif'
severity: 'CRITICAL,HIGH'
# Pin the Trivy binary independently of the action SHA. v0.36.0 is
# the latest trivy-action release but bundles Trivy v0.70.0, which
# panics in adaptDefaultTags on terraform/environments/aws/main.tf
# (null default_tags vars). Fixed in Trivy >= v0.72.0. The action
# forwards this input to aquasecurity/setup-trivy, so both scan
# steps run the same pinned binary. Keep both steps on this version.
version: 'v0.72.0'
- name: Upload Trivy results to GitHub Security
# Tolerate a missing SARIF only when the Trivy fs step never ran.
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.trivy_fs.outcome != 'skipped'
uses: github/codeql-action/upload-sarif@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
with:
sarif_file: '${{ runner.temp }}/trivy-results.sarif'
# Terraform IaC misconfiguration scanning. Replaces the deprecated
# aquasecurity/tfsec-action, whose bundled HCL parser rejects Terraform
# 1.5+ `check {}` blocks (e.g. terraform/modules/deployment-checks/main.tf)
# with a hard "scan failed" parse error that soft_fail does not suppress
# (soft_fail only downgrades findings, not scan errors). Trivy is tfsec's
# official successor, parses `check {}` blocks, and (like the fs scan
# above) uses the default exit-code 0 so misconfig findings are reported
# to the Security tab without gating the job -- matching tfsec's prior
# soft_fail: true behaviour while preserving Terraform IaC coverage.
- name: Run Trivy IaC misconfiguration scanner (Terraform)
id: trivy_iac
# always(): don't let an earlier scanner's failure skip Terraform
# IaC coverage. Still requires checkout and Go setup to have
# succeeded.
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: 'config'
scan-ref: 'terraform/'
format: 'sarif'
# $RUNNER_TEMP, not the checkout root (repo coding guideline: never
# save working files in the repository root).
output: '${{ runner.temp }}/trivy-config-results.sarif'
severity: 'CRITICAL,HIGH'
# Same pinned Trivy binary as the filesystem scan above (>= v0.72.0
# avoids the adaptDefaultTags panic on null default_tags vars).
version: 'v0.72.0'
- name: Upload Trivy IaC results to GitHub Security
# Tolerate a missing SARIF only when the Trivy IaC step never ran.
if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.trivy_iac.outcome != 'skipped'
uses: github/codeql-action/upload-sarif@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
with:
sarif_file: '${{ runner.temp }}/trivy-config-results.sarif'
# Distinct category so this IaC analysis does not overwrite the
# filesystem Trivy analysis uploaded above (both report as "Trivy").
category: 'trivy-iac'
# Snyk security scanning
snyk-scan:
name: Snyk Security Scan
runs-on: ubuntu-latest
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Run Snyk to check for vulnerabilities
uses: snyk/actions/golang@b98d498629f1c368650224d6d212bf7dfa89e4bf # 0.4.0
continue-on-error: true
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
with:
args: --severity-threshold=high
# Docker Compose E2E tests: build the app + test-runner images, bring up
# postgres + cudly-app, then run the black-box suite in tests/e2e against
# the app over HTTP. The test-runner service is profile-gated, so every
# compose invocation needs --profile test (issue #1180).
e2e-tests:
name: E2E Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build images
run: |
docker compose -f docker-compose.test.yml --profile test build
- name: Run E2E tests with docker compose
run: |
docker compose -f docker-compose.test.yml --profile test up --abort-on-container-exit --exit-code-from test-runner
env:
COMPOSE_INTERACTIVE_NO_CLI: 1
- name: Cleanup
if: always()
run: |
docker compose -f docker-compose.test.yml --profile test down -v
# Assert that the Azure custom-role actions list is identical in the TF module
# and the ARM onboarding template. Fast (shell + jq only), so it always runs.
# Path changes that trigger drift will be caught regardless of PR context.
azure-role-parity:
name: Azure role actions parity (ARM vs TF)
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Assert ARM/TF actions parity
run: bash scripts/check-azure-role-parity.sh
- name: Run parity script self-tests
run: bash scripts/test-azure-role-parity.sh
# Assert that the AWS IAM action lists are identical across the CFN stack,
# the TF lambda/fargate modules, and the federation CFN/TF/CLI templates.
# Fast (shell only), so it always runs.
aws-iam-parity:
name: AWS IAM actions parity (CFN vs TF)
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Assert CFN/TF actions parity
run: bash scripts/check-aws-iam-parity.sh
- name: Run parity script self-tests
run: bash scripts/test-aws-iam-parity.sh
# Assert that no Terraform file grants a Secret Manager role at project,
# folder or organization scope; those scopes hand the member every secret in
# the scope. Fast (shell only), so it always runs.
gcp-secret-scope:
name: GCP Secret Manager grant scope
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Assert no scope-wide Secret Manager grants
run: bash scripts/check-gcp-secret-scope.sh
- name: Run guard script self-tests
run: bash scripts/test-gcp-secret-scope.sh
# Self-test for the git-secrets allowlist (#1972): asserts, through the
# real pre-commit hook scan path, that secret-shaped fixtures are still
# caught and that the documented false positives still scan clean.
git-secrets-allowlist:
name: git-secrets allowlist self-test
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Install git-secrets
# The 1.3.0 tag is mutable in principle, so clone it and then assert
# HEAD resolves to the exact commit verified at the time this was
# written (`git ls-remote --tags` against the upstream repo) before
# trusting it enough to `sudo make install`.
env:
GIT_SECRETS_SHA: ad82d68ee924906a0401dfd48de5057731a9bc84
run: |
set -euo pipefail
git clone --depth 1 --branch 1.3.0 https://github.com/awslabs/git-secrets.git /tmp/git-secrets
resolved="$(git -C /tmp/git-secrets rev-parse HEAD)"
if [ "${resolved}" != "${GIT_SECRETS_SHA}" ]; then
echo "git-secrets 1.3.0 tag resolved to ${resolved}, expected ${GIT_SECRETS_SHA}" >&2
exit 1
fi
sudo make -C /tmp/git-secrets install
- name: Run allowlist self-tests
run: bash scripts/test-git-secrets-allowlist.sh
# Assert that the ECR repository selector used by destroy-fargate-dev.yml and
# cleanup-staging.yml picks the repository each state owns and nothing else.
# The consumers force-delete what the selector prints, so both directions are
# asserted: over-matching deletes images the workflow does not own, and
# matching nothing leaves that state's repository behind. The suite also
# asserts the wiring, which is what #1592 and #1820 each escaped: all three
# destroy steps still call scripts/force-delete-owned-ecr-repo.sh, that script
# still deletes only what the selector yields, and nothing else under
# .github/workflows or scripts/ runs `aws ecr delete-repository` unguarded.
# That last claim is checked over a GLOB of both directories, not a list of
# known files, so a script added later is covered without anyone remembering
# to name it; the sweep is asserted to have opened a non-zero number of files
# first, since an empty swept set has no violations either.
# Fast (shell only), so it always runs.
ecr-delete-selection:
name: ECR delete selection scope
runs-on: ubuntu-latest
# This job checks out the tree and runs a shell script against it, so
# `contents: read` is all it needs. Same shape as security-scan above,
# which adds `security-events: write` only because it uploads SARIF.
# ci.yml now also declares `contents: read` at workflow level, so this
# block narrows nothing on its own; it is kept explicit so the job states
# its own requirement rather than inheriting silently.
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Run selector self-tests
run: bash scripts/test-ecr-delete-selection.sh
# Assert that the RDS instance selector used by destroy-fargate-dev.yml and
# cleanup-staging.yml unprotects the instance each state owns and nothing
# else. Deletion protection is the last line of defence on a database, so
# stripping it from an instance a state does not own leaves that database
# exposed to the next destroy that does match it. Both directions are
# asserted: a prefix near-miss is refused, and the owned instance is still
# selected, since a selector matching nothing passes every refusal assertion
# while leaving the destroy broken. The suite also asserts the wiring, which
# is what #1592, #1820 and #1821 each escaped: all three destroy steps call
# scripts/disable-owned-rds-deletion-protection.sh, that script unprotects
# only what the selector yields and swallows nothing, and nothing else under
# .github/workflows or scripts/ runs `aws rds modify-db-instance` unguarded.
# That last claim is checked over a GLOB of both directories, not a list of
# known files, so a script added later is covered without anyone remembering
# to name it; the sweep is asserted to have opened a non-zero number of files
# first, since an empty swept set has no violations either.
# It additionally runs the script end to end against stubbed terraform and aws,
# so "a failed strip is not swallowed" is asserted as behaviour, not as text.
# Fast (shell only), so it always runs.
rds-deletion-protection-scope:
name: RDS deletion protection scope
runs-on: ubuntu-latest
# Same shape as ecr-delete-selection above: this job checks out the tree and
# runs a shell script against it, so `contents: read` is all it needs.
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Run RDS scope self-tests
run: bash scripts/test-rds-deletion-protection-scope.sh
# Assert that the Cloud SQL instance selector used by cleanup-staging.yml's
# destroy-gcp job deletes the instance the staging state owns and nothing
# else. deletion_protection = true on that instance means a selector that
# matches nothing leaves it behind and the destroy fails on it; a selector
# that over-matches deletes a database this state never owned. Both
# directions are asserted: every near-miss name is refused, and the owned
# instance is still selected out of a hostile listing first, since a
# selector matching nothing passes every refusal assertion while leaving
# the destroy broken. The suite also asserts the wiring, which is what
# #1592, #1820 and #1821 each escaped on other resources: the destroy step
# calls scripts/delete-owned-cloud-sql-instance.sh, that script lists
# instances with no `--filter` and deletes only what the selector yields,
# the three `terraform state rm` calls are module-qualified and run only
# after a successful delete, and nothing else under .github/workflows or
# scripts/ runs `gcloud sql instances delete` unguarded. That last claim is
# checked over a GLOB of both directories, not a list of known files, so a
# script added later is covered without anyone remembering to name it; the
# sweep is asserted to have opened a non-zero number of files first, since
# an empty swept set has no violations either.
# It additionally runs the script end to end against stubbed terraform and
# gcloud, so "a failed delete is not swallowed" and "state is removed only
# after a successful delete" are asserted as behaviour, not as text.
# Fast (shell only), so it always runs.
cloud-sql-delete-scope:
name: Cloud SQL delete selection scope
runs-on: ubuntu-latest
# Same shape as ecr-delete-selection above: this job checks out the tree and
# runs a shell script against it, so `contents: read` is all it needs.
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Run Cloud SQL scope self-tests
run: bash scripts/test-cloud-sql-delete-scope.sh
# Assert that every job applying a `compute_platform` writes the Terraform
# state namespace that platform owns: `compute_platform=lambda` into
# github-<env>/, `compute_platform=fargate` into github-fargate-<env>/. The
# AWS environment is one Terraform root applied twice into two state objects,
# nothing in Terraform ties the backend key to the platform, and a job that
# pairs them wrongly initialises, plans and applies cleanly while rewriting
# the other platform's stack and recording it in the wrong state file. That
# was #1811, where rollback.yml's Fargate rollback keyed on the Lambda
# namespace, so both rollback jobs wrote the same object. A workflow run
# proves nothing about this, so the pairing is asserted as text.
# Both directions are asserted, and the positive one first: the seven real
# pairings are named and checked before any absence, since a scan that
# recognizes no state-writing job has no violations either. The negative half
# is checked over a GLOB of .github/workflows and scripts/, not a list of
# known files, so a job added later is covered without anyone naming it.
# Fast (shell only), so it always runs.
aws-tfstate-platform-key:
name: AWS Terraform state namespace per platform
runs-on: ubuntu-latest
# Same shape as ecr-delete-selection above: this job checks out the tree and
# runs a shell script against it, so `contents: read` is all it needs.
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Run state namespace self-tests
run: bash scripts/test-aws-tfstate-platform-key.sh
# Assert that no Terraform file declares an azurerm_key_vault_access_policy
# resource. This project's only Key Vault sets enable_rbac_authorization =
# true, and an RBAC-enabled vault ignores access policies entirely, so such a
# grant applies cleanly and then does nothing at runtime (#1621). The nested
# `access_policy` block form is deliberately not covered; #1839 tracks it.
# Fast (shell only), so it always runs.
azure-kv-access-policy:
name: Azure Key Vault grant model
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
with:
persist-credentials: false
- name: Assert no Key Vault access policies
run: bash scripts/check-azure-kv-access-policy.sh
- name: Run guard script self-tests
run: bash scripts/test-azure-kv-access-policy.sh
- name: Run smoke-gate self-test
run: bash scripts/test-wait-for-healthy.sh
# Summary job - all checks must pass
ci-success:
name: CI Success
runs-on: ubuntu-latest
needs:
- lint
- workflow-lint
- unit-tests
- integration-tests
- docker-build
- terraform-validate
- security-scan
- e2e-tests
- azure-role-parity
- aws-iam-parity
- gcp-secret-scope
- git-secrets-allowlist
- ecr-delete-selection
- rds-deletion-protection-scope
- cloud-sql-delete-scope
- aws-tfstate-platform-key
- azure-kv-access-policy
if: always()
permissions:
contents: read
steps:
- name: Check all jobs
# Allowlist on success instead of denylisting 'failure' and
# 'cancelled'. A job that never dispatched reports 'skipped', and the
# denylist form reported that as a pass -- so a gate could satisfy this
# summary by not running at all, which is the same shape as the scanner
# gap in #1836. Anything that is not exactly 'success' now fails, and
# names itself in the log.
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: |
# jq is preinstalled on the GitHub Ubuntu runner image (no new deps).
not_success="$(jq -r 'to_entries[]
| select(.value.result != "success")
| " \(.key): \(.value.result)"' <<<"$NEEDS_JSON")"
if [[ -n "$not_success" ]]; then
echo "::error::not every required CI job succeeded"
echo "$not_success"
exit 1
fi
echo "All CI checks passed!"
- name: Post status
run: |
{
echo "## CI Status"
echo ""
echo "✅ All CI checks completed successfully!"
} >> "$GITHUB_STEP_SUMMARY"