From 8ce28d72aad1d2c94c31624f29bd37e1a93c525a Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 27 Aug 2026 11:02:14 -0400 Subject: [PATCH 1/3] testinfra: Make AMI_ID and EXECUTION_ID required, fail early if missing AMI_ID is obviously always required but would fail later in the test, better to do so early IMO. This test isn't really being run locally at the moment if/when it does, I want to make it easier by using the nix package instead which will set this before running. --- testinfra/test_ami_nix.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index cfd77554d..399b8a074 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -13,15 +13,9 @@ from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger -# if EXECUTION_ID is not set, use a default value that includes the user and hostname -RUN_ID = os.environ.get( - "EXECUTION_ID", - "unknown-ci-run-" - + os.environ.get("USER", "unknown-user") - + "@" - + socket.gethostname(), -) -AMI_ID = os.environ.get("AMI_ID") +RUN_ID = os.environ["EXECUTION_ID"] +AMI_ID = os.environ["AMI_ID"] + postgresql_schema_sql_content = """ ALTER DATABASE postgres SET "app.settings.jwt_secret" TO 'my_jwt_secret_which_is_not_so_secret'; ALTER DATABASE postgres SET "app.settings.jwt_exp" TO 3600; From f7a2ae86d933a774d2ce909b27c61c7f491dcccd Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 27 Aug 2026 11:02:14 -0400 Subject: [PATCH 2/3] ci: Better AWS cleanup Our clean up has been pretty loose. We've had left over key pair, security groups and instances too. This new script fixes all of those. AMIs are only deleted if the --delete-ami is passed in which is not done for Release AMI Nix workflow. --- .github/workflows/ami-release-nix.yml | 18 +-- .github/workflows/qemu-image-build.yml | 9 +- .github/workflows/testinfra-ami-build.yml | 63 ++-------- amazon-amd64-nix.pkr.hcl | 17 +-- amazon-arm64-nix.pkr.hcl | 17 +-- ci/cleanup-ami-build-resources.sh | 139 ++++++++++++++++++++++ stage2-nix-psql.pkr.hcl | 11 +- 7 files changed, 181 insertions(+), 93 deletions(-) create mode 100644 ci/cleanup-ami-build-resources.sh diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 773aee018..10b81ae64 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -253,21 +253,13 @@ jobs: SLACK_MESSAGE: 'Building Postgres AMI failed' SLACK_FOOTER: '' - - name: configure aws credentials for cleanup - if: ${{ always() }} + - name: Configure AWS credentials for cleanup + if: always() uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: ${{ env.AWS_REGION }} - - name: Cleanup resources after build - if: ${{ always() }} - run: | - aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids - aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id - - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} - run: | - aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids - aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id + - name: Cleanup Packer resources + if: always() + run: ci/cleanup-ami-build-resources.sh "${{ matrix.postgres_version }}-${{ matrix.target.arch }}-${{ github.run_id }}" diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 8c1acdd87..7902c754d 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -145,12 +145,7 @@ jobs: SLACK_MESSAGE: 'Building Postgres QEMU artifact failed' SLACK_FOOTER: '' - - name: Cleanup resources after build - if: ${{ always() }} - run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids - - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} + - name: Cleanup Packer resources + if: always() run: | aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 728835839..e2430d257 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -131,63 +131,16 @@ jobs: pip3 install boto3 'boto3-stubs[essential]' ec2instanceconnectcli pytest 'pytest-testinfra[paramiko]' requests pytest -vv -s testinfra/test_ami_nix.py - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} - run: | - EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) - if [ -n "$INSTANCE_IDS" ]; then - echo "Terminating packer build instances: $INSTANCE_IDS" - echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids - else - echo "No packer build instances to clean up" - fi - VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) - if [ -n "$VOLUME_IDS" ]; then - echo "Deleting orphaned packer volumes: $VOLUME_IDS" - echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id - else - echo "No orphaned packer volumes to clean up" - fi - - - name: Cleanup resources after build - if: ${{ always() }} - run: | - EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) - if [ -n "$INSTANCE_IDS" ]; then - echo "Terminating testinfra instances: $INSTANCE_IDS" - echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids || true - else - echo "No testinfra instances to clean up" - fi - VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) - if [ -n "$VOLUME_IDS" ]; then - echo "Deleting orphaned packer volumes: $VOLUME_IDS" - echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id || true - else - echo "No orphaned packer volumes to clean up" - fi + - name: Configure AWS credentials for cleanup + if: always() + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: ${{ env.AWS_REGION }} - - name: Cleanup stage 2 AMI + - name: Cleanup Packer resources if: always() - run: | - EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - STAGE2_AMI_IDS=$(aws ec2 describe-images \ - --region "$AWS_REGION" \ - --owners self \ - --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" \ - --query 'Images[*].ImageId' \ - --output text) - - if [ -n "$STAGE2_AMI_IDS" ]; then - for ami_id in $STAGE2_AMI_IDS; do - echo "Deregistering stage 2 AMI: $ami_id" - aws ec2 deregister-image --region "$AWS_REGION" --image-id "$ami_id" || true - done - else - echo "No stage 2 AMI to clean up" - fi + run: ci/cleanup-ami-build-resources.sh --delete-amis "${{ matrix.postgres_version }}-${{ matrix.target.arch }}-${{ github.run_id }}" report-disk-usage: needs: test-ami-nix diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 445462186..7ce5f16ff 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -132,6 +132,7 @@ source "amazon-ebssurrogate" "source" { creator = "packer" appType = "postgres" packerExecutionId = "${var.packer-execution-id}" + supaCreatedAt = timestamp() } run_volume_tags = { creator = "packer" @@ -139,15 +140,17 @@ source "amazon-ebssurrogate" "source" { packerExecutionId = "${var.packer-execution-id}" } snapshot_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } tags = { - creator = "packer" - appType = "postgres" - postgresVersion = "${var.postgres-version}-stage1" - sourceSha = "${var.git-head-version}" - inputHash = "${var.input-hash}" + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}-stage1" + sourceSha = "${var.git-head-version}" + inputHash = "${var.input-hash}" + packerExecutionId = "${var.packer-execution-id}" } communicator = "ssh" diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 7bca54bbc..13c74fb20 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -132,6 +132,7 @@ source "amazon-ebssurrogate" "source" { creator = "packer" appType = "postgres" packerExecutionId = "${var.packer-execution-id}" + supaCreatedAt = timestamp() } run_volume_tags = { creator = "packer" @@ -139,15 +140,17 @@ source "amazon-ebssurrogate" "source" { packerExecutionId = "${var.packer-execution-id}" } snapshot_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } tags = { - creator = "packer" - appType = "postgres" - postgresVersion = "${var.postgres-version}-stage1" - sourceSha = "${var.git-head-version}" - inputHash = "${var.input-hash}" + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}-stage1" + sourceSha = "${var.git-head-version}" + inputHash = "${var.input-hash}" + packerExecutionId = "${var.packer-execution-id}" } communicator = "ssh" diff --git a/ci/cleanup-ami-build-resources.sh b/ci/cleanup-ami-build-resources.sh new file mode 100644 index 000000000..ac043e55c --- /dev/null +++ b/ci/cleanup-ami-build-resources.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +set -uo pipefail + +usage() { + echo "Usage: $0 [--delete-amis] " >&2 +} + +amis=false +if [[ ${1:-} == "--delete-amis" ]]; then + amis=true + shift +fi + +execution_id=${1:-} +if [[ -z $execution_id || $# -ne 1 ]]; then + usage + exit 2 +fi + +if [[ -z ${AWS_REGION:-} ]]; then + echo "AWS_REGION must be set" >&2 + exit 2 +fi + +failures=0 + +failed() { + echo "Cleanup failed: $*" >&2 + failures=$((failures + 1)) +} + +ids() { + local operation=$1 + local query=$2 + local output + shift 2 + + ids=() + if ! output=$(aws ec2 "$operation" --filters "$@" --query "$query" --output text); then + failed "unable to list resources with $operation using filters: $*" + return 1 + fi + read -r -a ids <<<"$output" + if ((${#ids[@]})); then + return 0 + fi + return 2 +} + +echo "Cleaning up AMI build resources for execution $execution_id in $AWS_REGION" >&2 + +status=Name=instance-state-name,Values=pending,running,stopping,stopped + +# Testinfra instances use a separate tag with same value +tag=Name=tag:testinfra-run-id,Values=$execution_id +if ids describe-instances Reservations[].Instances[].InstanceId $status "$tag"; then + echo "Terminating testinfra instances: ${ids[*]}" >&2 + aws ec2 terminate-instances --instance-ids "${ids[@]}" >/dev/null || failed "unable to terminate testinfra instances: ${ids[*]}" + aws ec2 wait instance-terminated --instance-ids "${ids[@]}" || failed "timed out waiting for testinfra instances to terminate: ${ids[*]}" +fi + +tag=Name=tag:packerExecutionId,Values=$execution_id +if ids describe-instances Reservations[].Instances[].InstanceId $status "$tag"; then + echo "Terminating Packer instances: ${ids[*]}" >&2 + aws ec2 terminate-instances --instance-ids "${ids[@]}" >/dev/null || failed "unable to terminate Packer instances: ${ids[*]}" + aws ec2 wait instance-terminated --instance-ids "${ids[@]}" || failed "timed out waiting for Packer instances to terminate: ${ids[*]}" +fi + +status=Name=status,Values=available +if ids describe-network-interfaces NetworkInterfaces[].NetworkInterfaceId $status "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting network interface: $id" >&2 + aws ec2 delete-network-interface --network-interface-id "$id" || failed "unable to delete network interface: $id" + done +fi + +if ids describe-volumes Volumes[].VolumeId $status "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting volume: $id" >&2 + aws ec2 delete-volume --volume-id "$id" || failed "unable to delete volume: $id" + done +fi + +if ids describe-security-groups SecurityGroups[].GroupId "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting security group: $id" >&2 + deleted=false + # AWS may return DependencyViolation until terminated instances release their ENIs + for _ in {1..6}; do + if aws ec2 delete-security-group --group-id "$id"; then + deleted=true + break + fi + sleep 10 + done + $deleted || failed "unable to delete security group: $id" + done +fi + +if ids describe-key-pairs KeyPairs[].KeyPairId "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting key pair: $id" >&2 + aws ec2 delete-key-pair --key-pair-id "$id" || failed "unable to delete key pair: $id" + done +fi + +if $amis; then + if ids describe-images Images[].ImageId "$tag"; then + for id in "${ids[@]}"; do + echo "Deregistering AMI and deleting associated snapshots: $id" >&2 + aws ec2 deregister-image --image-id "$id" --delete-associated-snapshots || failed "unable to deregister AMI and delete associated snapshots: $id" + done + fi +fi + +# Find any orphaned snapshots from a cancel before the AMI was finalized +if ids describe-snapshots Snapshots[].SnapshotId "$tag"; then + # going to call ids again below which clobbers ids, so copy to new var + snapshots=("${ids[@]}") + for snapshot in "${snapshots[@]}"; do + if ids describe-images Images[].ImageId "Name=block-device-mapping.snapshot-id,Values=$snapshot"; then + continue + elif (($? == 1)); then + # error with aws command, skip for safety + continue + fi + + echo "Deleting orphaned snapshot: $snapshot" >&2 + aws ec2 delete-snapshot --snapshot-id "$snapshot" || failed "unable to delete orphaned snapshot: $snapshot" + done +fi + +if ((failures)); then + echo "Packer cleanup completed with $failures error(s)" >&2 + exit 1 +fi + +echo "Packer cleanup complete" >&2 diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 6bd12a1c4..f4366015d 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -79,14 +79,17 @@ source "amazon-ebs" "ubuntu" { creator = "packer" appType = "postgres" packerExecutionId = "${var.packer-execution-id}" + supaCreatedAt = timestamp() } run_volume_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } snapshot_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } tags = { creator = "packer" From 600160d3cc19f886af04c4a8635ee9ab64873a42 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 27 Aug 2026 11:02:14 -0400 Subject: [PATCH 3/3] ci: Clean up :all_the_things: (resources)! The previous clean up was pretty lightweight this new version is more complete. First it looks for any old resources to get their executionId and then passes those down to the actual clean up script which is reusing the one that runs per-ci job so that we don't have duplication that may drift. AMIs are never deleted by this script so are safe. --- .github/workflows/cleanup-stale-ec2.yml | 36 ++++-------- ci/cleanup-stale-ami-build-resources.sh | 74 +++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 26 deletions(-) create mode 100755 ci/cleanup-stale-ami-build-resources.sh diff --git a/.github/workflows/cleanup-stale-ec2.yml b/.github/workflows/cleanup-stale-ec2.yml index 3c51b8342..f081a3a0a 100644 --- a/.github/workflows/cleanup-stale-ec2.yml +++ b/.github/workflows/cleanup-stale-ec2.yml @@ -1,4 +1,4 @@ -name: Cleanup Stale EC2 Instances +name: Cleanup Stale Build Resources on: schedule: @@ -8,45 +8,29 @@ on: jobs: cleanup: - name: Delete stale packer EC2 instances + name: Delete stale build resources runs-on: ubuntu-latest strategy: matrix: - region: [ap-southeast-1, us-east-1] + region: + - ap-southeast-1 + - us-east-1 permissions: id-token: write contents: read steps: + - name: Checkout Repo + uses: supabase/postgres/.github/actions/shared-checkout@HEAD + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: ${{ matrix.region }} - - name: Find and terminate stale instances + - name: Cleanup stale AMI build resources env: AWS_MAX_ATTEMPTS: 6 - run: | - cutoff=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ') - - echo "Looking for running packer builder instances launched before ${cutoff}..." - - instance_ids=$(aws ec2 describe-instances \ - --filters \ - "Name=tag:appType,Values=postgres" \ - "Name=tag:creator,Values=packer" \ - "Name=instance-state-name,Values=running" \ - --query "Reservations[].Instances[?LaunchTime<'${cutoff}'][].InstanceId" \ - --output text) - - if [ -z "$instance_ids" ]; then - echo "No stale instances found." - exit 0 - fi - - read -r -a instance_id_arr <<< "$instance_ids" - echo "Terminating instances: ${instance_id_arr[*]}" - aws ec2 terminate-instances --instance-ids "${instance_id_arr[@]}" - echo "Done." + run: ci/cleanup-stale-ami-build-resources.sh diff --git a/ci/cleanup-stale-ami-build-resources.sh b/ci/cleanup-stale-ami-build-resources.sh new file mode 100755 index 000000000..0c49fd721 --- /dev/null +++ b/ci/cleanup-stale-ami-build-resources.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# The script finds Packer resources and testinfra instances older than 24 hours across one AWS region. +# It uses each resource's timestamp, or Packer's creation tag for security groups, to identify stale builds. +# It then groups the resources by execution ID and passes each ID to the shared cleanup script. +# Registered AMIs and the snapshots they reference are left intact. + +if [[ -z ${AWS_REGION:-} ]]; then + echo "AWS_REGION must be set" >&2 + exit 2 +fi + +msg() { + echo "$*" >&2 +} + +cutoff=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ') +msg "Looking for Packer and testinfra executions with resources created before $cutoff..." + +args=( + --filters 'Name=tag-key,Values=packerExecutionId' 'Name=tag:appType,Values=postgres' + --output json +) +readarray -t ids < <( + { + { + msg "testinfra instances" + aws ec2 describe-instances \ + --filters \ + "Name=tag-key,Values=testinfra-run-id" \ + "Name=tag:creator,Values=testinfra-ci" \ + --output json + msg "packer instances" + aws ec2 describe-instances "${args[@]}" + } | + jq -r --arg cutoff "$cutoff" '.Reservations[].Instances[] | select(.LaunchTime < $cutoff)' | + tee .stale-instances.json + + msg "volumes" + aws ec2 describe-volumes "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.Volumes[] | select(.CreateTime < $cutoff)' | + tee .stale-volumes.json + + msg "key pairs" + aws ec2 describe-key-pairs "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.KeyPairs[] | select(.CreateTime < $cutoff)' | + tee .stale-key-pairs.json + + msg "snapshots" + aws ec2 describe-snapshots --owner-ids self "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.Snapshots[] | select(.StartTime < $cutoff)' | + tee .stale-snapshots.json + + msg "security groups" + aws ec2 describe-security-groups "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.SecurityGroups[] | select(any(.Tags[]?; .Key == "supaCreatedAt" and .Value < $cutoff))' | + tee .stale-security-groups.json + } | + jq -r '.Tags[]? | select(.Key == "packerExecutionId" or .Key == "testinfra-run-id") | .Value' | + sort -Vu | + tee .ids.txt +) + +if ((${#ids[@]} == 0)); then + echo "No stale Packer or testinfra executions found" >&2 + exit 0 +fi + +cleaner=$(dirname "$0")/cleanup-ami-build-resources.sh +for id in "${ids[@]}"; do + echo "$cleaner" "$id" +done