diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 773aee018c..10b81ae64d 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -253,21 +253,13 @@ jobs: SLACK_MESSAGE: 'Building Postgres AMI failed' SLACK_FOOTER: '' - - name: configure aws credentials for cleanup - if: ${{ always() }} + - name: Configure AWS credentials for cleanup + if: always() uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: ${{ env.AWS_REGION }} - - name: Cleanup resources after build - if: ${{ always() }} - run: | - aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids - aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id - - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} - run: | - aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids - aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id + - name: Cleanup Packer resources + if: always() + run: ci/cleanup-ami-build-resources.sh "${{ matrix.postgres_version }}-${{ matrix.target.arch }}-${{ github.run_id }}" diff --git a/.github/workflows/cleanup-stale-ec2.yml b/.github/workflows/cleanup-stale-ec2.yml index 3c51b83427..f081a3a0ae 100644 --- a/.github/workflows/cleanup-stale-ec2.yml +++ b/.github/workflows/cleanup-stale-ec2.yml @@ -1,4 +1,4 @@ -name: Cleanup Stale EC2 Instances +name: Cleanup Stale Build Resources on: schedule: @@ -8,45 +8,29 @@ on: jobs: cleanup: - name: Delete stale packer EC2 instances + name: Delete stale build resources runs-on: ubuntu-latest strategy: matrix: - region: [ap-southeast-1, us-east-1] + region: + - ap-southeast-1 + - us-east-1 permissions: id-token: write contents: read steps: + - name: Checkout Repo + uses: supabase/postgres/.github/actions/shared-checkout@HEAD + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: ${{ matrix.region }} - - name: Find and terminate stale instances + - name: Cleanup stale AMI build resources env: AWS_MAX_ATTEMPTS: 6 - run: | - cutoff=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ') - - echo "Looking for running packer builder instances launched before ${cutoff}..." - - instance_ids=$(aws ec2 describe-instances \ - --filters \ - "Name=tag:appType,Values=postgres" \ - "Name=tag:creator,Values=packer" \ - "Name=instance-state-name,Values=running" \ - --query "Reservations[].Instances[?LaunchTime<'${cutoff}'][].InstanceId" \ - --output text) - - if [ -z "$instance_ids" ]; then - echo "No stale instances found." - exit 0 - fi - - read -r -a instance_id_arr <<< "$instance_ids" - echo "Terminating instances: ${instance_id_arr[*]}" - aws ec2 terminate-instances --instance-ids "${instance_id_arr[@]}" - echo "Done." + run: ci/cleanup-stale-ami-build-resources.sh diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 8c1acdd879..7902c754d5 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -145,12 +145,7 @@ jobs: SLACK_MESSAGE: 'Building Postgres QEMU artifact failed' SLACK_FOOTER: '' - - name: Cleanup resources after build - if: ${{ always() }} - run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids - - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} + - name: Cleanup Packer resources + if: always() run: | aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 7288358399..e2430d2571 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -131,63 +131,16 @@ jobs: pip3 install boto3 'boto3-stubs[essential]' ec2instanceconnectcli pytest 'pytest-testinfra[paramiko]' requests pytest -vv -s testinfra/test_ami_nix.py - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} - run: | - EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) - if [ -n "$INSTANCE_IDS" ]; then - echo "Terminating packer build instances: $INSTANCE_IDS" - echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids - else - echo "No packer build instances to clean up" - fi - VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) - if [ -n "$VOLUME_IDS" ]; then - echo "Deleting orphaned packer volumes: $VOLUME_IDS" - echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id - else - echo "No orphaned packer volumes to clean up" - fi - - - name: Cleanup resources after build - if: ${{ always() }} - run: | - EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) - if [ -n "$INSTANCE_IDS" ]; then - echo "Terminating testinfra instances: $INSTANCE_IDS" - echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids || true - else - echo "No testinfra instances to clean up" - fi - VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) - if [ -n "$VOLUME_IDS" ]; then - echo "Deleting orphaned packer volumes: $VOLUME_IDS" - echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id || true - else - echo "No orphaned packer volumes to clean up" - fi + - name: Configure AWS credentials for cleanup + if: always() + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: ${{ env.AWS_REGION }} - - name: Cleanup stage 2 AMI + - name: Cleanup Packer resources if: always() - run: | - EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - STAGE2_AMI_IDS=$(aws ec2 describe-images \ - --region "$AWS_REGION" \ - --owners self \ - --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" \ - --query 'Images[*].ImageId' \ - --output text) - - if [ -n "$STAGE2_AMI_IDS" ]; then - for ami_id in $STAGE2_AMI_IDS; do - echo "Deregistering stage 2 AMI: $ami_id" - aws ec2 deregister-image --region "$AWS_REGION" --image-id "$ami_id" || true - done - else - echo "No stage 2 AMI to clean up" - fi + run: ci/cleanup-ami-build-resources.sh --delete-amis "${{ matrix.postgres_version }}-${{ matrix.target.arch }}-${{ github.run_id }}" report-disk-usage: needs: test-ami-nix diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 445462186a..7ce5f16ff8 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -132,6 +132,7 @@ source "amazon-ebssurrogate" "source" { creator = "packer" appType = "postgres" packerExecutionId = "${var.packer-execution-id}" + supaCreatedAt = timestamp() } run_volume_tags = { creator = "packer" @@ -139,15 +140,17 @@ source "amazon-ebssurrogate" "source" { packerExecutionId = "${var.packer-execution-id}" } snapshot_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } tags = { - creator = "packer" - appType = "postgres" - postgresVersion = "${var.postgres-version}-stage1" - sourceSha = "${var.git-head-version}" - inputHash = "${var.input-hash}" + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}-stage1" + sourceSha = "${var.git-head-version}" + inputHash = "${var.input-hash}" + packerExecutionId = "${var.packer-execution-id}" } communicator = "ssh" diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 7bca54bbc8..13c74fb20e 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -132,6 +132,7 @@ source "amazon-ebssurrogate" "source" { creator = "packer" appType = "postgres" packerExecutionId = "${var.packer-execution-id}" + supaCreatedAt = timestamp() } run_volume_tags = { creator = "packer" @@ -139,15 +140,17 @@ source "amazon-ebssurrogate" "source" { packerExecutionId = "${var.packer-execution-id}" } snapshot_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } tags = { - creator = "packer" - appType = "postgres" - postgresVersion = "${var.postgres-version}-stage1" - sourceSha = "${var.git-head-version}" - inputHash = "${var.input-hash}" + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}-stage1" + sourceSha = "${var.git-head-version}" + inputHash = "${var.input-hash}" + packerExecutionId = "${var.packer-execution-id}" } communicator = "ssh" diff --git a/ci/cleanup-ami-build-resources.sh b/ci/cleanup-ami-build-resources.sh new file mode 100644 index 0000000000..ac043e55ca --- /dev/null +++ b/ci/cleanup-ami-build-resources.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +set -uo pipefail + +usage() { + echo "Usage: $0 [--delete-amis] " >&2 +} + +amis=false +if [[ ${1:-} == "--delete-amis" ]]; then + amis=true + shift +fi + +execution_id=${1:-} +if [[ -z $execution_id || $# -ne 1 ]]; then + usage + exit 2 +fi + +if [[ -z ${AWS_REGION:-} ]]; then + echo "AWS_REGION must be set" >&2 + exit 2 +fi + +failures=0 + +failed() { + echo "Cleanup failed: $*" >&2 + failures=$((failures + 1)) +} + +ids() { + local operation=$1 + local query=$2 + local output + shift 2 + + ids=() + if ! output=$(aws ec2 "$operation" --filters "$@" --query "$query" --output text); then + failed "unable to list resources with $operation using filters: $*" + return 1 + fi + read -r -a ids <<<"$output" + if ((${#ids[@]})); then + return 0 + fi + return 2 +} + +echo "Cleaning up AMI build resources for execution $execution_id in $AWS_REGION" >&2 + +status=Name=instance-state-name,Values=pending,running,stopping,stopped + +# Testinfra instances use a separate tag with same value +tag=Name=tag:testinfra-run-id,Values=$execution_id +if ids describe-instances Reservations[].Instances[].InstanceId $status "$tag"; then + echo "Terminating testinfra instances: ${ids[*]}" >&2 + aws ec2 terminate-instances --instance-ids "${ids[@]}" >/dev/null || failed "unable to terminate testinfra instances: ${ids[*]}" + aws ec2 wait instance-terminated --instance-ids "${ids[@]}" || failed "timed out waiting for testinfra instances to terminate: ${ids[*]}" +fi + +tag=Name=tag:packerExecutionId,Values=$execution_id +if ids describe-instances Reservations[].Instances[].InstanceId $status "$tag"; then + echo "Terminating Packer instances: ${ids[*]}" >&2 + aws ec2 terminate-instances --instance-ids "${ids[@]}" >/dev/null || failed "unable to terminate Packer instances: ${ids[*]}" + aws ec2 wait instance-terminated --instance-ids "${ids[@]}" || failed "timed out waiting for Packer instances to terminate: ${ids[*]}" +fi + +status=Name=status,Values=available +if ids describe-network-interfaces NetworkInterfaces[].NetworkInterfaceId $status "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting network interface: $id" >&2 + aws ec2 delete-network-interface --network-interface-id "$id" || failed "unable to delete network interface: $id" + done +fi + +if ids describe-volumes Volumes[].VolumeId $status "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting volume: $id" >&2 + aws ec2 delete-volume --volume-id "$id" || failed "unable to delete volume: $id" + done +fi + +if ids describe-security-groups SecurityGroups[].GroupId "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting security group: $id" >&2 + deleted=false + # AWS may return DependencyViolation until terminated instances release their ENIs + for _ in {1..6}; do + if aws ec2 delete-security-group --group-id "$id"; then + deleted=true + break + fi + sleep 10 + done + $deleted || failed "unable to delete security group: $id" + done +fi + +if ids describe-key-pairs KeyPairs[].KeyPairId "$tag"; then + for id in "${ids[@]}"; do + echo "Deleting key pair: $id" >&2 + aws ec2 delete-key-pair --key-pair-id "$id" || failed "unable to delete key pair: $id" + done +fi + +if $amis; then + if ids describe-images Images[].ImageId "$tag"; then + for id in "${ids[@]}"; do + echo "Deregistering AMI and deleting associated snapshots: $id" >&2 + aws ec2 deregister-image --image-id "$id" --delete-associated-snapshots || failed "unable to deregister AMI and delete associated snapshots: $id" + done + fi +fi + +# Find any orphaned snapshots from a cancel before the AMI was finalized +if ids describe-snapshots Snapshots[].SnapshotId "$tag"; then + # going to call ids again below which clobbers ids, so copy to new var + snapshots=("${ids[@]}") + for snapshot in "${snapshots[@]}"; do + if ids describe-images Images[].ImageId "Name=block-device-mapping.snapshot-id,Values=$snapshot"; then + continue + elif (($? == 1)); then + # error with aws command, skip for safety + continue + fi + + echo "Deleting orphaned snapshot: $snapshot" >&2 + aws ec2 delete-snapshot --snapshot-id "$snapshot" || failed "unable to delete orphaned snapshot: $snapshot" + done +fi + +if ((failures)); then + echo "Packer cleanup completed with $failures error(s)" >&2 + exit 1 +fi + +echo "Packer cleanup complete" >&2 diff --git a/ci/cleanup-stale-ami-build-resources.sh b/ci/cleanup-stale-ami-build-resources.sh new file mode 100755 index 0000000000..0c49fd7215 --- /dev/null +++ b/ci/cleanup-stale-ami-build-resources.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# The script finds Packer resources and testinfra instances older than 24 hours across one AWS region. +# It uses each resource's timestamp, or Packer's creation tag for security groups, to identify stale builds. +# It then groups the resources by execution ID and passes each ID to the shared cleanup script. +# Registered AMIs and the snapshots they reference are left intact. + +if [[ -z ${AWS_REGION:-} ]]; then + echo "AWS_REGION must be set" >&2 + exit 2 +fi + +msg() { + echo "$*" >&2 +} + +cutoff=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ') +msg "Looking for Packer and testinfra executions with resources created before $cutoff..." + +args=( + --filters 'Name=tag-key,Values=packerExecutionId' 'Name=tag:appType,Values=postgres' + --output json +) +readarray -t ids < <( + { + { + msg "testinfra instances" + aws ec2 describe-instances \ + --filters \ + "Name=tag-key,Values=testinfra-run-id" \ + "Name=tag:creator,Values=testinfra-ci" \ + --output json + msg "packer instances" + aws ec2 describe-instances "${args[@]}" + } | + jq -r --arg cutoff "$cutoff" '.Reservations[].Instances[] | select(.LaunchTime < $cutoff)' | + tee .stale-instances.json + + msg "volumes" + aws ec2 describe-volumes "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.Volumes[] | select(.CreateTime < $cutoff)' | + tee .stale-volumes.json + + msg "key pairs" + aws ec2 describe-key-pairs "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.KeyPairs[] | select(.CreateTime < $cutoff)' | + tee .stale-key-pairs.json + + msg "snapshots" + aws ec2 describe-snapshots --owner-ids self "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.Snapshots[] | select(.StartTime < $cutoff)' | + tee .stale-snapshots.json + + msg "security groups" + aws ec2 describe-security-groups "${args[@]}" | + jq -r --arg cutoff "$cutoff" '.SecurityGroups[] | select(any(.Tags[]?; .Key == "supaCreatedAt" and .Value < $cutoff))' | + tee .stale-security-groups.json + } | + jq -r '.Tags[]? | select(.Key == "packerExecutionId" or .Key == "testinfra-run-id") | .Value' | + sort -Vu | + tee .ids.txt +) + +if ((${#ids[@]} == 0)); then + echo "No stale Packer or testinfra executions found" >&2 + exit 0 +fi + +cleaner=$(dirname "$0")/cleanup-ami-build-resources.sh +for id in "${ids[@]}"; do + echo "$cleaner" "$id" +done diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 6bd12a1c4b..f4366015d1 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -79,14 +79,17 @@ source "amazon-ebs" "ubuntu" { creator = "packer" appType = "postgres" packerExecutionId = "${var.packer-execution-id}" + supaCreatedAt = timestamp() } run_volume_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } snapshot_tags = { - creator = "packer" - appType = "postgres" + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" } tags = { creator = "packer" diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index cfd77554d6..399b8a0742 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -13,15 +13,9 @@ from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger -# if EXECUTION_ID is not set, use a default value that includes the user and hostname -RUN_ID = os.environ.get( - "EXECUTION_ID", - "unknown-ci-run-" - + os.environ.get("USER", "unknown-user") - + "@" - + socket.gethostname(), -) -AMI_ID = os.environ.get("AMI_ID") +RUN_ID = os.environ["EXECUTION_ID"] +AMI_ID = os.environ["AMI_ID"] + postgresql_schema_sql_content = """ ALTER DATABASE postgres SET "app.settings.jwt_secret" TO 'my_jwt_secret_which_is_not_so_secret'; ALTER DATABASE postgres SET "app.settings.jwt_exp" TO 3600;