diff --git a/.github/workflows/betacrew-safety.yml b/.github/workflows/betacrew-safety.yml new file mode 100644 index 0000000..a585584 --- /dev/null +++ b/.github/workflows/betacrew-safety.yml @@ -0,0 +1,28 @@ +name: BetaCrew database safety +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] +permissions: + contents: read +jobs: + safety: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: [self-hosted, linux, x64, makepad] + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + - name: Database bootstrap, HBA and restore guard regression tests + run: python3 scripts/test-betacrew-safety.py + - name: Real encrypted database restore over verified TLS + run: python3 scripts/test-betacrew-encrypted-restore.py + - name: Encryption contract (mock database dump) + run: bash scripts/test-betacrew-backup.sh + - name: Configuration and shell syntax + run: | + bash scripts/validate-betacrew-config.sh + bash -n scripts/verify-betacrew-restore.sh scripts/apply-betacrew-hba.sh diff --git a/README.md b/README.md index 4fe7c03..8266c24 100644 --- a/README.md +++ b/README.md @@ -953,3 +953,29 @@ The verifier checks both the app migration table and Keycloak realm table. Jotwink host backup is opt-in through the `jotwink` Compose profile. Provision and validate its recipient, credentials, TLS and restore evidence before enabling that profile. The shared deployment workflow preserves its existing release transaction. + +## BetaCrew Backup And Restore + +BetaCrew has an independent six-hour backup service for `betacrew` and +`keycloak_betacrew`. Dumps are validated, encrypted with OpenSSL CMS +AES-256-GCM before persistent storage, checksummed, and retained for 35 days. +Provision `/var/lib/makepad/postgres-backups/betacrew` for uid 70 and install +the public recipient certificate at +`/etc/makepad/postgres-backup/betacrew-recipient.pem`; keep its private key +offline. Replicate completed backup directories to independently administered +storage. + +Run a restore drill against empty non-production databases with: + +```bash +export PGSERVICEFILE=/etc/makepad/postgres-restore-services.conf +export BETACREW_RESTORE_SERVICE=betacrew_restore_test +export KEYCLOAK_BETACREW_RESTORE_SERVICE=keycloak_betacrew_restore_test +export BETACREW_RESTORE_CONFIRM=replace-nonproduction-restore-targets +export BETACREW_BACKUP_DECRYPTION_CERT=/secure/betacrew-recipient.pem +export BETACREW_BACKUP_DECRYPTION_KEY=/secure/betacrew-recipient.key +scripts/verify-betacrew-restore.sh /var/lib/makepad/postgres-backups/betacrew/ +``` + + +BetaCrew backup uses uid 999 and a dedicated mode-0400 credential file owned by that uid. The shared host Compose service requires the `betacrew` profile; the separate `compose.betacrew.host.yml` remains available for a scoped backup-only deployment. diff --git a/bootstrap/betacrew-databases.sql b/bootstrap/betacrew-databases.sql new file mode 100644 index 0000000..f69690b --- /dev/null +++ b/bootstrap/betacrew-databases.sql @@ -0,0 +1,67 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql +-- variables so credentials never enter source control. +\if :{?betacrew_app_password} +\else + \echo 'missing required psql variable: betacrew_app_password' + SELECT 1/0; +\endif +\if :{?keycloak_betacrew_app_password} +\else + \echo 'missing required psql variable: keycloak_betacrew_app_password' + SELECT 1/0; +\endif + +SELECT length(:'betacrew_app_password') >= 32 AS betacrew_password_ok \gset +\if :betacrew_password_ok +\else + \echo 'empty required psql variable: betacrew_app_password' + SELECT 1/0; +\endif +SELECT length(:'keycloak_betacrew_app_password') >= 32 AS keycloak_betacrew_password_ok \gset +\if :keycloak_betacrew_password_ok +\else + \echo 'empty required psql variable: keycloak_betacrew_app_password' + SELECT 1/0; +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); + +-- Validate both pairs before changing either. Existing objects must already be +-- isolated; never seize ownership or rotate a live credential during bootstrap. +DO $$ +DECLARE target record; role_row pg_roles%ROWTYPE; database_row pg_database%ROWTYPE; +BEGIN + FOR target IN SELECT * FROM (VALUES + ('betacrew','betacrew_app'),('keycloak_betacrew','keycloak_betacrew_app') + ) AS targets(database_name,role_name) LOOP + SELECT * INTO role_row FROM pg_roles WHERE rolname=target.role_name; + SELECT * INTO database_row FROM pg_database WHERE datname=target.database_name; + IF (role_row.oid IS NULL) <> (database_row.oid IS NULL) THEN + RAISE EXCEPTION 'Incomplete existing BetaCrew role/database pair'; + END IF; + IF role_row.oid IS NOT NULL THEN + IF NOT role_row.rolcanlogin OR role_row.rolsuper OR role_row.rolcreatedb + OR role_row.rolcreaterole OR role_row.rolreplication OR role_row.rolbypassrls + OR EXISTS(SELECT 1 FROM pg_auth_members WHERE member=role_row.oid OR roleid=role_row.oid) + OR database_row.datdba<>role_row.oid + OR EXISTS(SELECT 1 FROM aclexplode(COALESCE(database_row.datacl,acldefault('d',database_row.datdba))) WHERE grantee<>role_row.oid) + THEN RAISE EXCEPTION 'Existing BetaCrew privileges, membership, ownership or ACL require manual inspection'; + END IF; + END IF; + END LOOP; +END $$; + +SELECT format('CREATE ROLE betacrew_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'betacrew_app_password') +WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='betacrew_app') \gexec +SELECT 'CREATE DATABASE betacrew OWNER betacrew_app' +WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='betacrew') \gexec +REVOKE ALL ON DATABASE betacrew FROM PUBLIC; +SELECT format('CREATE ROLE keycloak_betacrew_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'keycloak_betacrew_app_password') +WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='keycloak_betacrew_app') \gexec +SELECT 'CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app' +WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='keycloak_betacrew') \gexec +REVOKE ALL ON DATABASE keycloak_betacrew FROM PUBLIC; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); diff --git a/compose.betacrew.host.yml b/compose.betacrew.host.yml new file mode 100644 index 0000000..a1a69b1 --- /dev/null +++ b/compose.betacrew.host.yml @@ -0,0 +1,42 @@ +services: + betacrew_backup: + image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825} + user: "999:999" + network_mode: host + command: [/usr/local/bin/run-betacrew-backup-loop.sh] + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem + BETACREW_BACKUP_ROOT: /backups + BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600} + BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} + BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + restart: unless-stopped + volumes: + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro" + healthcheck: + test: [CMD, /usr/local/bin/run-betacrew-backup-loop.sh, healthcheck] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/compose.host.yml b/compose.host.yml index dd32d2b..fba5250 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -181,3 +181,52 @@ services: options: max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + + betacrew_backup: + image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825} + profiles: [betacrew] + user: "999:999" + network_mode: host + command: + - /usr/local/bin/run-betacrew-backup-loop.sh + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem + BETACREW_BACKUP_ROOT: /backups + BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600} + BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} + BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + volumes: + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro" + healthcheck: + test: ["CMD", "/usr/local/bin/run-betacrew-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 08be544..022cf69 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -1,5 +1,5 @@ -# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also -# source-restricted to the application WireGuard peer and Keycloak host. +# Runtrace, Jotwink, and BetaCrew customer/identity traffic must use TLS. +# Jotwink and BetaCrew are source-restricted to the app WireGuard peer and Keycloak host. # Other shared databases retain their current SCRAM policy during migration. local all all trust hostnossl runtrace all all reject @@ -16,6 +16,14 @@ hostssl jotwink postgres 127.0.0.1/32 scram-sha-25 hostssl keycloak_jotwink postgres 127.0.0.1/32 scram-sha-256 hostssl jotwink all all reject hostssl keycloak_jotwink all all reject +hostnossl betacrew all all reject +hostnossl keycloak_betacrew all all reject +hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256 +hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256 +hostssl betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl betacrew all all reject +hostssl keycloak_betacrew all all reject hostssl brio_staging brio_staging_app all scram-sha-256 hostssl brio_staging brio_staging_backup all scram-sha-256 hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 diff --git a/envs/production/.env.db b/envs/production/.env.db index ddc9168..c745dfb 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -45,3 +45,10 @@ MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres- MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35 +BETACREW_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825 +MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH=/var/lib/makepad/postgres-backups/betacrew +MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password +MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/betacrew-recipient.pem +MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS=35 diff --git a/scripts/apply-betacrew-hba.sh b/scripts/apply-betacrew-hba.sh new file mode 100755 index 0000000..21cea09 --- /dev/null +++ b/scripts/apply-betacrew-hba.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "$0")" && pwd) +exec python3 "$root/betacrew-hba.py" "${1:?Usage: apply-betacrew-hba.sh }" diff --git a/scripts/betacrew-hba.py b/scripts/betacrew-hba.py new file mode 100644 index 0000000..6d12bb0 --- /dev/null +++ b/scripts/betacrew-hba.py @@ -0,0 +1,48 @@ +"""Add or validate the complete BetaCrew HBA block without altering other rules.""" +import os +from pathlib import Path +import sys +import tempfile + +RULES = [ + 'hostnossl betacrew all all reject', + 'hostnossl keycloak_betacrew all all reject', + 'hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256', + 'hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256', + 'hostssl betacrew postgres 127.0.0.1/32 scram-sha-256', + 'hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256', + 'hostssl betacrew all all reject', + 'hostssl keycloak_betacrew all all reject', +] + +def apply(path): + if path.is_symlink() or not path.is_file(): + raise ValueError('HBA path must be a regular non-symlink file') + stat = path.stat() + lines = path.read_text().splitlines(keepends=True) + parsed = [(i, line.split('#', 1)[0].split()) for i, line in enumerate(lines)] + scoped = [(i, ' '.join(parts)) for i, parts in parsed if len(parts)>1 and set(parts[1].split(',')) & {'betacrew','keycloak_betacrew'}] + broad = [i for i, parts in parsed if len(parts)>1 and parts[0].startswith('host') and 'all' in parts[1].split(',') and not (len(parts) == 5 and parts[-1] == 'reject' and not set(parts[2].split(',')) & {'all', 'betacrew_app', 'keycloak_betacrew_app', 'postgres'} and not any(x.startswith('+') for x in parts[2].split(',')))] + fallback = [i for i, parts in parsed if parts == 'host all all all scram-sha-256'.split()] + if len(fallback)!=1 or broad != fallback: + raise ValueError('Ambiguous shared fallback rules; manual inspection required') + if scoped: + if [rule for _, rule in scoped] != RULES or max(i for i,_ in scoped)>=fallback[0]: + raise ValueError('Partial, reordered or shadowed BetaCrew policy; refusing mutation') + return + lines.insert(fallback[0], '# BetaCrew TLS and exact source restrictions\n'+'\n'.join(RULES)+'\n') + fd, temporary = tempfile.mkstemp(prefix=path.name+'.betacrew.', dir=path.parent) + try: + with os.fdopen(fd, 'w') as out: + os.fchmod(out.fileno(), stat.st_mode & 0o7777) + if (os.fstat(out.fileno()).st_uid, os.fstat(out.fileno()).st_gid)!=(stat.st_uid,stat.st_gid): + os.fchown(out.fileno(),stat.st_uid,stat.st_gid) + out.write(''.join(lines));out.flush();os.fsync(out.fileno()) + os.replace(temporary,path) + finally: + if os.path.exists(temporary):os.unlink(temporary) + +if __name__=='__main__': + try: apply(Path(sys.argv[1])) + except (ValueError,OSError) as error: + print(str(error),file=sys.stderr);sys.exit(1) diff --git a/scripts/run-betacrew-backup-loop.sh b/scripts/run-betacrew-backup-loop.sh new file mode 100755 index 0000000..fc351dd --- /dev/null +++ b/scripts/run-betacrew-backup-loop.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +interval_seconds=${BETACREW_BACKUP_INTERVAL_SECONDS:-21600} +retry_seconds=${BETACREW_BACKUP_RETRY_SECONDS:-300} +backup_root=${BETACREW_BACKUP_ROOT:-/backups} + +case "${interval_seconds}:${retry_seconds}" in + *[!0-9:]*) echo "BetaCrew backup timing values must be positive integers." >&2; exit 1 ;; +esac +[ "${interval_seconds}" -ge 300 ] && [ "${retry_seconds}" -ge 30 ] || { echo "BetaCrew backup interval or retry delay is too short." >&2; exit 1; } + +if [ "${1:-}" = healthcheck ]; then + status_file=${backup_root}/last-success.json + [ -s "${status_file}" ] || exit 1 + now=$(date +%s) + if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi + [ $((now - modified)) -le $((interval_seconds * 2 + retry_seconds)) ] + exit +fi + +while :; do + if /usr/local/bin/run-betacrew-backup.sh; then + sleep "${interval_seconds}" + else + echo "BetaCrew PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2 + sleep "${retry_seconds}" + fi +done diff --git a/scripts/run-betacrew-backup.sh b/scripts/run-betacrew-backup.sh new file mode 100755 index 0000000..c7442f0 --- /dev/null +++ b/scripts/run-betacrew-backup.sh @@ -0,0 +1,82 @@ +#!/bin/sh +set -eu + +umask 077 +[ "${PGSSLMODE:-verify-full}" = verify-full ] || { echo "BetaCrew backups require certificate-verified TLS." >&2; exit 1; } + +backup_root=${BETACREW_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_SUPERUSER_PASSWORD_FILE:-/run/secrets/postgres_superuser_password} +encryption_cert=${BETACREW_BACKUP_ENCRYPTION_CERT:-/run/config/betacrew_backup_recipient.pem} +retention_days=${BETACREW_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:-127.0.0.1} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:-postgres} + +case "${retention_days}" in + ''|*[!0-9]*) echo "BETACREW_BACKUP_RETENTION_DAYS must be a positive integer." >&2; exit 1 ;; +esac +[ "${retention_days}" -ge 1 ] || { echo "BETACREW_BACKUP_RETENTION_DAYS must be at least 1." >&2; exit 1; } +[ -s "${password_file}" ] || { echo "PostgreSQL backup credential file is missing or empty." >&2; exit 1; } +[ -s "${encryption_cert}" ] || { echo "BetaCrew backup encryption certificate is missing or empty." >&2; exit 1; } +command -v openssl >/dev/null 2>&1 || { echo "openssl is required to encrypt BetaCrew backups." >&2; exit 1; } +openssl x509 -in "${encryption_cert}" -noout >/dev/null 2>&1 || { echo "BetaCrew backup certificate is invalid." >&2; exit 1; } + +mkdir -p "${backup_root}" +chmod 700 "${backup_root}" +lock_dir=${backup_root}/.betacrew-backup.lock +mkdir "${lock_dir}" 2>/dev/null || { echo "Another BetaCrew backup is running." >&2; exit 1; } + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/betacrew-backup.${timestamp}.XXXXXX") +pgpass=${work_dir}/.pgpass + +cleanup() { + if [ -d "${partial_dir}" ]; then find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${partial_dir}" 2>/dev/null || true; fi + if [ -d "${work_dir}" ]; then find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +[ ! -e "${final_dir}" ] && [ ! -L "${final_dir}" ] || { echo "Backup destination already exists." >&2; exit 1; } +password=$(tr -d '\r\n' < "${password_file}") +[ -n "${password}" ] || { echo "PostgreSQL backup credential contains no usable value." >&2; exit 1; } +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" + +mkdir "${partial_dir}" +for database in betacrew keycloak_betacrew; do + dump_path=${work_dir}/${database}.dump + encrypted_path=${partial_dir}/${database}.dump.cms + pg_dump --host="${pg_host}" --port="${pg_port}" --username="${pg_user}" --dbname="${database}" \ + --format=custom --compress=9 --no-owner --no-acl --file="${dump_path}" + pg_restore --list "${dump_path}" >/dev/null + openssl cms -encrypt -binary -outform DER -aes-256-gcm -in "${dump_path}" -out "${encrypted_path}" "${encryption_cert}" + openssl cms -cmsout -inform DER -in "${encrypted_path}" -noout >/dev/null + find "${dump_path}" -delete +done + +(cd "${partial_dir}" && sha256sum betacrew.dump.cms keycloak_betacrew.dump.cms > SHA256SUMS) +cat > "${partial_dir}/metadata.json" <&2; exit 1; } +ln -sfn "${timestamp}" "${backup_root}/latest" +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","databases":["betacrew","keycloak_betacrew"]}\n' "${timestamp}" "${timestamp}" > "${status_tmp}" +chmod 600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_days}" -exec sh -c ' + for directory do find "$directory" -mindepth 1 -delete; rmdir "$directory"; done +' sh {} + + +echo "BetaCrew PostgreSQL backup completed: ${final_dir}" diff --git a/scripts/test-betacrew-backup.sh b/scripts/test-betacrew-backup.sh new file mode 100755 index 0000000..3f0c406 --- /dev/null +++ b/scripts/test-betacrew-backup.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups" +printf 'validation-password\n' > "${work_dir}/password" +chmod 600 "${work_dir}/password" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=betacrew-backup-test' \ + -keyout "${work_dir}/recipient.key" -out "${work_dir}/recipient.pem" >/dev/null 2>&1 + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +for argument in "$@"; do + case "${argument}" in --file=*) output=${argument#--file=} ;; --dbname=*) database=${argument#--dbname=} ;; esac +done +[ -n "${output:-}" ] && [ -n "${database:-}" ] +printf 'validated dump for %s\n' "${database}" > "${output}" +EOF +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "--list" ] && [ -s "$2" ] +EOF +chmod 700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" + +PATH="${work_dir}/bin:${PATH}" BETACREW_BACKUP_ROOT="${work_dir}/backups" \ +POSTGRES_SUPERUSER_PASSWORD_FILE="${work_dir}/password" \ +BETACREW_BACKUP_ENCRYPTION_CERT="${work_dir}/recipient.pem" \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" sh "${repo_root}/scripts/run-betacrew-backup.sh" + +latest=$(readlink "${work_dir}/backups/latest") +backup_dir=${work_dir}/backups/${latest} +for expected in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do test -s "${backup_dir}/${expected}"; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS >/dev/null) +for database in betacrew keycloak_betacrew; do + ! grep -a -q 'validated dump' "${backup_dir}/${database}.dump.cms" + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${work_dir}/recipient.pem" -inkey "${work_dir}/recipient.key" -out "${work_dir}/${database}.decrypted" + grep -q "validated dump for ${database}" "${work_dir}/${database}.decrypted" +done +BETACREW_BACKUP_ROOT="${work_dir}/backups" BETACREW_BACKUP_INTERVAL_SECONDS=300 \ +BETACREW_BACKUP_RETRY_SECONDS=30 sh "${repo_root}/scripts/run-betacrew-backup-loop.sh" healthcheck +echo "BetaCrew PostgreSQL encrypted backup contract passed." diff --git a/scripts/test-betacrew-encrypted-restore.py b/scripts/test-betacrew-encrypted-restore.py new file mode 100644 index 0000000..ddbc7df --- /dev/null +++ b/scripts/test-betacrew-encrypted-restore.py @@ -0,0 +1,63 @@ +"""Exercise real pg_dump/CMS/pg_restore over verified TLS in disposable PostgreSQL.""" +import os +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + +SCRIPTS=Path(__file__).resolve().parent +IMAGE='postgres@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea' +container='betacrew-encrypted-test-'+uuid.uuid4().hex[:10] + +def run(*args, **kw): + result=subprocess.run(args,capture_output=True,text=True,**kw) + if result.returncode: + raise RuntimeError('Disposable fixture operation failed: '+result.stderr) + return result.stdout.strip() + +with tempfile.TemporaryDirectory(prefix='betacrew-encrypted-') as tmp: + root=Path(tmp);root.chmod(0o755) + cert=root/'server.crt';key=root/'server.key' + run('openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost,IP:127.0.0.1','-keyout',str(key),'-out',str(cert)) + # Separate recipient key from server transport credentials. + recipient=root/'recipient.pem';private=root/'recipient.key' + run('openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=backup-test','-keyout',str(private),'-out',str(recipient)) + password=root/'password';password.write_text('disposable-fixture-password\n');password.chmod(0o600) + bin=root/'bin';bin.mkdir();work=root/'tmp';work.mkdir() + for tool in ['pg_dump','pg_restore','psql']: + (bin/tool).write_text('#!/bin/sh\nexec docker exec -i -e PGPASSFILE -e PGSERVICEFILE -e PGSSLMODE -e PGSSLROOTCERT "$TEST_CONTAINER" '+tool+' "$@"\n') + (bin/tool).chmod(0o700) + env=dict(os.environ,PATH=str(bin)+os.pathsep+os.environ['PATH'],TEST_CONTAINER=container,TMPDIR=str(work),PGUSER='postgres',PGHOST='127.0.0.1',PGSSLMODE='verify-full',PGSSLROOTCERT=str(cert),POSTGRES_SUPERUSER_PASSWORD_FILE=str(password),BETACREW_BACKUP_ENCRYPTION_CERT=str(recipient),BETACREW_BACKUP_ROOT=str(root/'backups')) + def sql(database,query): + return run('docker','exec','-i',container,'psql','-U','postgres','-d',database,'-XAt','-v','ON_ERROR_STOP=1',input=query) + try: + run('docker','run','-d','--network','none','--name',container,'--tmpfs','/var/lib/postgresql/data','--volume',tmp+':'+tmp,'-e','POSTGRES_PASSWORD=disposable-fixture-password',IMAGE) + for _ in range(30): + if subprocess.run(['docker','exec',container,'pg_isready','-U','postgres'],capture_output=True).returncode==0:break + time.sleep(1) + run('docker','cp',str(cert),container+':/tmp/server.crt');run('docker','cp',str(key),container+':/tmp/server.key') + run('docker','exec',container,'chown','postgres:postgres','/tmp/server.crt','/tmp/server.key') + run('docker','exec',container,'chmod','600','/tmp/server.key') + sql('postgres',"ALTER SYSTEM SET ssl='on';\nALTER SYSTEM SET ssl_cert_file='/tmp/server.crt';\nALTER SYSTEM SET ssl_key_file='/tmp/server.key';\nSELECT pg_reload_conf();") + sql('postgres','CREATE DATABASE betacrew; CREATE DATABASE keycloak_betacrew; CREATE DATABASE betacrew_restore_test; CREATE DATABASE keycloak_betacrew_restore_test;') + sql('betacrew',"CREATE TABLE schema_migrations(version text); INSERT INTO schema_migrations VALUES ('real-backup-test');") + sql('keycloak_betacrew',"CREATE TABLE realm(id text); INSERT INTO realm VALUES ('isolated-identity-test');") + run('sh',str(SCRIPTS/'run-betacrew-backup.sh'),env=env) + backup=(root/'backups/latest').resolve() + services=root/'services.conf' + services.write_text(''.join(f'[{name}]\nhost=127.0.0.1\nuser=postgres\npassword=disposable-fixture-password\ndbname={db}\nsslrootcert={cert}\n' for name,db in [('app','betacrew_restore_test'),('identity','keycloak_betacrew_restore_test')])) + services.chmod(0o600) + env.update(PGSERVICEFILE=str(services),BETACREW_RESTORE_SERVICE='app',KEYCLOAK_BETACREW_RESTORE_SERVICE='identity',BETACREW_RESTORE_CONFIRM='replace-nonproduction-restore-targets',BETACREW_BACKUP_DECRYPTION_CERT=str(recipient),BETACREW_BACKUP_DECRYPTION_KEY=str(private)) + run('bash',str(SCRIPTS/'verify-betacrew-restore.sh'),str(backup),env=env) + assert sql('betacrew_restore_test','SELECT version FROM schema_migrations')=='real-backup-test' + assert sql('keycloak_betacrew_restore_test','SELECT id FROM realm')=='isolated-identity-test' + # Re-running against populated destinations must fail before replacing rows. + failed=subprocess.run(['bash',str(SCRIPTS/'verify-betacrew-restore.sh'),str(backup)],env=env,capture_output=True) + assert failed.returncode != 0 + assert sql('betacrew_restore_test','SELECT version FROM schema_migrations')=='real-backup-test' + # Disabling TLS cannot create an allegedly verified backup. + assert subprocess.run(['sh',str(SCRIPTS/'run-betacrew-backup.sh')],env=dict(env,PGSSLMODE='disable'),capture_output=True).returncode != 0 + print('PASS: real PostgreSQL dumps, verified TLS, CMS encryption, both restored datasets, nonempty-target preservation and TLS-downgrade rejection') + finally: + subprocess.run(['docker','rm','-f',container],capture_output=True) diff --git a/scripts/test-betacrew-safety.py b/scripts/test-betacrew-safety.py new file mode 100644 index 0000000..e42bd02 --- /dev/null +++ b/scripts/test-betacrew-safety.py @@ -0,0 +1,106 @@ +import importlib.util +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +import time +import uuid + +ROOT=Path(__file__).resolve().parent +spec=importlib.util.spec_from_file_location('hba',ROOT/'betacrew-hba.py') +hba=importlib.util.module_from_spec(spec);spec.loader.exec_module(hba) + +class HBA(unittest.TestCase): + def test_shipped_policy_preserves_neighbor_denials(self): + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba'; p.write_text((ROOT.parent/'config/runtrace-pg_hba.conf').read_text()) + before=p.read_bytes(); hba.apply(p); self.assertEqual(before,p.read_bytes()) + + def test_rerun_preserves_neighbor_rules_and_permissions(self): + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';original='local all all trust\nhostssl other other 10.1.0.1/32 scram-sha-256\nhost all all all scram-sha-256\n';p.write_text(original);p.chmod(0o640) + hba.apply(p);first=p.read_bytes();hba.apply(p) + self.assertEqual(first,p.read_bytes());self.assertEqual(p.stat().st_mode&0o777,0o640) + self.assertIn('hostssl other other 10.1.0.1/32 scram-sha-256',p.read_text()) + def test_each_missing_rule_rejected_without_write(self): + for i in range(len(hba.RULES)): + with self.subTest(rule=i),tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text('\n'.join(hba.RULES[:i]+hba.RULES[i+1:])+ '\nhost all all all scram-sha-256\n');before=p.read_bytes() + with self.assertRaises(ValueError):hba.apply(p) + self.assertEqual(before,p.read_bytes()) + def test_shadowed_block_and_earlier_broad_allow_rejected(self): + for content in ['host all all all scram-sha-256\n'+'\n'.join(hba.RULES), 'hostssl all all 0.0.0.0/0 trust\n'+'\n'.join(hba.RULES)+'\nhost all all all scram-sha-256\n']: + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text(content) + with self.assertRaises(ValueError):hba.apply(p) + +class Restore(unittest.TestCase): + def test_rejects_destinations_before_any_restore(self): + for reply,service in [('f','app'),('error','app'),('t','app dbname=betacrew'),('second-f','app')]: + with self.subTest(reply=reply),tempfile.TemporaryDirectory() as t: + p=Path(t);bin=p/'bin';bin.mkdir();backup=p/'backup';backup.mkdir() + for name in ['betacrew.dump.cms','keycloak_betacrew.dump.cms','SHA256SUMS','metadata.json']:(backup/name).write_text('fixture') + for name in ['service','cert','key']:(p/name).write_text('fixture') + (bin/'sha256sum').write_text('#!/bin/sh\nexit 0\n') + (bin/'psql').write_text('''#!/bin/sh +cat > "$TEST_ROOT/query" +printf '%s\\n' "$*" >> "$TEST_ROOT/args" +case "$TEST_REPLY" in +error) exit 1;; +second-f) case "$*" in *service=identity*) echo f;; *) echo t;; esac;; +*) echo "$TEST_REPLY";; +esac +''') + for name in ['pg_restore','openssl']:(bin/name).write_text('#!/bin/sh\ntouch "$TEST_ROOT/mutation"\nexit 1\n') + for f in bin.iterdir():f.chmod(0o700) + env=dict(os.environ,PATH=str(bin)+os.pathsep+os.environ['PATH'],TEST_ROOT=t,TEST_REPLY=reply,PGSERVICEFILE=str(p/'service'),BETACREW_BACKUP_DECRYPTION_CERT=str(p/'cert'),BETACREW_BACKUP_DECRYPTION_KEY=str(p/'key'),BETACREW_RESTORE_SERVICE=service,KEYCLOAK_BETACREW_RESTORE_SERVICE='identity',BETACREW_RESTORE_CONFIRM='replace-nonproduction-restore-targets') + result=subprocess.run(['bash',str(ROOT/'verify-betacrew-restore.sh'),str(backup)],env=env,capture_output=True) + self.assertNotEqual(result.returncode,0);self.assertFalse((p/'mutation').exists()) + if (p/'args').exists():self.assertIn('sslmode=verify-full',(p/'args').read_text()) + +class DatabaseGuard(unittest.TestCase): + def test_actual_database_name_and_nonempty_objects(self): + container='betacrew-restore-guard-'+uuid.uuid4().hex[:10] + image='postgres@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea' + def command(*args,input=None): + return subprocess.run(args,input=input,text=True,capture_output=True,check=True).stdout.strip() + query=(ROOT/'verify-betacrew-restore.sh').read_text().split("<<'SQL'\n",1)[1].split('\nSQL',1)[0] + def sql(database,text,expected='betacrew_restore_test'): + return command('docker','exec','-i',container,'psql','-U','postgres','-d',database,'-XAt','-v','ON_ERROR_STOP=1','-v','expected_database='+expected,input=text) + try: + command('docker','run','-d','--network','none','--name',container,'--tmpfs','/var/lib/postgresql/data','-e','POSTGRES_PASSWORD=disposable-fixture-password',image) + for _ in range(30): + if subprocess.run(['docker','exec',container,'pg_isready','-U','postgres'],capture_output=True).returncode==0:break + time.sleep(1) + sql('postgres','CREATE DATABASE betacrew; CREATE DATABASE betacrew_restore_test; CREATE DATABASE keycloak_betacrew_restore_test;') + self.assertEqual(sql('betacrew',query),'f') + self.assertEqual(sql('betacrew_restore_test',query),'t') + self.assertEqual(sql('keycloak_betacrew_restore_test',query),'f') + self.assertEqual(sql('keycloak_betacrew_restore_test',query,'keycloak_betacrew_restore_test'),'t') + for ddl,cleanup in [('CREATE TABLE sentinel(id int)','DROP TABLE sentinel'),('CREATE SEQUENCE sentinel','DROP SEQUENCE sentinel'),('CREATE VIEW sentinel AS SELECT 1','DROP VIEW sentinel')]: + sql('betacrew_restore_test',ddl) + self.assertEqual(sql('betacrew_restore_test',query),'f') + sql('betacrew_restore_test',cleanup) + sql('postgres','DROP DATABASE betacrew;') + bootstrap=(ROOT.parent/'bootstrap/betacrew-databases.sql').read_text() + def provision(password='disposable-app-password-at-least-32-characters'): + return command('docker','exec','-i',container,'psql','-U','postgres','-d','postgres','-XAt','-v','ON_ERROR_STOP=1','-v','betacrew_app_password='+password,'-v','keycloak_betacrew_app_password=disposable-identity-password-at-least-32-characters',input=bootstrap) + with self.assertRaises(subprocess.CalledProcessError):provision('weak') + self.assertEqual(sql('postgres',"SELECT count(*) FROM pg_roles WHERE rolname='betacrew_app'"),'0') + provision() + original=sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='betacrew_app'") + provision('different-password-must-not-replace-live-credential') + self.assertEqual(original,sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='betacrew_app'")) + self.assertEqual(sql('postgres',"SELECT has_database_privilege('betacrew_app','keycloak_betacrew','CONNECT')"),'f') + self.assertEqual(sql('postgres',"SELECT has_database_privilege('keycloak_betacrew_app','betacrew','CONNECT')"),'f') + sql('postgres','ALTER ROLE keycloak_betacrew_app SUPERUSER;') + with self.assertRaises(subprocess.CalledProcessError):provision() + self.assertEqual(original,sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='betacrew_app'")) + sql('postgres','ALTER ROLE keycloak_betacrew_app NOSUPERUSER; GRANT CONNECT ON DATABASE betacrew TO PUBLIC;') + with self.assertRaises(subprocess.CalledProcessError):provision() + + finally: + subprocess.run(['docker','rm','-f',container],capture_output=True) + +if __name__=='__main__':unittest.main() diff --git a/scripts/validate-betacrew-config.sh b/scripts/validate-betacrew-config.sh new file mode 100755 index 0000000..e4ed5fa --- /dev/null +++ b/scripts/validate-betacrew-config.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +require_literal() { grep -Fq -- "$2" "${repo_root}/$1" || { echo "Missing BetaCrew contract in $1: $2" >&2; exit 1; }; } + +require_literal bootstrap/betacrew-databases.sql "pg_advisory_lock" +require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE betacrew OWNER betacrew_app" +require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app" +require_literal config/runtrace-pg_hba.conf "hostssl betacrew betacrew_app 10.80.0.1/32" +require_literal config/runtrace-pg_hba.conf "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32" +require_literal compose.host.yml "betacrew_backup:" +require_literal compose.betacrew.host.yml "betacrew_backup:" +require_literal compose.host.yml "BETACREW_BACKUP_INTERVAL_SECONDS" +require_literal compose.host.yml "BETACREW_BACKUP_ENCRYPTION_CERT" +require_literal envs/production/.env.db "MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600" +require_literal scripts/run-betacrew-backup.sh "-aes-256-gcm" +require_literal scripts/verify-betacrew-restore.sh "replace-nonproduction-restore-targets" +require_literal scripts/verify-betacrew-restore.sh "--single-transaction" +bash -n "${repo_root}/scripts/apply-betacrew-hba.sh" +echo "BetaCrew PostgreSQL configuration is valid." diff --git a/scripts/verify-betacrew-restore.sh b/scripts/verify-betacrew-restore.sh new file mode 100755 index 0000000..f83e132 --- /dev/null +++ b/scripts/verify-betacrew-restore.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +backup_dir=${1:?Usage: verify-betacrew-restore.sh } +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a root-owned libpq service file}" +: "${BETACREW_RESTORE_SERVICE:?BETACREW_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_BETACREW_RESTORE_SERVICE:?KEYCLOAK_BETACREW_RESTORE_SERVICE must name an empty non-production database}" +: "${BETACREW_RESTORE_CONFIRM:?set BETACREW_RESTORE_CONFIRM=replace-nonproduction-restore-targets}" +: "${BETACREW_BACKUP_DECRYPTION_CERT:?BETACREW_BACKUP_DECRYPTION_CERT must identify the recipient certificate}" +: "${BETACREW_BACKUP_DECRYPTION_KEY:?BETACREW_BACKUP_DECRYPTION_KEY must identify the offline private key}" + +[[ "${BETACREW_RESTORE_CONFIRM}" == replace-nonproduction-restore-targets ]] || { echo "Restore requires explicit non-production confirmation." >&2; exit 1; } +for path in "${PGSERVICEFILE}" "${BETACREW_BACKUP_DECRYPTION_CERT}" "${BETACREW_BACKUP_DECRYPTION_KEY}"; do + [[ -f "${path}" && ! -L "${path}" ]] || { echo "Restore input must be a regular non-symlink file: ${path}" >&2; exit 1; } +done +[[ -d "${backup_dir}" && ! -L "${backup_dir}" ]] || { echo "Backup directory is invalid." >&2; exit 1; } +for required in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do [[ -s "${backup_dir}/${required}" ]] || exit 1; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS) + +# Validate both actual destinations before decrypting or restoring either dump. +# Service names are identifiers, never arbitrary libpq connection strings. +for service in "$BETACREW_RESTORE_SERVICE" "$KEYCLOAK_BETACREW_RESTORE_SERVICE"; do + [[ "$service" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "Invalid restore service identifier." >&2; exit 1; } +done +verify_target() { + local service=$1 expected=$2 result + result=$(psql "service=$service sslmode=verify-full" -X -v ON_ERROR_STOP=1 -At \ + -v expected_database="$expected" <<'SQL' +SELECT current_database() = :'expected_database' + AND NOT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + AND c.relkind IN ('r','p','v','m','S','f') + ); +SQL + ) + [[ "$result" == t ]] || { echo "Restore target must be the named empty disposable database." >&2; exit 1; } +} +verify_target "$BETACREW_RESTORE_SERVICE" betacrew_restore_test +verify_target "$KEYCLOAK_BETACREW_RESTORE_SERVICE" keycloak_betacrew_restore_test + +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/betacrew-restore.XXXXXX") +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT HUP INT TERM + +for database in betacrew keycloak_betacrew; do + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${BETACREW_BACKUP_DECRYPTION_CERT}" -inkey "${BETACREW_BACKUP_DECRYPTION_KEY}" \ + -out "${work_dir}/${database}.dump" + pg_restore --list "${work_dir}/${database}.dump" >/dev/null +done + +pg_restore --dbname="service=${BETACREW_RESTORE_SERVICE} sslmode=verify-full" --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/betacrew.dump" +pg_restore --dbname="service=${KEYCLOAK_BETACREW_RESTORE_SERVICE} sslmode=verify-full" --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/keycloak_betacrew.dump" + +[[ $(psql "service=${BETACREW_RESTORE_SERVICE} sslmode=verify-full" -X -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.schema_migrations') IS NOT NULL;") == t ]] +[[ $(psql "service=${KEYCLOAK_BETACREW_RESTORE_SERVICE} sslmode=verify-full" -X -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.realm') IS NOT NULL;") == t ]] +echo "BetaCrew and Keycloak restore verification completed against non-production targets."