diff --git a/.github/workflows/jotwink-safety.yml b/.github/workflows/jotwink-safety.yml new file mode 100644 index 0000000..60ed7f1 --- /dev/null +++ b/.github/workflows/jotwink-safety.yml @@ -0,0 +1,28 @@ +name: Jotwink database safety +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] +permissions: + contents: read +jobs: + safety: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: [self-hosted, linux, x64, makepad] + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + - name: Database bootstrap, HBA and restore guard regression tests + run: python3 scripts/test-jotwink-safety.py + - name: Real encrypted database restore over verified TLS + run: python3 scripts/test-jotwink-encrypted-restore.py + - name: Encryption contract (mock database dump) + run: bash scripts/test-jotwink-backup.sh + - name: Configuration and shell syntax + run: | + bash scripts/validate-jotwink-config.sh + bash -n scripts/verify-jotwink-restore.sh diff --git a/README.md b/README.md index cfa2aa2..4fe7c03 100644 --- a/README.md +++ b/README.md @@ -911,3 +911,45 @@ logical backup using the previous image. CartHop bootstrap and Runtrace TLS fixtures also use the production candidate. These fixtures do not replace a fresh verified backup of every production database or a scoped rollout and rollback plan for the shared service. + +## Jotwink Backup And Restore + +Jotwink has an independent backup service and health timestamp so a failed +Jotwink dump cannot be hidden by a successful Runtrace dump. It creates +custom-format dumps of `jotwink` and `keycloak_jotwink` every six hours, +validates each archive, encrypts it in ephemeral tmpfs with OpenSSL CMS and +AES-256-GCM before it reaches persistent storage, writes SHA-256 checksums, and +retains 35 days by default. Provision its root and a public recipient +certificate before starting `compose.host.yml`: + +```bash +sudo install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/jotwink +sudo install -d -o root -g root -m 0755 /etc/makepad/postgres-backup +sudo install -o root -g root -m 0444 jotwink-recipient.pem /etc/makepad/postgres-backup/jotwink-recipient.pem +``` + +Use a dedicated mode-0400 backup credential file owned by uid 999 and set +`MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH` to that file. Keep the +matching private key offline; it is used only during a controlled restore drill. + +Replicate every completed timestamp directory and `last-success.json` to an +off-host, separately administered encrypted store. Alert before the health +timestamp exceeds two intervals. At least quarterly, and before launch or a +database upgrade, restore the newest artifacts into empty non-production +databases: + +```bash +export PGSERVICEFILE=/etc/makepad/postgres-restore-services.conf +export JOTWINK_RESTORE_SERVICE=jotwink_restore_test +export KEYCLOAK_JOTWINK_RESTORE_SERVICE=keycloak_jotwink_restore_test +export JOTWINK_RESTORE_CONFIRM=replace-nonproduction-restore-targets +export JOTWINK_BACKUP_DECRYPTION_CERT=/secure/jotwink-recipient.pem +export JOTWINK_BACKUP_DECRYPTION_KEY=/secure/jotwink-recipient.key +scripts/verify-jotwink-restore.sh /var/lib/makepad/postgres-backups/jotwink/ +``` + +Record the artifact checksum, elapsed time, recovery-point age, and operator. +The verifier checks both the app migration table and Keycloak realm table. + + +Jotwink host backup is opt-in through the `jotwink` Compose profile. Provision and validate its recipient, credentials, TLS and restore evidence before enabling that profile. The shared deployment workflow preserves its existing release transaction. diff --git a/bootstrap/jotwink-databases.sql b/bootstrap/jotwink-databases.sql new file mode 100644 index 0000000..af42f43 --- /dev/null +++ b/bootstrap/jotwink-databases.sql @@ -0,0 +1,67 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql +-- variables so credentials never enter source control. +\if :{?jotwink_app_password} +\else + \echo 'missing required psql variable: jotwink_app_password' + SELECT 1/0; +\endif +\if :{?keycloak_jotwink_app_password} +\else + \echo 'missing required psql variable: keycloak_jotwink_app_password' + SELECT 1/0; +\endif + +SELECT length(:'jotwink_app_password') >= 32 AS jotwink_password_ok \gset +\if :jotwink_password_ok +\else + \echo 'empty required psql variable: jotwink_app_password' + SELECT 1/0; +\endif +SELECT length(:'keycloak_jotwink_app_password') >= 32 AS keycloak_jotwink_password_ok \gset +\if :keycloak_jotwink_password_ok +\else + \echo 'empty required psql variable: keycloak_jotwink_app_password' + SELECT 1/0; +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('jotwink-databases-bootstrap')); + +-- Validate both pairs before changing either. Existing objects must already be +-- isolated; never seize ownership or rotate a live credential during bootstrap. +DO $$ +DECLARE target record; role_row pg_roles%ROWTYPE; database_row pg_database%ROWTYPE; +BEGIN + FOR target IN SELECT * FROM (VALUES + ('jotwink','jotwink_app'),('keycloak_jotwink','keycloak_jotwink_app') + ) AS targets(database_name,role_name) LOOP + SELECT * INTO role_row FROM pg_roles WHERE rolname=target.role_name; + SELECT * INTO database_row FROM pg_database WHERE datname=target.database_name; + IF (role_row.oid IS NULL) <> (database_row.oid IS NULL) THEN + RAISE EXCEPTION 'Incomplete existing Jotwink role/database pair'; + END IF; + IF role_row.oid IS NOT NULL THEN + IF NOT role_row.rolcanlogin OR role_row.rolsuper OR role_row.rolcreatedb + OR role_row.rolcreaterole OR role_row.rolreplication OR role_row.rolbypassrls + OR EXISTS(SELECT 1 FROM pg_auth_members WHERE member=role_row.oid OR roleid=role_row.oid) + OR database_row.datdba<>role_row.oid + OR EXISTS(SELECT 1 FROM aclexplode(COALESCE(database_row.datacl,acldefault('d',database_row.datdba))) WHERE grantee<>role_row.oid) + THEN RAISE EXCEPTION 'Existing Jotwink privileges, membership, ownership or ACL require manual inspection'; + END IF; + END IF; + END LOOP; +END $$; + +SELECT format('CREATE ROLE jotwink_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'jotwink_app_password') +WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='jotwink_app') \gexec +SELECT 'CREATE DATABASE jotwink OWNER jotwink_app' +WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='jotwink') \gexec +REVOKE ALL ON DATABASE jotwink FROM PUBLIC; +SELECT format('CREATE ROLE keycloak_jotwink_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'keycloak_jotwink_app_password') +WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='keycloak_jotwink_app') \gexec +SELECT 'CREATE DATABASE keycloak_jotwink OWNER keycloak_jotwink_app' +WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='keycloak_jotwink') \gexec +REVOKE ALL ON DATABASE keycloak_jotwink FROM PUBLIC; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('jotwink-databases-bootstrap')); diff --git a/compose.host.yml b/compose.host.yml index 978b063..dd32d2b 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -132,3 +132,52 @@ services: options: max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + + jotwink_backup: + image: ${JOTWINK_BACKUP_IMAGE:-postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825} + profiles: [jotwink] + user: "999:999" + network_mode: host + command: + - /usr/local/bin/run-jotwink-backup-loop.sh + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + JOTWINK_BACKUP_ENCRYPTION_CERT: /run/config/jotwink_backup_recipient.pem + JOTWINK_BACKUP_ROOT: /backups + JOTWINK_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS:-21600} + JOTWINK_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS:-300} + JOTWINK_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=999,gid=999 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + volumes: + - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH:-/var/lib/makepad/postgres-backups/jotwink}:/backups" + - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/jotwink-recipient.pem}:/run/config/jotwink_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-jotwink-backup.sh}:/usr/local/bin/run-jotwink-backup.sh:ro" + - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-jotwink-backup-loop.sh}:/usr/local/bin/run-jotwink-backup-loop.sh:ro" + healthcheck: + test: ["CMD", "/usr/local/bin/run-jotwink-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/compose.yml b/compose.yml index 328e229..7ff32ea 100644 --- a/compose.yml +++ b/compose.yml @@ -56,6 +56,10 @@ configs: file: ./scripts/run-runtrace-backup.sh runtrace_backup_loop_script: file: ./scripts/run-runtrace-backup-loop.sh + jotwink_backup_script: + file: ./scripts/run-jotwink-backup.sh + jotwink_backup_loop_script: + file: ./scripts/run-jotwink-backup-loop.sh brio_encrypted_backup_script: file: ./scripts/run-brio-encrypted-backup.sh brio_encrypted_backup_loop_script: diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index a1d759e..08be544 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -1,5 +1,6 @@ -# Runtrace customer data and identity traffic must use TLS. Other shared -# databases retain their current SCRAM transport policy during migration. +# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also +# source-restricted to the application WireGuard peer and Keycloak host. +# Other shared databases retain their current SCRAM policy during migration. local all all trust hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject @@ -7,6 +8,14 @@ hostnossl brio_staging all all reject hostnossl keycloak_brio_staging all all reject hostssl runtrace all all scram-sha-256 hostssl keycloak_runtrace all all scram-sha-256 +hostnossl jotwink all all reject +hostnossl keycloak_jotwink all all reject +hostssl jotwink jotwink_app 10.80.0.1/32 scram-sha-256 +hostssl keycloak_jotwink keycloak_jotwink_app 88.99.209.165/32 scram-sha-256 +hostssl jotwink postgres 127.0.0.1/32 scram-sha-256 +hostssl keycloak_jotwink postgres 127.0.0.1/32 scram-sha-256 +hostssl jotwink all all reject +hostssl keycloak_jotwink all all reject hostssl brio_staging brio_staging_app all scram-sha-256 hostssl brio_staging brio_staging_backup all scram-sha-256 hostssl keycloak_brio_staging keycloak_brio_staging_app all scram-sha-256 diff --git a/envs/production/.env.db b/envs/production/.env.db index 07e312a..ddc9168 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -37,3 +37,11 @@ MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE=20m MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES=5 MAKEPAD_POSTGRES_LOG_MAX_SIZE=20m MAKEPAD_POSTGRES_LOG_MAX_FILES=5 +JOTWINK_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825 +MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH=/var/lib/makepad/postgres-backups/jotwink +MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password +MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_CONFIG=makepad_postgres_jotwink_backup_recipient_v1 +MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/jotwink-recipient.pem +MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35 diff --git a/scripts/jotwink-hba.py b/scripts/jotwink-hba.py new file mode 100644 index 0000000..4eab696 --- /dev/null +++ b/scripts/jotwink-hba.py @@ -0,0 +1,48 @@ +"""Add or validate the complete Jotwink HBA block without altering other rules.""" +import os +from pathlib import Path +import sys +import tempfile + +RULES = [ + 'hostnossl jotwink all all reject', + 'hostnossl keycloak_jotwink all all reject', + 'hostssl jotwink jotwink_app 10.80.0.1/32 scram-sha-256', + 'hostssl keycloak_jotwink keycloak_jotwink_app 88.99.209.165/32 scram-sha-256', + 'hostssl jotwink postgres 127.0.0.1/32 scram-sha-256', + 'hostssl keycloak_jotwink postgres 127.0.0.1/32 scram-sha-256', + 'hostssl jotwink all all reject', + 'hostssl keycloak_jotwink all all reject', +] + +def apply(path): + if path.is_symlink() or not path.is_file(): + raise ValueError('HBA path must be a regular non-symlink file') + stat = path.stat() + lines = path.read_text().splitlines(keepends=True) + parsed = [(i, line.split('#', 1)[0].split()) for i, line in enumerate(lines)] + scoped = [(i, ' '.join(parts)) for i, parts in parsed if len(parts)>1 and set(parts[1].split(',')) & {'jotwink','keycloak_jotwink'}] + broad = [i for i, parts in parsed if len(parts)>1 and parts[0].startswith('host') and 'all' in parts[1].split(',') and not (len(parts) == 5 and parts[-1] == 'reject' and not set(parts[2].split(',')) & {'all', 'jotwink_app', 'keycloak_jotwink_app', 'postgres'} and not any(x.startswith('+') for x in parts[2].split(',')))] + fallback = [i for i, parts in parsed if parts == 'host all all all scram-sha-256'.split()] + if len(fallback)!=1 or broad != fallback: + raise ValueError('Ambiguous shared fallback rules; manual inspection required') + if scoped: + if [rule for _, rule in scoped] != RULES or max(i for i,_ in scoped)>=fallback[0]: + raise ValueError('Partial, reordered or shadowed Jotwink policy; refusing mutation') + return + lines.insert(fallback[0], '# Jotwink TLS and exact source restrictions\n'+'\n'.join(RULES)+'\n') + fd, temporary = tempfile.mkstemp(prefix=path.name+'.jotwink.', dir=path.parent) + try: + with os.fdopen(fd, 'w') as out: + os.fchmod(out.fileno(), stat.st_mode & 0o7777) + if (os.fstat(out.fileno()).st_uid, os.fstat(out.fileno()).st_gid)!=(stat.st_uid,stat.st_gid): + os.fchown(out.fileno(),stat.st_uid,stat.st_gid) + out.write(''.join(lines));out.flush();os.fsync(out.fileno()) + os.replace(temporary,path) + finally: + if os.path.exists(temporary):os.unlink(temporary) + +if __name__=='__main__': + try: apply(Path(sys.argv[1])) + except (ValueError,OSError) as error: + print(str(error),file=sys.stderr);sys.exit(1) diff --git a/scripts/run-jotwink-backup-loop.sh b/scripts/run-jotwink-backup-loop.sh new file mode 100755 index 0000000..6930603 --- /dev/null +++ b/scripts/run-jotwink-backup-loop.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +interval_seconds=${JOTWINK_BACKUP_INTERVAL_SECONDS:-21600} +retry_seconds=${JOTWINK_BACKUP_RETRY_SECONDS:-300} +backup_root=${JOTWINK_BACKUP_ROOT:-/backups} + +validate_timing_value() { + case "$2" in + ''|*[!0-9]*) echo "$1 must be a positive integer." >&2; exit 1 ;; + esac +} +validate_timing_value JOTWINK_BACKUP_INTERVAL_SECONDS "${interval_seconds}" +validate_timing_value JOTWINK_BACKUP_RETRY_SECONDS "${retry_seconds}" +if [ "${interval_seconds}" -lt 300 ] || [ "${retry_seconds}" -lt 30 ]; then + echo "Jotwink backup interval must be at least 300 seconds and retry delay at least 30 seconds." >&2 + exit 1 +fi + +if [ "${1:-}" = "healthcheck" ]; then + status_file=${backup_root}/last-success.json + [ -s "${status_file}" ] || exit 1 + now=$(date +%s) + if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi + max_age=$((interval_seconds * 2 + retry_seconds)) + [ $((now - modified)) -le "${max_age}" ] + exit +fi + +while :; do + if /usr/local/bin/run-jotwink-backup.sh; then + sleep "${interval_seconds}" + else + echo "Jotwink PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2 + sleep "${retry_seconds}" + fi +done diff --git a/scripts/run-jotwink-backup.sh b/scripts/run-jotwink-backup.sh new file mode 100755 index 0000000..263fc8d --- /dev/null +++ b/scripts/run-jotwink-backup.sh @@ -0,0 +1,113 @@ +#!/bin/sh +set -eu + +umask 077 +[ "${PGSSLMODE:-verify-full}" = verify-full ] || { echo "Jotwink backups require certificate-verified TLS." >&2; exit 1; } + +backup_root=${JOTWINK_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_SUPERUSER_PASSWORD_FILE:-/run/secrets/postgres_superuser_password} +encryption_cert=${JOTWINK_BACKUP_ENCRYPTION_CERT:-/run/config/jotwink_backup_recipient.pem} +retention_days=${JOTWINK_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:-127.0.0.1} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:-postgres} + +case "${retention_days}" in + ''|*[!0-9]*) echo "JOTWINK_BACKUP_RETENTION_DAYS must be a positive integer." >&2; exit 1 ;; +esac +[ "${retention_days}" -ge 1 ] || { echo "JOTWINK_BACKUP_RETENTION_DAYS must be at least 1." >&2; exit 1; } +[ -s "${password_file}" ] || { echo "PostgreSQL backup credential file is missing or empty." >&2; exit 1; } +[ -s "${encryption_cert}" ] || { echo "Jotwink backup encryption certificate is missing or empty." >&2; exit 1; } +command -v openssl >/dev/null 2>&1 || { echo "openssl is required to encrypt Jotwink backups." >&2; exit 1; } +openssl x509 -in "${encryption_cert}" -noout >/dev/null 2>&1 || { echo "Jotwink backup encryption certificate is not a valid PEM X.509 certificate." >&2; exit 1; } + +mkdir -p "${backup_root}" +chmod 700 "${backup_root}" +lock_dir=${backup_root}/.jotwink-backup.lock +if ! mkdir "${lock_dir}" 2>/dev/null; then + echo "Another Jotwink PostgreSQL backup is already running." >&2 + exit 1 +fi + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/jotwink-backup.${timestamp}.XXXXXX") +pgpass=${work_dir}/.pgpass + +cleanup() { + if [ -d "${partial_dir}" ]; then + find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${partial_dir}" 2>/dev/null || true + fi + if [ -d "${work_dir}" ]; then + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true + fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +[ ! -e "${final_dir}" ] && [ ! -L "${final_dir}" ] || { echo "Backup destination already exists for ${timestamp}." >&2; exit 1; } +password=$(tr -d '\r\n' < "${password_file}") +[ -n "${password}" ] || { echo "PostgreSQL backup credential file contains no usable value." >&2; exit 1; } +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" + +mkdir "${partial_dir}" +for database in jotwink keycloak_jotwink; do + dump_path=${work_dir}/${database}.dump + encrypted_path=${partial_dir}/${database}.dump.cms + pg_dump \ + --host="${pg_host}" \ + --port="${pg_port}" \ + --username="${pg_user}" \ + --dbname="${database}" \ + --format=custom \ + --compress=9 \ + --no-owner \ + --no-acl \ + --file="${dump_path}" + pg_restore --list "${dump_path}" >/dev/null + openssl cms \ + -encrypt \ + -binary \ + -outform DER \ + -aes-256-gcm \ + -in "${dump_path}" \ + -out "${encrypted_path}" \ + "${encryption_cert}" + openssl cms -cmsout -inform DER -in "${encrypted_path}" -noout >/dev/null + find "${dump_path}" -delete +done + +( + cd "${partial_dir}" + sha256sum jotwink.dump.cms keycloak_jotwink.dump.cms > SHA256SUMS +) +cat > "${partial_dir}/metadata.json" <&2; exit 1; } +ln -sfn "${timestamp}" "${backup_root}/latest" + +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","databases":["jotwink","keycloak_jotwink"]}\n' "${timestamp}" "${timestamp}" > "${status_tmp}" +chmod 600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_days}" -exec sh -c ' + for directory do + find "$directory" -mindepth 1 -delete + rmdir "$directory" + done +' sh {} + + +echo "Jotwink PostgreSQL backup completed: ${final_dir}" diff --git a/scripts/test-jotwink-backup.sh b/scripts/test-jotwink-backup.sh new file mode 100755 index 0000000..8c054f7 --- /dev/null +++ b/scripts/test-jotwink-backup.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) + +cleanup() { + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups" +printf 'validation-password\n' > "${work_dir}/password" +chmod 600 "${work_dir}/password" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=jotwink-backup-test' \ + -keyout "${work_dir}/recipient.key" -out "${work_dir}/recipient.pem" >/dev/null 2>&1 + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +output= +database= +for argument in "$@"; do + case "${argument}" in + --file=*) output=${argument#--file=} ;; + --dbname=*) database=${argument#--dbname=} ;; + esac +done +[ -n "${output}" ] && [ -n "${database}" ] +printf 'validated dump for %s\n' "${database}" > "${output}" +EOF +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "--list" ] +[ -s "$2" ] +EOF +chmod 700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" + +PATH="${work_dir}/bin:${PATH}" \ +JOTWINK_BACKUP_ROOT="${work_dir}/backups" \ +POSTGRES_SUPERUSER_PASSWORD_FILE="${work_dir}/password" \ +JOTWINK_BACKUP_ENCRYPTION_CERT="${work_dir}/recipient.pem" \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ +sh "${repo_root}/scripts/run-jotwink-backup.sh" + +latest=$(readlink "${work_dir}/backups/latest") +backup_dir=${work_dir}/backups/${latest} +for expected in jotwink.dump.cms keycloak_jotwink.dump.cms SHA256SUMS metadata.json; do + test -s "${backup_dir}/${expected}" +done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS >/dev/null) +for database in jotwink keycloak_jotwink; do + if grep -a -q 'validated dump' "${backup_dir}/${database}.dump.cms"; then + echo "Plaintext leaked into the persistent backup artifact." >&2 + exit 1 + fi + openssl cms -decrypt -binary -inform DER \ + -in "${backup_dir}/${database}.dump.cms" \ + -recip "${work_dir}/recipient.pem" \ + -inkey "${work_dir}/recipient.key" \ + -out "${work_dir}/${database}.decrypted" + grep -q "validated dump for ${database}" "${work_dir}/${database}.decrypted" +done +test -s "${work_dir}/backups/last-success.json" + +JOTWINK_BACKUP_ROOT="${work_dir}/backups" \ +JOTWINK_BACKUP_INTERVAL_SECONDS=300 \ +JOTWINK_BACKUP_RETRY_SECONDS=30 \ +sh "${repo_root}/scripts/run-jotwink-backup-loop.sh" healthcheck + +echo "Jotwink PostgreSQL backup contract passed." diff --git a/scripts/test-jotwink-encrypted-restore.py b/scripts/test-jotwink-encrypted-restore.py new file mode 100644 index 0000000..669a1e5 --- /dev/null +++ b/scripts/test-jotwink-encrypted-restore.py @@ -0,0 +1,63 @@ +"""Exercise real pg_dump/CMS/pg_restore over verified TLS in disposable PostgreSQL.""" +import os +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + +SCRIPTS=Path(__file__).resolve().parent +IMAGE='postgres@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea' +container='jotwink-encrypted-test-'+uuid.uuid4().hex[:10] + +def run(*args, **kw): + result=subprocess.run(args,capture_output=True,text=True,**kw) + if result.returncode: + raise RuntimeError('Disposable fixture operation failed: '+result.stderr) + return result.stdout.strip() + +with tempfile.TemporaryDirectory(prefix='jotwink-encrypted-') as tmp: + root=Path(tmp);root.chmod(0o755) + cert=root/'server.crt';key=root/'server.key' + run('openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost,IP:127.0.0.1','-keyout',str(key),'-out',str(cert)) + # Separate recipient key from server transport credentials. + recipient=root/'recipient.pem';private=root/'recipient.key' + run('openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=backup-test','-keyout',str(private),'-out',str(recipient)) + password=root/'password';password.write_text('disposable-fixture-password\n');password.chmod(0o600) + bin=root/'bin';bin.mkdir();work=root/'tmp';work.mkdir() + for tool in ['pg_dump','pg_restore','psql']: + (bin/tool).write_text('#!/bin/sh\nexec docker exec -i -e PGPASSFILE -e PGSERVICEFILE -e PGSSLMODE -e PGSSLROOTCERT "$TEST_CONTAINER" '+tool+' "$@"\n') + (bin/tool).chmod(0o700) + env=dict(os.environ,PATH=str(bin)+os.pathsep+os.environ['PATH'],TEST_CONTAINER=container,TMPDIR=str(work),PGUSER='postgres',PGHOST='127.0.0.1',PGSSLMODE='verify-full',PGSSLROOTCERT=str(cert),POSTGRES_SUPERUSER_PASSWORD_FILE=str(password),JOTWINK_BACKUP_ENCRYPTION_CERT=str(recipient),JOTWINK_BACKUP_ROOT=str(root/'backups')) + def sql(database,query): + return run('docker','exec','-i',container,'psql','-U','postgres','-d',database,'-XAt','-v','ON_ERROR_STOP=1',input=query) + try: + run('docker','run','-d','--network','none','--name',container,'--tmpfs','/var/lib/postgresql/data','--volume',tmp+':'+tmp,'-e','POSTGRES_PASSWORD=disposable-fixture-password',IMAGE) + for _ in range(30): + if subprocess.run(['docker','exec',container,'pg_isready','-U','postgres'],capture_output=True).returncode==0:break + time.sleep(1) + run('docker','cp',str(cert),container+':/tmp/server.crt');run('docker','cp',str(key),container+':/tmp/server.key') + run('docker','exec',container,'chown','postgres:postgres','/tmp/server.crt','/tmp/server.key') + run('docker','exec',container,'chmod','600','/tmp/server.key') + sql('postgres',"ALTER SYSTEM SET ssl='on';\nALTER SYSTEM SET ssl_cert_file='/tmp/server.crt';\nALTER SYSTEM SET ssl_key_file='/tmp/server.key';\nSELECT pg_reload_conf();") + sql('postgres','CREATE DATABASE jotwink; CREATE DATABASE keycloak_jotwink; CREATE DATABASE jotwink_restore_test; CREATE DATABASE keycloak_jotwink_restore_test;') + sql('jotwink',"CREATE TABLE schema_migrations(version text); INSERT INTO schema_migrations VALUES ('real-backup-test');") + sql('keycloak_jotwink',"CREATE TABLE realm(id text); INSERT INTO realm VALUES ('isolated-identity-test');") + run('sh',str(SCRIPTS/'run-jotwink-backup.sh'),env=env) + backup=(root/'backups/latest').resolve() + services=root/'services.conf' + services.write_text(''.join(f'[{name}]\nhost=127.0.0.1\nuser=postgres\npassword=disposable-fixture-password\ndbname={db}\nsslrootcert={cert}\n' for name,db in [('app','jotwink_restore_test'),('identity','keycloak_jotwink_restore_test')])) + services.chmod(0o600) + env.update(PGSERVICEFILE=str(services),JOTWINK_RESTORE_SERVICE='app',KEYCLOAK_JOTWINK_RESTORE_SERVICE='identity',JOTWINK_RESTORE_CONFIRM='replace-nonproduction-restore-targets',JOTWINK_BACKUP_DECRYPTION_CERT=str(recipient),JOTWINK_BACKUP_DECRYPTION_KEY=str(private)) + run('bash',str(SCRIPTS/'verify-jotwink-restore.sh'),str(backup),env=env) + assert sql('jotwink_restore_test','SELECT version FROM schema_migrations')=='real-backup-test' + assert sql('keycloak_jotwink_restore_test','SELECT id FROM realm')=='isolated-identity-test' + # Re-running against populated destinations must fail before replacing rows. + failed=subprocess.run(['bash',str(SCRIPTS/'verify-jotwink-restore.sh'),str(backup)],env=env,capture_output=True) + assert failed.returncode != 0 + assert sql('jotwink_restore_test','SELECT version FROM schema_migrations')=='real-backup-test' + # Disabling TLS cannot create an allegedly verified backup. + assert subprocess.run(['sh',str(SCRIPTS/'run-jotwink-backup.sh')],env=dict(env,PGSSLMODE='disable'),capture_output=True).returncode != 0 + print('PASS: real PostgreSQL dumps, verified TLS, CMS encryption, both restored datasets, nonempty-target preservation and TLS-downgrade rejection') + finally: + subprocess.run(['docker','rm','-f',container],capture_output=True) diff --git a/scripts/test-jotwink-safety.py b/scripts/test-jotwink-safety.py new file mode 100644 index 0000000..37c808d --- /dev/null +++ b/scripts/test-jotwink-safety.py @@ -0,0 +1,106 @@ +import importlib.util +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +import time +import uuid + +ROOT=Path(__file__).resolve().parent +spec=importlib.util.spec_from_file_location('hba',ROOT/'jotwink-hba.py') +hba=importlib.util.module_from_spec(spec);spec.loader.exec_module(hba) + +class HBA(unittest.TestCase): + def test_shipped_policy_has_complete_ordered_rules(self): + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text((ROOT.parent/'config/runtrace-pg_hba.conf').read_text()) + before=p.read_bytes();hba.apply(p);self.assertEqual(before,p.read_bytes()) + + def test_rerun_preserves_neighbor_rules_and_permissions(self): + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';original='local all all trust\nhostssl other other 10.1.0.1/32 scram-sha-256\nhost all neighbor all reject\nhost all all all scram-sha-256\n';p.write_text(original);p.chmod(0o640) + hba.apply(p);first=p.read_bytes();hba.apply(p) + self.assertEqual(first,p.read_bytes());self.assertEqual(p.stat().st_mode&0o777,0o640) + self.assertIn('hostssl other other 10.1.0.1/32 scram-sha-256',p.read_text()) + def test_each_missing_rule_rejected_without_write(self): + for i in range(len(hba.RULES)): + with self.subTest(rule=i),tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text('\n'.join(hba.RULES[:i]+hba.RULES[i+1:])+ '\nhost all all all scram-sha-256\n');before=p.read_bytes() + with self.assertRaises(ValueError):hba.apply(p) + self.assertEqual(before,p.read_bytes()) + def test_shadowed_block_and_earlier_broad_allow_rejected(self): + for content in ['host all all all scram-sha-256\n'+'\n'.join(hba.RULES), 'hostssl all all 0.0.0.0/0 trust\n'+'\n'.join(hba.RULES)+'\nhost all all all scram-sha-256\n']: + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text(content) + with self.assertRaises(ValueError):hba.apply(p) + +class Restore(unittest.TestCase): + def test_rejects_destinations_before_any_restore(self): + for reply,service in [('f','app'),('error','app'),('t','app dbname=jotwink'),('second-f','app')]: + with self.subTest(reply=reply),tempfile.TemporaryDirectory() as t: + p=Path(t);bin=p/'bin';bin.mkdir();backup=p/'backup';backup.mkdir() + for name in ['jotwink.dump.cms','keycloak_jotwink.dump.cms','SHA256SUMS','metadata.json']:(backup/name).write_text('fixture') + for name in ['service','cert','key']:(p/name).write_text('fixture') + (bin/'sha256sum').write_text('#!/bin/sh\nexit 0\n') + (bin/'psql').write_text('''#!/bin/sh +cat > "$TEST_ROOT/query" +printf '%s\\n' "$*" >> "$TEST_ROOT/args" +case "$TEST_REPLY" in +error) exit 1;; +second-f) case "$*" in *service=identity*) echo f;; *) echo t;; esac;; +*) echo "$TEST_REPLY";; +esac +''') + for name in ['pg_restore','openssl']:(bin/name).write_text('#!/bin/sh\ntouch "$TEST_ROOT/mutation"\nexit 1\n') + for f in bin.iterdir():f.chmod(0o700) + env=dict(os.environ,PATH=str(bin)+os.pathsep+os.environ['PATH'],TEST_ROOT=t,TEST_REPLY=reply,PGSERVICEFILE=str(p/'service'),JOTWINK_BACKUP_DECRYPTION_CERT=str(p/'cert'),JOTWINK_BACKUP_DECRYPTION_KEY=str(p/'key'),JOTWINK_RESTORE_SERVICE=service,KEYCLOAK_JOTWINK_RESTORE_SERVICE='identity',JOTWINK_RESTORE_CONFIRM='replace-nonproduction-restore-targets') + result=subprocess.run(['bash',str(ROOT/'verify-jotwink-restore.sh'),str(backup)],env=env,capture_output=True) + self.assertNotEqual(result.returncode,0);self.assertFalse((p/'mutation').exists()) + if (p/'args').exists():self.assertIn('sslmode=verify-full',(p/'args').read_text()) + +class DatabaseGuard(unittest.TestCase): + def test_actual_database_name_and_nonempty_objects(self): + container='jotwink-restore-guard-'+uuid.uuid4().hex[:10] + image='postgres@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea' + def command(*args,input=None): + return subprocess.run(args,input=input,text=True,capture_output=True,check=True).stdout.strip() + query=(ROOT/'verify-jotwink-restore.sh').read_text().split("<<'SQL'\n",1)[1].split('\nSQL',1)[0] + def sql(database,text,expected='jotwink_restore_test'): + return command('docker','exec','-i',container,'psql','-U','postgres','-d',database,'-XAt','-v','ON_ERROR_STOP=1','-v','expected_database='+expected,input=text) + try: + command('docker','run','-d','--network','none','--name',container,'--tmpfs','/var/lib/postgresql/data','-e','POSTGRES_PASSWORD=disposable-fixture-password',image) + for _ in range(30): + if subprocess.run(['docker','exec',container,'pg_isready','-U','postgres'],capture_output=True).returncode==0:break + time.sleep(1) + sql('postgres','CREATE DATABASE jotwink; CREATE DATABASE jotwink_restore_test; CREATE DATABASE keycloak_jotwink_restore_test;') + self.assertEqual(sql('jotwink',query),'f') + self.assertEqual(sql('jotwink_restore_test',query),'t') + self.assertEqual(sql('keycloak_jotwink_restore_test',query),'f') + self.assertEqual(sql('keycloak_jotwink_restore_test',query,'keycloak_jotwink_restore_test'),'t') + for ddl,cleanup in [('CREATE TABLE sentinel(id int)','DROP TABLE sentinel'),('CREATE SEQUENCE sentinel','DROP SEQUENCE sentinel'),('CREATE VIEW sentinel AS SELECT 1','DROP VIEW sentinel')]: + sql('jotwink_restore_test',ddl) + self.assertEqual(sql('jotwink_restore_test',query),'f') + sql('jotwink_restore_test',cleanup) + sql('postgres','DROP DATABASE jotwink;') + bootstrap=(ROOT.parent/'bootstrap/jotwink-databases.sql').read_text() + def provision(password='disposable-app-password-at-least-32-characters'): + return command('docker','exec','-i',container,'psql','-U','postgres','-d','postgres','-XAt','-v','ON_ERROR_STOP=1','-v','jotwink_app_password='+password,'-v','keycloak_jotwink_app_password=disposable-identity-password-at-least-32-characters',input=bootstrap) + with self.assertRaises(subprocess.CalledProcessError):provision('weak') + self.assertEqual(sql('postgres',"SELECT count(*) FROM pg_roles WHERE rolname='jotwink_app'"),'0') + provision() + original=sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='jotwink_app'") + provision('different-password-must-not-replace-live-credential') + self.assertEqual(original,sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='jotwink_app'")) + self.assertEqual(sql('postgres',"SELECT has_database_privilege('jotwink_app','keycloak_jotwink','CONNECT')"),'f') + self.assertEqual(sql('postgres',"SELECT has_database_privilege('keycloak_jotwink_app','jotwink','CONNECT')"),'f') + sql('postgres','ALTER ROLE keycloak_jotwink_app SUPERUSER;') + with self.assertRaises(subprocess.CalledProcessError):provision() + self.assertEqual(original,sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='jotwink_app'")) + sql('postgres','ALTER ROLE keycloak_jotwink_app NOSUPERUSER; GRANT CONNECT ON DATABASE jotwink TO PUBLIC;') + with self.assertRaises(subprocess.CalledProcessError):provision() + + finally: + subprocess.run(['docker','rm','-f',container],capture_output=True) + +if __name__=='__main__':unittest.main() diff --git a/scripts/validate-jotwink-config.sh b/scripts/validate-jotwink-config.sh new file mode 100755 index 0000000..6fab5e7 --- /dev/null +++ b/scripts/validate-jotwink-config.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +require_literal() { + local file=$1 + local literal=$2 + grep -Fq -- "${literal}" "${repo_root}/${file}" || { + echo "Missing Jotwink contract in ${file}: ${literal}" >&2 + exit 1 + } +} + +for literal in \ + "jotwink_app_password" \ + "keycloak_jotwink_app_password" \ + "CREATE DATABASE jotwink OWNER jotwink_app" \ + "CREATE DATABASE keycloak_jotwink OWNER keycloak_jotwink_app" \ + "pg_advisory_lock" \ + "pg_advisory_unlock"; do + require_literal bootstrap/jotwink-databases.sql "${literal}" +done + +require_literal config/runtrace-pg_hba.conf "hostnossl jotwink" +require_literal config/runtrace-pg_hba.conf "hostnossl keycloak_jotwink" +require_literal config/runtrace-pg_hba.conf "hostssl jotwink jotwink_app 10.80.0.1/32" +require_literal config/runtrace-pg_hba.conf "hostssl keycloak_jotwink keycloak_jotwink_app 88.99.209.165/32" +require_literal compose.host.yml "jotwink_backup:" +require_literal compose.host.yml "profiles: [jotwink]" +require_literal compose.host.yml 'user: "999:999"' +require_literal compose.host.yml "JOTWINK_BACKUP_INTERVAL_SECONDS" +require_literal compose.host.yml "JOTWINK_BACKUP_ENCRYPTION_CERT" +require_literal compose.host.yml "no-new-privileges:true" +require_literal compose.yml "jotwink_backup_script:" +require_literal compose.yml "jotwink_backup_loop_script:" +require_literal envs/production/.env.db "MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600" +require_literal envs/production/.env.db "MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_CONFIG=" +require_literal scripts/verify-jotwink-restore.sh "replace-nonproduction-restore-targets" +require_literal scripts/verify-jotwink-restore.sh "--single-transaction" +require_literal scripts/run-jotwink-backup.sh "-aes-256-gcm" +require_literal scripts/run-jotwink-backup.sh "mktemp -d" + +if grep -Eq 'postgres://(jotwink_app|keycloak_jotwink_app):[^[:space:]]*sslmode=disable' "${repo_root}/README.md"; then + echo "README must not document plaintext-capable Jotwink database URLs." >&2 + exit 1 +fi + +echo "Jotwink PostgreSQL configuration is valid." diff --git a/scripts/verify-jotwink-restore.sh b/scripts/verify-jotwink-restore.sh new file mode 100755 index 0000000..b217c2d --- /dev/null +++ b/scripts/verify-jotwink-restore.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +backup_dir=${1:?Usage: verify-jotwink-restore.sh } +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a root-owned libpq service file}" +: "${JOTWINK_RESTORE_SERVICE:?JOTWINK_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_JOTWINK_RESTORE_SERVICE:?KEYCLOAK_JOTWINK_RESTORE_SERVICE must name an empty non-production database}" +: "${JOTWINK_RESTORE_CONFIRM:?set JOTWINK_RESTORE_CONFIRM=replace-nonproduction-restore-targets}" +: "${JOTWINK_BACKUP_DECRYPTION_CERT:?JOTWINK_BACKUP_DECRYPTION_CERT must identify the recipient certificate}" +: "${JOTWINK_BACKUP_DECRYPTION_KEY:?JOTWINK_BACKUP_DECRYPTION_KEY must identify the offline private key}" + +[[ "${JOTWINK_RESTORE_CONFIRM}" == replace-nonproduction-restore-targets ]] || { echo "Restore requires explicit non-production confirmation." >&2; exit 1; } +for path in "${PGSERVICEFILE}" "${JOTWINK_BACKUP_DECRYPTION_CERT}" "${JOTWINK_BACKUP_DECRYPTION_KEY}"; do + [[ -f "${path}" && ! -L "${path}" ]] || { echo "Restore input must be a regular non-symlink file: ${path}" >&2; exit 1; } +done +[[ -d "${backup_dir}" && ! -L "${backup_dir}" ]] || { echo "Backup directory is invalid." >&2; exit 1; } +for required in jotwink.dump.cms keycloak_jotwink.dump.cms SHA256SUMS metadata.json; do [[ -s "${backup_dir}/${required}" ]] || exit 1; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS) + +# Validate both actual destinations before decrypting or restoring either dump. +# Service names are identifiers, never arbitrary libpq connection strings. +for service in "$JOTWINK_RESTORE_SERVICE" "$KEYCLOAK_JOTWINK_RESTORE_SERVICE"; do + [[ "$service" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "Invalid restore service identifier." >&2; exit 1; } +done +verify_target() { + local service=$1 expected=$2 result + result=$(psql "service=$service sslmode=verify-full" -X -v ON_ERROR_STOP=1 -At \ + -v expected_database="$expected" <<'SQL' +SELECT current_database() = :'expected_database' + AND NOT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + AND c.relkind IN ('r','p','v','m','S','f') + ); +SQL + ) + [[ "$result" == t ]] || { echo "Restore target must be the named empty disposable database." >&2; exit 1; } +} +verify_target "$JOTWINK_RESTORE_SERVICE" jotwink_restore_test +verify_target "$KEYCLOAK_JOTWINK_RESTORE_SERVICE" keycloak_jotwink_restore_test + +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/jotwink-restore.XXXXXX") +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT HUP INT TERM + +for database in jotwink keycloak_jotwink; do + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${JOTWINK_BACKUP_DECRYPTION_CERT}" -inkey "${JOTWINK_BACKUP_DECRYPTION_KEY}" \ + -out "${work_dir}/${database}.dump" + pg_restore --list "${work_dir}/${database}.dump" >/dev/null +done + +pg_restore --dbname="service=${JOTWINK_RESTORE_SERVICE} sslmode=verify-full" --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/jotwink.dump" +pg_restore --dbname="service=${KEYCLOAK_JOTWINK_RESTORE_SERVICE} sslmode=verify-full" --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/keycloak_jotwink.dump" + +[[ $(psql "service=${JOTWINK_RESTORE_SERVICE} sslmode=verify-full" -X -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.schema_migrations') IS NOT NULL;") == t ]] +[[ $(psql "service=${KEYCLOAK_JOTWINK_RESTORE_SERVICE} sslmode=verify-full" -X -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.realm') IS NOT NULL;") == t ]] +echo "Jotwink and Keycloak restore verification completed against non-production targets."