From 5fad4c7c594dcadba19acb4a2bc3f07eab601dcb Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 23:29:13 +0200 Subject: [PATCH 1/4] fix(postgres): provision BetaCrew databases and encrypted backups --- README.md | 38 +++++++++++++- bootstrap/betacrew-databases.sql | 53 +++++++++++++++++++ compose.host.yml | 48 +++++++++++++++++ config/runtrace-pg_hba.conf | 12 ++++- envs/production/.env.db | 7 +++ scripts/run-betacrew-backup-loop.sh | 29 +++++++++++ scripts/run-betacrew-backup.sh | 81 +++++++++++++++++++++++++++++ scripts/test-betacrew-backup.sh | 48 +++++++++++++++++ scripts/validate-betacrew-config.sh | 19 +++++++ scripts/verify-betacrew-restore.sh | 36 +++++++++++++ 10 files changed, 368 insertions(+), 3 deletions(-) create mode 100644 bootstrap/betacrew-databases.sql create mode 100755 scripts/run-betacrew-backup-loop.sh create mode 100755 scripts/run-betacrew-backup.sh create mode 100755 scripts/test-betacrew-backup.sh create mode 100755 scripts/validate-betacrew-config.sh create mode 100755 scripts/verify-betacrew-restore.sh diff --git a/README.md b/README.md index 26b7d6d..0d75e41 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,14 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database - `bootstrap/jotwink-databases.sql`: idempotent bootstrap for the isolated Jotwink application and identity databases +- `bootstrap/betacrew-databases.sql`: idempotent bootstrap for the isolated BetaCrew application and identity databases - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets - `scripts/run-jotwink-backup.sh`: certificate-verified logical backup for Jotwink application and identity data - `scripts/verify-jotwink-restore.sh`: destructive Jotwink restore verification against explicit non-production targets +- `scripts/run-betacrew-backup.sh`: six-hour encrypted BetaCrew application and identity backup +- `scripts/verify-betacrew-restore.sh`: destructive BetaCrew restore verification against explicit non-production targets ## Networks @@ -72,7 +75,8 @@ Keycloak while requiring TLS and SCRAM for `runtrace` and `keycloak_runtrace`. Jotwink is stricter: `jotwink_app` can reach `jotwink` only from the application WireGuard peer `10.80.0.1`, and `keycloak_jotwink_app` can reach `keycloak_jotwink` only from the dedicated -Keycloak host. Other databases keep their existing SCRAM transport policy. +Keycloak host. BetaCrew uses the same source isolation with independent roles +and databases. Other databases keep their existing SCRAM transport policy. Required environment secrets: @@ -172,6 +176,29 @@ scripts/verify-jotwink-restore.sh /var/lib/makepad/postgres-backups/jotwink/ +``` + ## Application Databases Create one database and one dedicated user per application. @@ -197,6 +224,8 @@ Jotwink application and identity persistence use separate roles: | --- | --- | --- | | Jotwink app | `jotwink` | `jotwink_app` | | Jotwink Keycloak | `keycloak_jotwink` | `keycloak_jotwink_app` | +| BetaCrew app | `betacrew` | `betacrew_app` | +| BetaCrew Keycloak | `keycloak_betacrew` | `keycloak_betacrew_app` | OpenPanel application persistence uses: @@ -216,6 +245,8 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must : "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}" : "${JOTWINK_DB_PASSWORD:?set JOTWINK_DB_PASSWORD to a generated password}" : "${KEYCLOAK_JOTWINK_DB_PASSWORD:?set KEYCLOAK_JOTWINK_DB_PASSWORD to a generated password}" +: "${BETACREW_DB_PASSWORD:?set BETACREW_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_BETACREW_DB_PASSWORD:?set KEYCLOAK_BETACREW_DB_PASSWORD to a generated password}" psql "$POSTGRES_ADMIN_URL" \ -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \ @@ -241,6 +272,11 @@ psql "$POSTGRES_ADMIN_URL" \ -v jotwink_app_password="$JOTWINK_DB_PASSWORD" \ -v keycloak_jotwink_app_password="$KEYCLOAK_JOTWINK_DB_PASSWORD" \ -f bootstrap/jotwink-databases.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v betacrew_app_password="$BETACREW_DB_PASSWORD" \ + -v keycloak_betacrew_app_password="$KEYCLOAK_BETACREW_DB_PASSWORD" \ + -f bootstrap/betacrew-databases.sql ``` Jotwink application traffic follows the existing WireGuard path from diff --git a/bootstrap/betacrew-databases.sql b/bootstrap/betacrew-databases.sql new file mode 100644 index 0000000..e8b63bb --- /dev/null +++ b/bootstrap/betacrew-databases.sql @@ -0,0 +1,53 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql +-- variables so credentials never enter source control. +\if :{?betacrew_app_password} +\else + \echo 'missing required psql variable: betacrew_app_password' + \quit 1 +\endif +\if :{?keycloak_betacrew_app_password} +\else + \echo 'missing required psql variable: keycloak_betacrew_app_password' + \quit 1 +\endif + +SELECT NULLIF(btrim(:'betacrew_app_password'), '') IS NOT NULL AS betacrew_password_ok \gset +\if :betacrew_password_ok +\else + \echo 'empty required psql variable: betacrew_app_password' + \quit 1 +\endif +SELECT NULLIF(btrim(:'keycloak_betacrew_app_password'), '') IS NOT NULL AS keycloak_betacrew_password_ok \gset +\if :keycloak_betacrew_password_ok +\else + \echo 'empty required psql variable: keycloak_betacrew_app_password' + \quit 1 +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'betacrew_app') THEN + CREATE ROLE betacrew_app LOGIN; + END IF; +END $$; +ALTER ROLE betacrew_app LOGIN PASSWORD :'betacrew_app_password'; +SELECT 'CREATE DATABASE betacrew OWNER betacrew_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'betacrew') \gexec +ALTER DATABASE betacrew OWNER TO betacrew_app; +GRANT CONNECT ON DATABASE betacrew TO betacrew_app; + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_betacrew_app') THEN + CREATE ROLE keycloak_betacrew_app LOGIN; + END IF; +END $$; +ALTER ROLE keycloak_betacrew_app LOGIN PASSWORD :'keycloak_betacrew_app_password'; +SELECT 'CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_betacrew') \gexec +ALTER DATABASE keycloak_betacrew OWNER TO keycloak_betacrew_app; +GRANT CONNECT ON DATABASE keycloak_betacrew TO keycloak_betacrew_app; + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); diff --git a/compose.host.yml b/compose.host.yml index 79f1d25..9f3c8f2 100644 --- a/compose.host.yml +++ b/compose.host.yml @@ -129,3 +129,51 @@ services: options: max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" + + betacrew_backup: + image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8} + user: "70:70" + network_mode: host + command: + - /usr/local/bin/run-betacrew-backup-loop.sh + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem + BETACREW_BACKUP_ROOT: /backups + BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600} + BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} + BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: + - /tmp:mode=0700,uid=70,gid=70 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + volumes: + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro" + healthcheck: + test: ["CMD", "/usr/local/bin/run-betacrew-backup-loop.sh", "healthcheck"] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 82597e0..0f3b8d1 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -1,5 +1,5 @@ -# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also -# source-restricted to the application WireGuard peer and Keycloak host. +# Runtrace, Jotwink, and BetaCrew customer/identity traffic must use TLS. +# Jotwink and BetaCrew are source-restricted to the app WireGuard peer and Keycloak host. # Other shared databases retain their current SCRAM policy during migration. local all all trust hostnossl runtrace all all reject @@ -14,4 +14,12 @@ hostssl jotwink postgres 127.0.0.1/32 scram-sha-25 hostssl keycloak_jotwink postgres 127.0.0.1/32 scram-sha-256 hostssl jotwink all all reject hostssl keycloak_jotwink all all reject +hostnossl betacrew all all reject +hostnossl keycloak_betacrew all all reject +hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256 +hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256 +hostssl betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256 +hostssl betacrew all all reject +hostssl keycloak_betacrew all all reject host all all all scram-sha-256 diff --git a/envs/production/.env.db b/envs/production/.env.db index da9638b..7b963c9 100644 --- a/envs/production/.env.db +++ b/envs/production/.env.db @@ -1,6 +1,7 @@ # Production PostgreSQL settings POSTGRES_IMAGE=postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 JOTWINK_BACKUP_IMAGE=postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8 +BETACREW_BACKUP_IMAGE=postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8 POSTGRES_DB=postgres POSTGRES_USER=postgres MAKEPAD_POSTGRES_DATA_PATH=/var/lib/makepad/postgres @@ -21,6 +22,12 @@ MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres- MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35 +MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH=/var/lib/makepad/postgres-backups/betacrew +MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password +MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/betacrew-recipient.pem +MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600 +MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS=300 +MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS=35 MAKEPAD_POSTGRES_BACKUP_CPU_LIMIT=1.0 MAKEPAD_POSTGRES_BACKUP_MEMORY_LIMIT=1G MAKEPAD_POSTGRES_BACKUP_CPU_RESERVATION=0.1 diff --git a/scripts/run-betacrew-backup-loop.sh b/scripts/run-betacrew-backup-loop.sh new file mode 100755 index 0000000..fc351dd --- /dev/null +++ b/scripts/run-betacrew-backup-loop.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +interval_seconds=${BETACREW_BACKUP_INTERVAL_SECONDS:-21600} +retry_seconds=${BETACREW_BACKUP_RETRY_SECONDS:-300} +backup_root=${BETACREW_BACKUP_ROOT:-/backups} + +case "${interval_seconds}:${retry_seconds}" in + *[!0-9:]*) echo "BetaCrew backup timing values must be positive integers." >&2; exit 1 ;; +esac +[ "${interval_seconds}" -ge 300 ] && [ "${retry_seconds}" -ge 30 ] || { echo "BetaCrew backup interval or retry delay is too short." >&2; exit 1; } + +if [ "${1:-}" = healthcheck ]; then + status_file=${backup_root}/last-success.json + [ -s "${status_file}" ] || exit 1 + now=$(date +%s) + if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi + [ $((now - modified)) -le $((interval_seconds * 2 + retry_seconds)) ] + exit +fi + +while :; do + if /usr/local/bin/run-betacrew-backup.sh; then + sleep "${interval_seconds}" + else + echo "BetaCrew PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2 + sleep "${retry_seconds}" + fi +done diff --git a/scripts/run-betacrew-backup.sh b/scripts/run-betacrew-backup.sh new file mode 100755 index 0000000..e4f2bf9 --- /dev/null +++ b/scripts/run-betacrew-backup.sh @@ -0,0 +1,81 @@ +#!/bin/sh +set -eu + +umask 077 + +backup_root=${BETACREW_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_SUPERUSER_PASSWORD_FILE:-/run/secrets/postgres_superuser_password} +encryption_cert=${BETACREW_BACKUP_ENCRYPTION_CERT:-/run/config/betacrew_backup_recipient.pem} +retention_days=${BETACREW_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:-127.0.0.1} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:-postgres} + +case "${retention_days}" in + ''|*[!0-9]*) echo "BETACREW_BACKUP_RETENTION_DAYS must be a positive integer." >&2; exit 1 ;; +esac +[ "${retention_days}" -ge 1 ] || { echo "BETACREW_BACKUP_RETENTION_DAYS must be at least 1." >&2; exit 1; } +[ -s "${password_file}" ] || { echo "PostgreSQL backup credential file is missing or empty." >&2; exit 1; } +[ -s "${encryption_cert}" ] || { echo "BetaCrew backup encryption certificate is missing or empty." >&2; exit 1; } +command -v openssl >/dev/null 2>&1 || { echo "openssl is required to encrypt BetaCrew backups." >&2; exit 1; } +openssl x509 -in "${encryption_cert}" -noout >/dev/null 2>&1 || { echo "BetaCrew backup certificate is invalid." >&2; exit 1; } + +mkdir -p "${backup_root}" +chmod 700 "${backup_root}" +lock_dir=${backup_root}/.betacrew-backup.lock +mkdir "${lock_dir}" 2>/dev/null || { echo "Another BetaCrew backup is running." >&2; exit 1; } + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/betacrew-backup.${timestamp}.XXXXXX") +pgpass=${work_dir}/.pgpass + +cleanup() { + if [ -d "${partial_dir}" ]; then find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${partial_dir}" 2>/dev/null || true; fi + if [ -d "${work_dir}" ]; then find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +[ ! -e "${final_dir}" ] && [ ! -L "${final_dir}" ] || { echo "Backup destination already exists." >&2; exit 1; } +password=$(tr -d '\r\n' < "${password_file}") +[ -n "${password}" ] || { echo "PostgreSQL backup credential contains no usable value." >&2; exit 1; } +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" + +mkdir "${partial_dir}" +for database in betacrew keycloak_betacrew; do + dump_path=${work_dir}/${database}.dump + encrypted_path=${partial_dir}/${database}.dump.cms + pg_dump --host="${pg_host}" --port="${pg_port}" --username="${pg_user}" --dbname="${database}" \ + --format=custom --compress=9 --no-owner --no-acl --file="${dump_path}" + pg_restore --list "${dump_path}" >/dev/null + openssl cms -encrypt -binary -outform DER -aes-256-gcm -in "${dump_path}" -out "${encrypted_path}" "${encryption_cert}" + openssl cms -cmsout -inform DER -in "${encrypted_path}" -noout >/dev/null + find "${dump_path}" -delete +done + +(cd "${partial_dir}" && sha256sum betacrew.dump.cms keycloak_betacrew.dump.cms > SHA256SUMS) +cat > "${partial_dir}/metadata.json" <&2; exit 1; } +ln -sfn "${timestamp}" "${backup_root}/latest" +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","databases":["betacrew","keycloak_betacrew"]}\n' "${timestamp}" "${timestamp}" > "${status_tmp}" +chmod 600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_days}" -exec sh -c ' + for directory do find "$directory" -mindepth 1 -delete; rmdir "$directory"; done +' sh {} + + +echo "BetaCrew PostgreSQL backup completed: ${final_dir}" diff --git a/scripts/test-betacrew-backup.sh b/scripts/test-betacrew-backup.sh new file mode 100755 index 0000000..3f0c406 --- /dev/null +++ b/scripts/test-betacrew-backup.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups" +printf 'validation-password\n' > "${work_dir}/password" +chmod 600 "${work_dir}/password" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=betacrew-backup-test' \ + -keyout "${work_dir}/recipient.key" -out "${work_dir}/recipient.pem" >/dev/null 2>&1 + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +for argument in "$@"; do + case "${argument}" in --file=*) output=${argument#--file=} ;; --dbname=*) database=${argument#--dbname=} ;; esac +done +[ -n "${output:-}" ] && [ -n "${database:-}" ] +printf 'validated dump for %s\n' "${database}" > "${output}" +EOF +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "--list" ] && [ -s "$2" ] +EOF +chmod 700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" + +PATH="${work_dir}/bin:${PATH}" BETACREW_BACKUP_ROOT="${work_dir}/backups" \ +POSTGRES_SUPERUSER_PASSWORD_FILE="${work_dir}/password" \ +BETACREW_BACKUP_ENCRYPTION_CERT="${work_dir}/recipient.pem" \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" sh "${repo_root}/scripts/run-betacrew-backup.sh" + +latest=$(readlink "${work_dir}/backups/latest") +backup_dir=${work_dir}/backups/${latest} +for expected in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do test -s "${backup_dir}/${expected}"; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS >/dev/null) +for database in betacrew keycloak_betacrew; do + ! grep -a -q 'validated dump' "${backup_dir}/${database}.dump.cms" + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${work_dir}/recipient.pem" -inkey "${work_dir}/recipient.key" -out "${work_dir}/${database}.decrypted" + grep -q "validated dump for ${database}" "${work_dir}/${database}.decrypted" +done +BETACREW_BACKUP_ROOT="${work_dir}/backups" BETACREW_BACKUP_INTERVAL_SECONDS=300 \ +BETACREW_BACKUP_RETRY_SECONDS=30 sh "${repo_root}/scripts/run-betacrew-backup-loop.sh" healthcheck +echo "BetaCrew PostgreSQL encrypted backup contract passed." diff --git a/scripts/validate-betacrew-config.sh b/scripts/validate-betacrew-config.sh new file mode 100755 index 0000000..0bfb53f --- /dev/null +++ b/scripts/validate-betacrew-config.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +require_literal() { grep -Fq -- "$2" "${repo_root}/$1" || { echo "Missing BetaCrew contract in $1: $2" >&2; exit 1; }; } + +require_literal bootstrap/betacrew-databases.sql "pg_advisory_lock" +require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE betacrew OWNER betacrew_app" +require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app" +require_literal config/runtrace-pg_hba.conf "hostssl betacrew betacrew_app 10.80.0.1/32" +require_literal config/runtrace-pg_hba.conf "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32" +require_literal compose.host.yml "betacrew_backup:" +require_literal compose.host.yml "BETACREW_BACKUP_INTERVAL_SECONDS" +require_literal compose.host.yml "BETACREW_BACKUP_ENCRYPTION_CERT" +require_literal envs/production/.env.db "MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600" +require_literal scripts/run-betacrew-backup.sh "-aes-256-gcm" +require_literal scripts/verify-betacrew-restore.sh "replace-nonproduction-restore-targets" +require_literal scripts/verify-betacrew-restore.sh "--single-transaction" +echo "BetaCrew PostgreSQL configuration is valid." diff --git a/scripts/verify-betacrew-restore.sh b/scripts/verify-betacrew-restore.sh new file mode 100755 index 0000000..15dda1b --- /dev/null +++ b/scripts/verify-betacrew-restore.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +backup_dir=${1:?Usage: verify-betacrew-restore.sh } +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a root-owned libpq service file}" +: "${BETACREW_RESTORE_SERVICE:?BETACREW_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_BETACREW_RESTORE_SERVICE:?KEYCLOAK_BETACREW_RESTORE_SERVICE must name an empty non-production database}" +: "${BETACREW_RESTORE_CONFIRM:?set BETACREW_RESTORE_CONFIRM=replace-nonproduction-restore-targets}" +: "${BETACREW_BACKUP_DECRYPTION_CERT:?BETACREW_BACKUP_DECRYPTION_CERT must identify the recipient certificate}" +: "${BETACREW_BACKUP_DECRYPTION_KEY:?BETACREW_BACKUP_DECRYPTION_KEY must identify the offline private key}" + +[[ "${BETACREW_RESTORE_CONFIRM}" == replace-nonproduction-restore-targets ]] || { echo "Restore requires explicit non-production confirmation." >&2; exit 1; } +for path in "${PGSERVICEFILE}" "${BETACREW_BACKUP_DECRYPTION_CERT}" "${BETACREW_BACKUP_DECRYPTION_KEY}"; do + [[ -f "${path}" && ! -L "${path}" ]] || { echo "Restore input must be a regular non-symlink file: ${path}" >&2; exit 1; } +done +[[ -d "${backup_dir}" && ! -L "${backup_dir}" ]] || { echo "Backup directory is invalid." >&2; exit 1; } +for required in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do [[ -s "${backup_dir}/${required}" ]] || exit 1; done +(cd "${backup_dir}" && sha256sum --check SHA256SUMS) + +work_dir=$(mktemp -d) +cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } +trap cleanup EXIT HUP INT TERM + +for database in betacrew keycloak_betacrew; do + openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ + -recip "${BETACREW_BACKUP_DECRYPTION_CERT}" -inkey "${BETACREW_BACKUP_DECRYPTION_KEY}" \ + -out "${work_dir}/${database}.dump" + pg_restore --list "${work_dir}/${database}.dump" >/dev/null +done + +pg_restore --dbname="service=${BETACREW_RESTORE_SERVICE}" --clean --if-exists --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/betacrew.dump" +pg_restore --dbname="service=${KEYCLOAK_BETACREW_RESTORE_SERVICE}" --clean --if-exists --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/keycloak_betacrew.dump" + +[[ $(psql "service=${BETACREW_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.schema_migrations') IS NOT NULL;") == t ]] +[[ $(psql "service=${KEYCLOAK_BETACREW_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.realm') IS NOT NULL;") == t ]] +echo "BetaCrew and Keycloak restore verification completed against non-production targets." From 97ddfc0694942bb2e9e8acd3c12ff5d02afc5d9b Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 23:37:15 +0200 Subject: [PATCH 2/4] fix(postgres): preserve deployed PostgreSQL host additions --- compose.betacrew.host.yml | 41 +++++++++++++++++++++++++++++ scripts/apply-betacrew-hba.sh | 41 +++++++++++++++++++++++++++++ scripts/validate-betacrew-config.sh | 2 ++ 3 files changed, 84 insertions(+) create mode 100644 compose.betacrew.host.yml create mode 100755 scripts/apply-betacrew-hba.sh diff --git a/compose.betacrew.host.yml b/compose.betacrew.host.yml new file mode 100644 index 0000000..cf6c163 --- /dev/null +++ b/compose.betacrew.host.yml @@ -0,0 +1,41 @@ +services: + betacrew_backup: + image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8} + user: "70:70" + network_mode: host + command: [/usr/local/bin/run-betacrew-backup-loop.sh] + environment: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-postgres} + PGSSLMODE: verify-full + PGSSLROOTCERT: /etc/postgresql/ca.crt + POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password + BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem + BETACREW_BACKUP_ROOT: /backups + BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600} + BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} + BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} + read_only: true + tmpfs: [/tmp:mode=0700,uid=70,gid=70] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + restart: unless-stopped + volumes: + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro" + - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro" + - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro" + healthcheck: + test: [CMD, /usr/local/bin/run-betacrew-backup-loop.sh, healthcheck] + interval: 5m + timeout: 10s + retries: 3 + start_period: 10m + logging: + driver: json-file + options: + max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}" + max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}" diff --git a/scripts/apply-betacrew-hba.sh b/scripts/apply-betacrew-hba.sh new file mode 100755 index 0000000..01341f7 --- /dev/null +++ b/scripts/apply-betacrew-hba.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +hba_path=${1:?Usage: apply-betacrew-hba.sh } +[[ -f "${hba_path}" && ! -L "${hba_path}" ]] || { echo "HBA path must be a regular non-symlink file." >&2; exit 1; } + +marker='hostnossl betacrew' +if grep -Fq "${marker}" "${hba_path}"; then + for required in \ + 'hostssl betacrew betacrew_app 10.80.0.1/32' \ + 'hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32' \ + 'hostssl keycloak_betacrew all all reject'; do + grep -Fq "${required}" "${hba_path}" || { echo "Partial BetaCrew HBA policy found; refusing to modify it." >&2; exit 1; } + done + exit 0 +fi + +fallback='host all all all scram-sha-256' +[[ $(grep -Fxc "${fallback}" "${hba_path}") == 1 ]] || { echo "Expected exactly one shared HBA fallback rule." >&2; exit 1; } + +temporary=$(mktemp "${hba_path}.betacrew.XXXXXX") +cleanup() { [[ ! -e "${temporary}" ]] || rm -f "${temporary}"; } +trap cleanup EXIT HUP INT TERM + +awk -v fallback="${fallback}" ' + $0 == fallback { + print "# BetaCrew app and identity traffic require TLS and exact source roles." + print "hostnossl betacrew all all reject" + print "hostnossl keycloak_betacrew all all reject" + print "hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256" + print "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256" + print "hostssl betacrew postgres 127.0.0.1/32 scram-sha-256" + print "hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256" + print "hostssl betacrew all all reject" + print "hostssl keycloak_betacrew all all reject" + } + { print } +' "${hba_path}" > "${temporary}" +if original_mode=$(stat -c '%a' "${hba_path}" 2>/dev/null); then :; else original_mode=$(stat -f '%Lp' "${hba_path}"); fi +chmod "${original_mode}" "${temporary}" +mv "${temporary}" "${hba_path}" diff --git a/scripts/validate-betacrew-config.sh b/scripts/validate-betacrew-config.sh index 0bfb53f..e4ed5fa 100755 --- a/scripts/validate-betacrew-config.sh +++ b/scripts/validate-betacrew-config.sh @@ -10,10 +10,12 @@ require_literal bootstrap/betacrew-databases.sql "CREATE DATABASE keycloak_betac require_literal config/runtrace-pg_hba.conf "hostssl betacrew betacrew_app 10.80.0.1/32" require_literal config/runtrace-pg_hba.conf "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32" require_literal compose.host.yml "betacrew_backup:" +require_literal compose.betacrew.host.yml "betacrew_backup:" require_literal compose.host.yml "BETACREW_BACKUP_INTERVAL_SECONDS" require_literal compose.host.yml "BETACREW_BACKUP_ENCRYPTION_CERT" require_literal envs/production/.env.db "MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600" require_literal scripts/run-betacrew-backup.sh "-aes-256-gcm" require_literal scripts/verify-betacrew-restore.sh "replace-nonproduction-restore-targets" require_literal scripts/verify-betacrew-restore.sh "--single-transaction" +bash -n "${repo_root}/scripts/apply-betacrew-hba.sh" echo "BetaCrew PostgreSQL configuration is valid." From abe8adb4d691b7f56f9f2dcc625775198f15f7ec Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 23:38:27 +0200 Subject: [PATCH 3/4] fix(postgres): fix host backup tmpfs mount --- compose.betacrew.host.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose.betacrew.host.yml b/compose.betacrew.host.yml index cf6c163..414ab99 100644 --- a/compose.betacrew.host.yml +++ b/compose.betacrew.host.yml @@ -17,7 +17,8 @@ services: BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300} BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35} read_only: true - tmpfs: [/tmp:mode=0700,uid=70,gid=70] + tmpfs: + - /tmp:mode=0700,uid=70,gid=70 cap_drop: [ALL] security_opt: [no-new-privileges:true] restart: unless-stopped From 0b050dab9da9e85fdd283dd7fc223b6decb18b4f Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 8 Oct 2026 08:44:55 +0200 Subject: [PATCH 4/4] fix(postgres): guard BetaCrew bootstrap and restore destinations --- .github/actionlint.yaml | 3 + .github/workflows/betacrew-safety.yml | 26 +++++++ README.md | 20 +++++ bootstrap/betacrew-databases.sql | 62 ++++++++++------ scripts/apply-betacrew-hba.sh | 41 +---------- scripts/betacrew-hba.py | 48 ++++++++++++ scripts/run-betacrew-backup.sh | 1 + scripts/test-betacrew-safety.py | 101 ++++++++++++++++++++++++++ scripts/verify-betacrew-restore.sh | 31 +++++++- 9 files changed, 266 insertions(+), 67 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/betacrew-safety.yml create mode 100644 scripts/betacrew-hba.py create mode 100644 scripts/test-betacrew-safety.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..628281c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - makepad diff --git a/.github/workflows/betacrew-safety.yml b/.github/workflows/betacrew-safety.yml new file mode 100644 index 0000000..1b98500 --- /dev/null +++ b/.github/workflows/betacrew-safety.yml @@ -0,0 +1,26 @@ +name: BetaCrew database safety +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] +permissions: + contents: read +jobs: + safety: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: [self-hosted, linux, x64, makepad] + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + - name: Database bootstrap, HBA and restore guard regression tests + run: python3 scripts/test-betacrew-safety.py + - name: Encryption contract (mock database dump) + run: bash scripts/test-betacrew-backup.sh + - name: Configuration and shell syntax + run: | + bash scripts/validate-betacrew-config.sh + bash -n scripts/verify-betacrew-restore.sh scripts/apply-betacrew-hba.sh diff --git a/README.md b/README.md index 0d75e41..3a74bf4 100644 --- a/README.md +++ b/README.md @@ -337,3 +337,23 @@ bash scripts/test-runtrace-backup.sh bash scripts/validate-jotwink-config.sh bash scripts/test-jotwink-backup.sh ``` + +### BetaCrew safety gates + +Bootstrap requires passwords of at least 32 characters. It preserves existing +passwords and refuses partial role/database pairs, elevated roles, role memberships, +unexpected database owners, or non-owner database ACLs. It does not repair live +privilege drift or rotate credentials. New databases revoke PUBLIC access. + +The restore verifier checks both actual connected database names before writing: +`betacrew_restore_test` and `keycloak_betacrew_restore_test`. Both must be empty +(no user tables, views or sequences). Service identifiers cannot contain libpq +connection options; certificate-verified TLS is forced. Restore does not use +`--clean`, so a concurrent object creation cannot cause its silent replacement. +The HBA helper validates all eight ordered rules before the shared fallback and +rejects incomplete or shadowed policies, preserving file ownership and mode. + +Run `python3 scripts/test-betacrew-safety.py` for disposable PostgreSQL and guard +regressions. Production release still requires current shared-topology review, +protected backup credentials, a real encrypted restore drill and off-host backup +verification; the encryption-only fixture is not evidence of a database restore. diff --git a/bootstrap/betacrew-databases.sql b/bootstrap/betacrew-databases.sql index e8b63bb..f69690b 100644 --- a/bootstrap/betacrew-databases.sql +++ b/bootstrap/betacrew-databases.sql @@ -5,49 +5,63 @@ \if :{?betacrew_app_password} \else \echo 'missing required psql variable: betacrew_app_password' - \quit 1 + SELECT 1/0; \endif \if :{?keycloak_betacrew_app_password} \else \echo 'missing required psql variable: keycloak_betacrew_app_password' - \quit 1 + SELECT 1/0; \endif -SELECT NULLIF(btrim(:'betacrew_app_password'), '') IS NOT NULL AS betacrew_password_ok \gset +SELECT length(:'betacrew_app_password') >= 32 AS betacrew_password_ok \gset \if :betacrew_password_ok \else \echo 'empty required psql variable: betacrew_app_password' - \quit 1 + SELECT 1/0; \endif -SELECT NULLIF(btrim(:'keycloak_betacrew_app_password'), '') IS NOT NULL AS keycloak_betacrew_password_ok \gset +SELECT length(:'keycloak_betacrew_app_password') >= 32 AS keycloak_betacrew_password_ok \gset \if :keycloak_betacrew_password_ok \else \echo 'empty required psql variable: keycloak_betacrew_app_password' - \quit 1 + SELECT 1/0; \endif SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); -DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'betacrew_app') THEN - CREATE ROLE betacrew_app LOGIN; - END IF; +-- Validate both pairs before changing either. Existing objects must already be +-- isolated; never seize ownership or rotate a live credential during bootstrap. +DO $$ +DECLARE target record; role_row pg_roles%ROWTYPE; database_row pg_database%ROWTYPE; +BEGIN + FOR target IN SELECT * FROM (VALUES + ('betacrew','betacrew_app'),('keycloak_betacrew','keycloak_betacrew_app') + ) AS targets(database_name,role_name) LOOP + SELECT * INTO role_row FROM pg_roles WHERE rolname=target.role_name; + SELECT * INTO database_row FROM pg_database WHERE datname=target.database_name; + IF (role_row.oid IS NULL) <> (database_row.oid IS NULL) THEN + RAISE EXCEPTION 'Incomplete existing BetaCrew role/database pair'; + END IF; + IF role_row.oid IS NOT NULL THEN + IF NOT role_row.rolcanlogin OR role_row.rolsuper OR role_row.rolcreatedb + OR role_row.rolcreaterole OR role_row.rolreplication OR role_row.rolbypassrls + OR EXISTS(SELECT 1 FROM pg_auth_members WHERE member=role_row.oid OR roleid=role_row.oid) + OR database_row.datdba<>role_row.oid + OR EXISTS(SELECT 1 FROM aclexplode(COALESCE(database_row.datacl,acldefault('d',database_row.datdba))) WHERE grantee<>role_row.oid) + THEN RAISE EXCEPTION 'Existing BetaCrew privileges, membership, ownership or ACL require manual inspection'; + END IF; + END IF; + END LOOP; END $$; -ALTER ROLE betacrew_app LOGIN PASSWORD :'betacrew_app_password'; -SELECT 'CREATE DATABASE betacrew OWNER betacrew_app' -WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'betacrew') \gexec -ALTER DATABASE betacrew OWNER TO betacrew_app; -GRANT CONNECT ON DATABASE betacrew TO betacrew_app; -DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_betacrew_app') THEN - CREATE ROLE keycloak_betacrew_app LOGIN; - END IF; -END $$; -ALTER ROLE keycloak_betacrew_app LOGIN PASSWORD :'keycloak_betacrew_app_password'; +SELECT format('CREATE ROLE betacrew_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'betacrew_app_password') +WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='betacrew_app') \gexec +SELECT 'CREATE DATABASE betacrew OWNER betacrew_app' +WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='betacrew') \gexec +REVOKE ALL ON DATABASE betacrew FROM PUBLIC; +SELECT format('CREATE ROLE keycloak_betacrew_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'keycloak_betacrew_app_password') +WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='keycloak_betacrew_app') \gexec SELECT 'CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app' -WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_betacrew') \gexec -ALTER DATABASE keycloak_betacrew OWNER TO keycloak_betacrew_app; -GRANT CONNECT ON DATABASE keycloak_betacrew TO keycloak_betacrew_app; +WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='keycloak_betacrew') \gexec +REVOKE ALL ON DATABASE keycloak_betacrew FROM PUBLIC; SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap')); diff --git a/scripts/apply-betacrew-hba.sh b/scripts/apply-betacrew-hba.sh index 01341f7..21cea09 100755 --- a/scripts/apply-betacrew-hba.sh +++ b/scripts/apply-betacrew-hba.sh @@ -1,41 +1,4 @@ #!/usr/bin/env bash set -euo pipefail - -hba_path=${1:?Usage: apply-betacrew-hba.sh } -[[ -f "${hba_path}" && ! -L "${hba_path}" ]] || { echo "HBA path must be a regular non-symlink file." >&2; exit 1; } - -marker='hostnossl betacrew' -if grep -Fq "${marker}" "${hba_path}"; then - for required in \ - 'hostssl betacrew betacrew_app 10.80.0.1/32' \ - 'hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32' \ - 'hostssl keycloak_betacrew all all reject'; do - grep -Fq "${required}" "${hba_path}" || { echo "Partial BetaCrew HBA policy found; refusing to modify it." >&2; exit 1; } - done - exit 0 -fi - -fallback='host all all all scram-sha-256' -[[ $(grep -Fxc "${fallback}" "${hba_path}") == 1 ]] || { echo "Expected exactly one shared HBA fallback rule." >&2; exit 1; } - -temporary=$(mktemp "${hba_path}.betacrew.XXXXXX") -cleanup() { [[ ! -e "${temporary}" ]] || rm -f "${temporary}"; } -trap cleanup EXIT HUP INT TERM - -awk -v fallback="${fallback}" ' - $0 == fallback { - print "# BetaCrew app and identity traffic require TLS and exact source roles." - print "hostnossl betacrew all all reject" - print "hostnossl keycloak_betacrew all all reject" - print "hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256" - print "hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256" - print "hostssl betacrew postgres 127.0.0.1/32 scram-sha-256" - print "hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256" - print "hostssl betacrew all all reject" - print "hostssl keycloak_betacrew all all reject" - } - { print } -' "${hba_path}" > "${temporary}" -if original_mode=$(stat -c '%a' "${hba_path}" 2>/dev/null); then :; else original_mode=$(stat -f '%Lp' "${hba_path}"); fi -chmod "${original_mode}" "${temporary}" -mv "${temporary}" "${hba_path}" +root=$(cd "$(dirname "$0")" && pwd) +exec python3 "$root/betacrew-hba.py" "${1:?Usage: apply-betacrew-hba.sh }" diff --git a/scripts/betacrew-hba.py b/scripts/betacrew-hba.py new file mode 100644 index 0000000..6110d1a --- /dev/null +++ b/scripts/betacrew-hba.py @@ -0,0 +1,48 @@ +"""Add or validate the complete BetaCrew HBA block without altering other rules.""" +import os +from pathlib import Path +import sys +import tempfile + +RULES = [ + 'hostnossl betacrew all all reject', + 'hostnossl keycloak_betacrew all all reject', + 'hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256', + 'hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256', + 'hostssl betacrew postgres 127.0.0.1/32 scram-sha-256', + 'hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256', + 'hostssl betacrew all all reject', + 'hostssl keycloak_betacrew all all reject', +] + +def apply(path): + if path.is_symlink() or not path.is_file(): + raise ValueError('HBA path must be a regular non-symlink file') + stat = path.stat() + lines = path.read_text().splitlines(keepends=True) + parsed = [(i, line.split('#', 1)[0].split()) for i, line in enumerate(lines)] + scoped = [(i, ' '.join(parts)) for i, parts in parsed if len(parts)>1 and set(parts[1].split(',')) & {'betacrew','keycloak_betacrew'}] + broad = [i for i, parts in parsed if len(parts)>1 and parts[0].startswith('host') and 'all' in parts[1].split(',')] + fallback = [i for i, parts in parsed if parts == 'host all all all scram-sha-256'.split()] + if len(fallback)!=1 or broad != fallback: + raise ValueError('Ambiguous shared fallback rules; manual inspection required') + if scoped: + if [rule for _, rule in scoped] != RULES or max(i for i,_ in scoped)>=fallback[0]: + raise ValueError('Partial, reordered or shadowed BetaCrew policy; refusing mutation') + return + lines.insert(fallback[0], '# BetaCrew TLS and exact source restrictions\n'+'\n'.join(RULES)+'\n') + fd, temporary = tempfile.mkstemp(prefix=path.name+'.betacrew.', dir=path.parent) + try: + with os.fdopen(fd, 'w') as out: + os.fchmod(out.fileno(), stat.st_mode & 0o7777) + if (os.fstat(out.fileno()).st_uid, os.fstat(out.fileno()).st_gid)!=(stat.st_uid,stat.st_gid): + os.fchown(out.fileno(),stat.st_uid,stat.st_gid) + out.write(''.join(lines));out.flush();os.fsync(out.fileno()) + os.replace(temporary,path) + finally: + if os.path.exists(temporary):os.unlink(temporary) + +if __name__=='__main__': + try: apply(Path(sys.argv[1])) + except (ValueError,OSError) as error: + print(str(error),file=sys.stderr);sys.exit(1) diff --git a/scripts/run-betacrew-backup.sh b/scripts/run-betacrew-backup.sh index e4f2bf9..c7442f0 100755 --- a/scripts/run-betacrew-backup.sh +++ b/scripts/run-betacrew-backup.sh @@ -2,6 +2,7 @@ set -eu umask 077 +[ "${PGSSLMODE:-verify-full}" = verify-full ] || { echo "BetaCrew backups require certificate-verified TLS." >&2; exit 1; } backup_root=${BETACREW_BACKUP_ROOT:-/backups} password_file=${POSTGRES_SUPERUSER_PASSWORD_FILE:-/run/secrets/postgres_superuser_password} diff --git a/scripts/test-betacrew-safety.py b/scripts/test-betacrew-safety.py new file mode 100644 index 0000000..cee6bfa --- /dev/null +++ b/scripts/test-betacrew-safety.py @@ -0,0 +1,101 @@ +import importlib.util +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +import time +import uuid + +ROOT=Path(__file__).resolve().parent +spec=importlib.util.spec_from_file_location('hba',ROOT/'betacrew-hba.py') +hba=importlib.util.module_from_spec(spec);spec.loader.exec_module(hba) + +class HBA(unittest.TestCase): + def test_rerun_preserves_neighbor_rules_and_permissions(self): + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';original='local all all trust\nhostssl other other 10.1.0.1/32 scram-sha-256\nhost all all all scram-sha-256\n';p.write_text(original);p.chmod(0o640) + hba.apply(p);first=p.read_bytes();hba.apply(p) + self.assertEqual(first,p.read_bytes());self.assertEqual(p.stat().st_mode&0o777,0o640) + self.assertIn('hostssl other other 10.1.0.1/32 scram-sha-256',p.read_text()) + def test_each_missing_rule_rejected_without_write(self): + for i in range(len(hba.RULES)): + with self.subTest(rule=i),tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text('\n'.join(hba.RULES[:i]+hba.RULES[i+1:])+ '\nhost all all all scram-sha-256\n');before=p.read_bytes() + with self.assertRaises(ValueError):hba.apply(p) + self.assertEqual(before,p.read_bytes()) + def test_shadowed_block_and_earlier_broad_allow_rejected(self): + for content in ['host all all all scram-sha-256\n'+'\n'.join(hba.RULES), 'hostssl all all 0.0.0.0/0 trust\n'+'\n'.join(hba.RULES)+'\nhost all all all scram-sha-256\n']: + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'hba';p.write_text(content) + with self.assertRaises(ValueError):hba.apply(p) + +class Restore(unittest.TestCase): + def test_rejects_destinations_before_any_restore(self): + for reply,service in [('f','app'),('error','app'),('t','app dbname=betacrew'),('second-f','app')]: + with self.subTest(reply=reply),tempfile.TemporaryDirectory() as t: + p=Path(t);bin=p/'bin';bin.mkdir();backup=p/'backup';backup.mkdir() + for name in ['betacrew.dump.cms','keycloak_betacrew.dump.cms','SHA256SUMS','metadata.json']:(backup/name).write_text('fixture') + for name in ['service','cert','key']:(p/name).write_text('fixture') + (bin/'sha256sum').write_text('#!/bin/sh\nexit 0\n') + (bin/'psql').write_text('''#!/bin/sh +cat > "$TEST_ROOT/query" +printf '%s\\n' "$*" >> "$TEST_ROOT/args" +case "$TEST_REPLY" in +error) exit 1;; +second-f) case "$*" in *service=identity*) echo f;; *) echo t;; esac;; +*) echo "$TEST_REPLY";; +esac +''') + for name in ['pg_restore','openssl']:(bin/name).write_text('#!/bin/sh\ntouch "$TEST_ROOT/mutation"\nexit 1\n') + for f in bin.iterdir():f.chmod(0o700) + env=dict(os.environ,PATH=str(bin)+os.pathsep+os.environ['PATH'],TEST_ROOT=t,TEST_REPLY=reply,PGSERVICEFILE=str(p/'service'),BETACREW_BACKUP_DECRYPTION_CERT=str(p/'cert'),BETACREW_BACKUP_DECRYPTION_KEY=str(p/'key'),BETACREW_RESTORE_SERVICE=service,KEYCLOAK_BETACREW_RESTORE_SERVICE='identity',BETACREW_RESTORE_CONFIRM='replace-nonproduction-restore-targets') + result=subprocess.run(['bash',str(ROOT/'verify-betacrew-restore.sh'),str(backup)],env=env,capture_output=True) + self.assertNotEqual(result.returncode,0);self.assertFalse((p/'mutation').exists()) + if (p/'args').exists():self.assertIn('sslmode=verify-full',(p/'args').read_text()) + +class DatabaseGuard(unittest.TestCase): + def test_actual_database_name_and_nonempty_objects(self): + container='betacrew-restore-guard-'+uuid.uuid4().hex[:10] + image='postgres@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea' + def command(*args,input=None): + return subprocess.run(args,input=input,text=True,capture_output=True,check=True).stdout.strip() + query=(ROOT/'verify-betacrew-restore.sh').read_text().split("<<'SQL'\n",1)[1].split('\nSQL',1)[0] + def sql(database,text,expected='betacrew_restore_test'): + return command('docker','exec','-i',container,'psql','-U','postgres','-d',database,'-XAt','-v','ON_ERROR_STOP=1','-v','expected_database='+expected,input=text) + try: + command('docker','run','-d','--network','none','--name',container,'--tmpfs','/var/lib/postgresql/data','-e','POSTGRES_PASSWORD=disposable-fixture-password',image) + for _ in range(30): + if subprocess.run(['docker','exec',container,'pg_isready','-U','postgres'],capture_output=True).returncode==0:break + time.sleep(1) + sql('postgres','CREATE DATABASE betacrew; CREATE DATABASE betacrew_restore_test; CREATE DATABASE keycloak_betacrew_restore_test;') + self.assertEqual(sql('betacrew',query),'f') + self.assertEqual(sql('betacrew_restore_test',query),'t') + self.assertEqual(sql('keycloak_betacrew_restore_test',query),'f') + self.assertEqual(sql('keycloak_betacrew_restore_test',query,'keycloak_betacrew_restore_test'),'t') + for ddl,cleanup in [('CREATE TABLE sentinel(id int)','DROP TABLE sentinel'),('CREATE SEQUENCE sentinel','DROP SEQUENCE sentinel'),('CREATE VIEW sentinel AS SELECT 1','DROP VIEW sentinel')]: + sql('betacrew_restore_test',ddl) + self.assertEqual(sql('betacrew_restore_test',query),'f') + sql('betacrew_restore_test',cleanup) + sql('postgres','DROP DATABASE betacrew;') + bootstrap=(ROOT.parent/'bootstrap/betacrew-databases.sql').read_text() + def provision(password='disposable-app-password-at-least-32-characters'): + return command('docker','exec','-i',container,'psql','-U','postgres','-d','postgres','-XAt','-v','ON_ERROR_STOP=1','-v','betacrew_app_password='+password,'-v','keycloak_betacrew_app_password=disposable-identity-password-at-least-32-characters',input=bootstrap) + with self.assertRaises(subprocess.CalledProcessError):provision('weak') + self.assertEqual(sql('postgres',"SELECT count(*) FROM pg_roles WHERE rolname='betacrew_app'"),'0') + provision() + original=sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='betacrew_app'") + provision('different-password-must-not-replace-live-credential') + self.assertEqual(original,sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='betacrew_app'")) + self.assertEqual(sql('postgres',"SELECT has_database_privilege('betacrew_app','keycloak_betacrew','CONNECT')"),'f') + self.assertEqual(sql('postgres',"SELECT has_database_privilege('keycloak_betacrew_app','betacrew','CONNECT')"),'f') + sql('postgres','ALTER ROLE keycloak_betacrew_app SUPERUSER;') + with self.assertRaises(subprocess.CalledProcessError):provision() + self.assertEqual(original,sql('postgres',"SELECT rolpassword FROM pg_authid WHERE rolname='betacrew_app'")) + sql('postgres','ALTER ROLE keycloak_betacrew_app NOSUPERUSER; GRANT CONNECT ON DATABASE betacrew TO PUBLIC;') + with self.assertRaises(subprocess.CalledProcessError):provision() + + finally: + subprocess.run(['docker','rm','-f',container],capture_output=True) + +if __name__=='__main__':unittest.main() diff --git a/scripts/verify-betacrew-restore.sh b/scripts/verify-betacrew-restore.sh index 15dda1b..57f5c2b 100755 --- a/scripts/verify-betacrew-restore.sh +++ b/scripts/verify-betacrew-restore.sh @@ -17,6 +17,29 @@ done for required in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do [[ -s "${backup_dir}/${required}" ]] || exit 1; done (cd "${backup_dir}" && sha256sum --check SHA256SUMS) +# Validate both actual destinations before decrypting or restoring either dump. +# Service names are identifiers, never arbitrary libpq connection strings. +for service in "$BETACREW_RESTORE_SERVICE" "$KEYCLOAK_BETACREW_RESTORE_SERVICE"; do + [[ "$service" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "Invalid restore service identifier." >&2; exit 1; } +done +verify_target() { + local service=$1 expected=$2 result + result=$(psql "service=$service sslmode=verify-full" -X -v ON_ERROR_STOP=1 -At \ + -v expected_database="$expected" <<'SQL' +SELECT current_database() = :'expected_database' + AND NOT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + AND c.relkind IN ('r','p','v','m','S','f') + ); +SQL + ) + [[ "$result" == t ]] || { echo "Restore target must be the named empty disposable database." >&2; exit 1; } +} +verify_target "$BETACREW_RESTORE_SERVICE" betacrew_restore_test +verify_target "$KEYCLOAK_BETACREW_RESTORE_SERVICE" keycloak_betacrew_restore_test + work_dir=$(mktemp -d) cleanup() { find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true; rmdir "${work_dir}" 2>/dev/null || true; } trap cleanup EXIT HUP INT TERM @@ -28,9 +51,9 @@ for database in betacrew keycloak_betacrew; do pg_restore --list "${work_dir}/${database}.dump" >/dev/null done -pg_restore --dbname="service=${BETACREW_RESTORE_SERVICE}" --clean --if-exists --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/betacrew.dump" -pg_restore --dbname="service=${KEYCLOAK_BETACREW_RESTORE_SERVICE}" --clean --if-exists --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/keycloak_betacrew.dump" +pg_restore --dbname="service=${BETACREW_RESTORE_SERVICE} sslmode=verify-full" --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/betacrew.dump" +pg_restore --dbname="service=${KEYCLOAK_BETACREW_RESTORE_SERVICE} sslmode=verify-full" --no-owner --no-acl --exit-on-error --single-transaction "${work_dir}/keycloak_betacrew.dump" -[[ $(psql "service=${BETACREW_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.schema_migrations') IS NOT NULL;") == t ]] -[[ $(psql "service=${KEYCLOAK_BETACREW_RESTORE_SERVICE}" -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.realm') IS NOT NULL;") == t ]] +[[ $(psql "service=${BETACREW_RESTORE_SERVICE} sslmode=verify-full" -X -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.schema_migrations') IS NOT NULL;") == t ]] +[[ $(psql "service=${KEYCLOAK_BETACREW_RESTORE_SERVICE} sslmode=verify-full" -X -v ON_ERROR_STOP=1 -Atc "SELECT to_regclass('public.realm') IS NOT NULL;") == t ]] echo "BetaCrew and Keycloak restore verification completed against non-production targets."