fix(postgres): guard BetaCrew database isolation and restore targets

.github/workflows/betacrew-safety.yml

@@ -0,0 +1,28 @@
+name: BetaCrew database safety
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  safety:
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: [self-hosted, linux, x64, makepad]
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      - name: Database bootstrap, HBA and restore guard regression tests
+        run: python3 scripts/test-betacrew-safety.py
+      - name: Real encrypted database restore over verified TLS
+        run: python3 scripts/test-betacrew-encrypted-restore.py
+      - name: Encryption contract (mock database dump)
+        run: bash scripts/test-betacrew-backup.sh
+      - name: Configuration and shell syntax
+        run: |
+          bash scripts/validate-betacrew-config.sh
+          bash -n scripts/verify-betacrew-restore.sh scripts/apply-betacrew-hba.sh

README.md

@@ -953,3 +953,29 @@ The verifier checks both the app migration table and Keycloak realm table.
 
 
 Jotwink host backup is opt-in through the `jotwink` Compose profile. Provision and validate its recipient, credentials, TLS and restore evidence before enabling that profile. The shared deployment workflow preserves its existing release transaction.
+
+## BetaCrew Backup And Restore
+
+BetaCrew has an independent six-hour backup service for `betacrew` and
+`keycloak_betacrew`. Dumps are validated, encrypted with OpenSSL CMS
+AES-256-GCM before persistent storage, checksummed, and retained for 35 days.
+Provision `/var/lib/makepad/postgres-backups/betacrew` for uid 70 and install
+the public recipient certificate at
+`/etc/makepad/postgres-backup/betacrew-recipient.pem`; keep its private key
+offline. Replicate completed backup directories to independently administered
+storage.
+
+Run a restore drill against empty non-production databases with:
+
+```bash
+export PGSERVICEFILE=/etc/makepad/postgres-restore-services.conf
+export BETACREW_RESTORE_SERVICE=betacrew_restore_test
+export KEYCLOAK_BETACREW_RESTORE_SERVICE=keycloak_betacrew_restore_test
+export BETACREW_RESTORE_CONFIRM=replace-nonproduction-restore-targets
+export BETACREW_BACKUP_DECRYPTION_CERT=/secure/betacrew-recipient.pem
+export BETACREW_BACKUP_DECRYPTION_KEY=/secure/betacrew-recipient.key
+scripts/verify-betacrew-restore.sh /var/lib/makepad/postgres-backups/betacrew/<timestamp>
+```
+
+
+BetaCrew backup uses uid 999 and a dedicated mode-0400 credential file owned by that uid. The shared host Compose service requires the `betacrew` profile; the separate `compose.betacrew.host.yml` remains available for a scoped backup-only deployment.

bootstrap/betacrew-databases.sql

@@ -0,0 +1,67 @@
+\set ON_ERROR_STOP on
+
+-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql
+-- variables so credentials never enter source control.
+\if :{?betacrew_app_password}
+\else
+  \echo 'missing required psql variable: betacrew_app_password'
+  SELECT 1/0;
+\endif
+\if :{?keycloak_betacrew_app_password}
+\else
+  \echo 'missing required psql variable: keycloak_betacrew_app_password'
+  SELECT 1/0;
+\endif
+
+SELECT length(:'betacrew_app_password') >= 32 AS betacrew_password_ok \gset
+\if :betacrew_password_ok
+\else
+  \echo 'empty required psql variable: betacrew_app_password'
+  SELECT 1/0;
+\endif
+SELECT length(:'keycloak_betacrew_app_password') >= 32 AS keycloak_betacrew_password_ok \gset
+\if :keycloak_betacrew_password_ok
+\else
+  \echo 'empty required psql variable: keycloak_betacrew_app_password'
+  SELECT 1/0;
+\endif
+
+SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap'));
+
+-- Validate both pairs before changing either. Existing objects must already be
+-- isolated; never seize ownership or rotate a live credential during bootstrap.
+DO $$
+DECLARE target record; role_row pg_roles%ROWTYPE; database_row pg_database%ROWTYPE;
+BEGIN
+  FOR target IN SELECT * FROM (VALUES
+    ('betacrew','betacrew_app'),('keycloak_betacrew','keycloak_betacrew_app')
+  ) AS targets(database_name,role_name) LOOP
+    SELECT * INTO role_row FROM pg_roles WHERE rolname=target.role_name;
+    SELECT * INTO database_row FROM pg_database WHERE datname=target.database_name;
+    IF (role_row.oid IS NULL) <> (database_row.oid IS NULL) THEN
+      RAISE EXCEPTION 'Incomplete existing BetaCrew role/database pair';
+    END IF;
+    IF role_row.oid IS NOT NULL THEN
+      IF NOT role_row.rolcanlogin OR role_row.rolsuper OR role_row.rolcreatedb
+         OR role_row.rolcreaterole OR role_row.rolreplication OR role_row.rolbypassrls
+         OR EXISTS(SELECT 1 FROM pg_auth_members WHERE member=role_row.oid OR roleid=role_row.oid)
+         OR database_row.datdba<>role_row.oid
+         OR EXISTS(SELECT 1 FROM aclexplode(COALESCE(database_row.datacl,acldefault('d',database_row.datdba))) WHERE grantee<>role_row.oid)
+      THEN RAISE EXCEPTION 'Existing BetaCrew privileges, membership, ownership or ACL require manual inspection';
+      END IF;
+    END IF;
+  END LOOP;
+END $$;
+
+SELECT format('CREATE ROLE betacrew_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'betacrew_app_password')
+WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='betacrew_app') \gexec
+SELECT 'CREATE DATABASE betacrew OWNER betacrew_app'
+WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='betacrew') \gexec
+REVOKE ALL ON DATABASE betacrew FROM PUBLIC;
+SELECT format('CREATE ROLE keycloak_betacrew_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'keycloak_betacrew_app_password')
+WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='keycloak_betacrew_app') \gexec
+SELECT 'CREATE DATABASE keycloak_betacrew OWNER keycloak_betacrew_app'
+WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='keycloak_betacrew') \gexec
+REVOKE ALL ON DATABASE keycloak_betacrew FROM PUBLIC;
+
+SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('betacrew-databases-bootstrap'));

compose.betacrew.host.yml

@@ -0,0 +1,42 @@
+services:
+  betacrew_backup:
+    image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825}
+    user: "999:999"
+    network_mode: host
+    command: [/usr/local/bin/run-betacrew-backup-loop.sh]
+    environment:
+      PGHOST: 127.0.0.1
+      PGPORT: "5432"
+      PGUSER: ${POSTGRES_USER:-postgres}
+      PGSSLMODE: verify-full
+      PGSSLROOTCERT: /etc/postgresql/ca.crt
+      POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password
+      BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem
+      BETACREW_BACKUP_ROOT: /backups
+      BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600}
+      BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300}
+      BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35}
+    read_only: true
+    tmpfs:
+      - /tmp:mode=0700,uid=999,gid=999
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]
+    restart: unless-stopped
+    volumes:
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro"
+      - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro"
+    healthcheck:
+      test: [CMD, /usr/local/bin/run-betacrew-backup-loop.sh, healthcheck]
+      interval: 5m
+      timeout: 10s
+      retries: 3
+      start_period: 10m
+    logging:
+      driver: json-file
+      options:
+        max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
+        max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"

compose.host.yml

@@ -181,3 +181,52 @@ services:
       options:
         max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
         max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"
+
+  betacrew_backup:
+    image: ${BETACREW_BACKUP_IMAGE:-postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825}
+    profiles: [betacrew]
+    user: "999:999"
+    network_mode: host
+    command:
+      - /usr/local/bin/run-betacrew-backup-loop.sh
+    environment:
+      PGHOST: 127.0.0.1
+      PGPORT: "5432"
+      PGUSER: ${POSTGRES_USER:-postgres}
+      PGSSLMODE: verify-full
+      PGSSLROOTCERT: /etc/postgresql/ca.crt
+      POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password
+      BETACREW_BACKUP_ENCRYPTION_CERT: /run/config/betacrew_backup_recipient.pem
+      BETACREW_BACKUP_ROOT: /backups
+      BETACREW_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS:-21600}
+      BETACREW_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS:-300}
+      BETACREW_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS:-35}
+    read_only: true
+    tmpfs:
+      - /tmp:mode=0700,uid=999,gid=999
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH:-/var/lib/makepad/postgres-backups/betacrew}:/backups"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro"
+      - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/betacrew-recipient.pem}:/run/config/betacrew_backup_recipient.pem:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup.sh}:/usr/local/bin/run-betacrew-backup.sh:ro"
+      - "${MAKEPAD_POSTGRES_BETACREW_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-betacrew-backup-loop.sh}:/usr/local/bin/run-betacrew-backup-loop.sh:ro"
+    healthcheck:
+      test: ["CMD", "/usr/local/bin/run-betacrew-backup-loop.sh", "healthcheck"]
+      interval: 5m
+      timeout: 10s
+      retries: 3
+      start_period: 10m
+    logging:
+      driver: json-file
+      options:
+        max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
+        max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"

config/runtrace-pg_hba.conf

@@ -1,5 +1,5 @@
-# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also
-# source-restricted to the application WireGuard peer and Keycloak host.
+# Runtrace, Jotwink, and BetaCrew customer/identity traffic must use TLS.
+# Jotwink and BetaCrew are source-restricted to the app WireGuard peer and Keycloak host.
 # Other shared databases retain their current SCRAM policy during migration.
 local   all                 all                                     trust
 hostnossl runtrace          all                 all                 reject
@@ -16,6 +16,14 @@ hostssl jotwink             postgres            127.0.0.1/32        scram-sha-25
 hostssl keycloak_jotwink    postgres            127.0.0.1/32        scram-sha-256
 hostssl jotwink             all                 all                 reject
 hostssl keycloak_jotwink    all                 all                 reject
+hostnossl betacrew          all                 all                 reject
+hostnossl keycloak_betacrew all                 all                 reject
+hostssl betacrew            betacrew_app         10.80.0.1/32        scram-sha-256
+hostssl keycloak_betacrew   keycloak_betacrew_app 88.99.209.165/32   scram-sha-256
+hostssl betacrew            postgres             127.0.0.1/32        scram-sha-256
+hostssl keycloak_betacrew   postgres             127.0.0.1/32        scram-sha-256
+hostssl betacrew            all                  all                 reject
+hostssl keycloak_betacrew   all                  all                 reject
 hostssl brio_staging        brio_staging_app    all                 scram-sha-256
 hostssl brio_staging        brio_staging_backup all                 scram-sha-256
 hostssl keycloak_brio_staging keycloak_brio_staging_app all        scram-sha-256

envs/production/.env.db

@@ -45,3 +45,10 @@ MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-
 MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600
 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300
 MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35
+BETACREW_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825
+MAKEPAD_POSTGRES_BETACREW_BACKUP_PATH=/var/lib/makepad/postgres-backups/betacrew
+MAKEPAD_POSTGRES_BETACREW_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password
+MAKEPAD_POSTGRES_BETACREW_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/betacrew-recipient.pem
+MAKEPAD_POSTGRES_BETACREW_BACKUP_INTERVAL_SECONDS=21600
+MAKEPAD_POSTGRES_BETACREW_BACKUP_RETRY_SECONDS=300
+MAKEPAD_POSTGRES_BETACREW_BACKUP_RETENTION_DAYS=35

scripts/apply-betacrew-hba.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+root=$(cd "$(dirname "$0")" && pwd)
+exec python3 "$root/betacrew-hba.py" "${1:?Usage: apply-betacrew-hba.sh <pg_hba.conf>}"

scripts/betacrew-hba.py

@@ -0,0 +1,48 @@
+"""Add or validate the complete BetaCrew HBA block without altering other rules."""
+import os
+from pathlib import Path
+import sys
+import tempfile
+
+RULES = [
+ 'hostnossl betacrew all all reject',
+ 'hostnossl keycloak_betacrew all all reject',
+ 'hostssl betacrew betacrew_app 10.80.0.1/32 scram-sha-256',
+ 'hostssl keycloak_betacrew keycloak_betacrew_app 88.99.209.165/32 scram-sha-256',
+ 'hostssl betacrew postgres 127.0.0.1/32 scram-sha-256',
+ 'hostssl keycloak_betacrew postgres 127.0.0.1/32 scram-sha-256',
+ 'hostssl betacrew all all reject',
+ 'hostssl keycloak_betacrew all all reject',
+]
+
+def apply(path):
+    if path.is_symlink() or not path.is_file():
+        raise ValueError('HBA path must be a regular non-symlink file')
+    stat = path.stat()
+    lines = path.read_text().splitlines(keepends=True)
+    parsed = [(i, line.split('#', 1)[0].split()) for i, line in enumerate(lines)]
+    scoped = [(i, ' '.join(parts)) for i, parts in parsed if len(parts)>1 and set(parts[1].split(',')) & {'betacrew','keycloak_betacrew'}]
+    broad = [i for i, parts in parsed if len(parts)>1 and parts[0].startswith('host') and 'all' in parts[1].split(',') and not (len(parts) == 5 and parts[-1] == 'reject' and not set(parts[2].split(',')) & {'all', 'betacrew_app', 'keycloak_betacrew_app', 'postgres'} and not any(x.startswith('+') for x in parts[2].split(',')))]
+    fallback = [i for i, parts in parsed if parts == 'host all all all scram-sha-256'.split()]
+    if len(fallback)!=1 or broad != fallback:
+        raise ValueError('Ambiguous shared fallback rules; manual inspection required')
+    if scoped:
+        if [rule for _, rule in scoped] != RULES or max(i for i,_ in scoped)>=fallback[0]:
+            raise ValueError('Partial, reordered or shadowed BetaCrew policy; refusing mutation')
+        return
+    lines.insert(fallback[0], '# BetaCrew TLS and exact source restrictions\n'+'\n'.join(RULES)+'\n')
+    fd, temporary = tempfile.mkstemp(prefix=path.name+'.betacrew.', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'w') as out:
+            os.fchmod(out.fileno(), stat.st_mode & 0o7777)
+            if (os.fstat(out.fileno()).st_uid, os.fstat(out.fileno()).st_gid)!=(stat.st_uid,stat.st_gid):
+                os.fchown(out.fileno(),stat.st_uid,stat.st_gid)
+            out.write(''.join(lines));out.flush();os.fsync(out.fileno())
+        os.replace(temporary,path)
+    finally:
+        if os.path.exists(temporary):os.unlink(temporary)
+
+if __name__=='__main__':
+    try: apply(Path(sys.argv[1]))
+    except (ValueError,OSError) as error:
+        print(str(error),file=sys.stderr);sys.exit(1)

scripts/run-betacrew-backup-loop.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -eu
+
+interval_seconds=${BETACREW_BACKUP_INTERVAL_SECONDS:-21600}
+retry_seconds=${BETACREW_BACKUP_RETRY_SECONDS:-300}
+backup_root=${BETACREW_BACKUP_ROOT:-/backups}
+
+case "${interval_seconds}:${retry_seconds}" in
+  *[!0-9:]*) echo "BetaCrew backup timing values must be positive integers." >&2; exit 1 ;;
+esac
+[ "${interval_seconds}" -ge 300 ] && [ "${retry_seconds}" -ge 30 ] || { echo "BetaCrew backup interval or retry delay is too short." >&2; exit 1; }
+
+if [ "${1:-}" = healthcheck ]; then
+  status_file=${backup_root}/last-success.json
+  [ -s "${status_file}" ] || exit 1
+  now=$(date +%s)
+  if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi
+  [ $((now - modified)) -le $((interval_seconds * 2 + retry_seconds)) ]
+  exit
+fi
+
+while :; do
+  if /usr/local/bin/run-betacrew-backup.sh; then
+    sleep "${interval_seconds}"
+  else
+    echo "BetaCrew PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2
+    sleep "${retry_seconds}"
+  fi
+done