fix(postgres): preserve Jotwink isolation and verify encrypted restores

.github/workflows/jotwink-safety.yml

@@ -0,0 +1,28 @@
+name: Jotwink database safety
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  safety:
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: [self-hosted, linux, x64, makepad]
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      - name: Database bootstrap, HBA and restore guard regression tests
+        run: python3 scripts/test-jotwink-safety.py
+      - name: Real encrypted database restore over verified TLS
+        run: python3 scripts/test-jotwink-encrypted-restore.py
+      - name: Encryption contract (mock database dump)
+        run: bash scripts/test-jotwink-backup.sh
+      - name: Configuration and shell syntax
+        run: |
+          bash scripts/validate-jotwink-config.sh
+          bash -n scripts/verify-jotwink-restore.sh

README.md

@@ -911,3 +911,45 @@ logical backup using the previous image. CartHop bootstrap and Runtrace TLS
 fixtures also use the production candidate. These fixtures do not replace a
 fresh verified backup of every production database or a scoped rollout and
 rollback plan for the shared service.
+
+## Jotwink Backup And Restore
+
+Jotwink has an independent backup service and health timestamp so a failed
+Jotwink dump cannot be hidden by a successful Runtrace dump. It creates
+custom-format dumps of `jotwink` and `keycloak_jotwink` every six hours,
+validates each archive, encrypts it in ephemeral tmpfs with OpenSSL CMS and
+AES-256-GCM before it reaches persistent storage, writes SHA-256 checksums, and
+retains 35 days by default. Provision its root and a public recipient
+certificate before starting `compose.host.yml`:
+
+```bash
+sudo install -d -o 999 -g 999 -m 0700 /var/lib/makepad/postgres-backups/jotwink
+sudo install -d -o root -g root -m 0755 /etc/makepad/postgres-backup
+sudo install -o root -g root -m 0444 jotwink-recipient.pem /etc/makepad/postgres-backup/jotwink-recipient.pem
+```
+
+Use a dedicated mode-0400 backup credential file owned by uid 999 and set
+`MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH` to that file. Keep the
+matching private key offline; it is used only during a controlled restore drill.
+
+Replicate every completed timestamp directory and `last-success.json` to an
+off-host, separately administered encrypted store. Alert before the health
+timestamp exceeds two intervals. At least quarterly, and before launch or a
+database upgrade, restore the newest artifacts into empty non-production
+databases:
+
+```bash
+export PGSERVICEFILE=/etc/makepad/postgres-restore-services.conf
+export JOTWINK_RESTORE_SERVICE=jotwink_restore_test
+export KEYCLOAK_JOTWINK_RESTORE_SERVICE=keycloak_jotwink_restore_test
+export JOTWINK_RESTORE_CONFIRM=replace-nonproduction-restore-targets
+export JOTWINK_BACKUP_DECRYPTION_CERT=/secure/jotwink-recipient.pem
+export JOTWINK_BACKUP_DECRYPTION_KEY=/secure/jotwink-recipient.key
+scripts/verify-jotwink-restore.sh /var/lib/makepad/postgres-backups/jotwink/<timestamp>
+```
+
+Record the artifact checksum, elapsed time, recovery-point age, and operator.
+The verifier checks both the app migration table and Keycloak realm table.
+
+
+Jotwink host backup is opt-in through the `jotwink` Compose profile. Provision and validate its recipient, credentials, TLS and restore evidence before enabling that profile. The shared deployment workflow preserves its existing release transaction.

bootstrap/jotwink-databases.sql

@@ -0,0 +1,67 @@
+\set ON_ERROR_STOP on
+
+-- Run with a PostgreSQL superuser connection. Passwords are supplied as psql
+-- variables so credentials never enter source control.
+\if :{?jotwink_app_password}
+\else
+  \echo 'missing required psql variable: jotwink_app_password'
+  SELECT 1/0;
+\endif
+\if :{?keycloak_jotwink_app_password}
+\else
+  \echo 'missing required psql variable: keycloak_jotwink_app_password'
+  SELECT 1/0;
+\endif
+
+SELECT length(:'jotwink_app_password') >= 32 AS jotwink_password_ok \gset
+\if :jotwink_password_ok
+\else
+  \echo 'empty required psql variable: jotwink_app_password'
+  SELECT 1/0;
+\endif
+SELECT length(:'keycloak_jotwink_app_password') >= 32 AS keycloak_jotwink_password_ok \gset
+\if :keycloak_jotwink_password_ok
+\else
+  \echo 'empty required psql variable: keycloak_jotwink_app_password'
+  SELECT 1/0;
+\endif
+
+SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('jotwink-databases-bootstrap'));
+
+-- Validate both pairs before changing either. Existing objects must already be
+-- isolated; never seize ownership or rotate a live credential during bootstrap.
+DO $$
+DECLARE target record; role_row pg_roles%ROWTYPE; database_row pg_database%ROWTYPE;
+BEGIN
+  FOR target IN SELECT * FROM (VALUES
+    ('jotwink','jotwink_app'),('keycloak_jotwink','keycloak_jotwink_app')
+  ) AS targets(database_name,role_name) LOOP
+    SELECT * INTO role_row FROM pg_roles WHERE rolname=target.role_name;
+    SELECT * INTO database_row FROM pg_database WHERE datname=target.database_name;
+    IF (role_row.oid IS NULL) <> (database_row.oid IS NULL) THEN
+      RAISE EXCEPTION 'Incomplete existing Jotwink role/database pair';
+    END IF;
+    IF role_row.oid IS NOT NULL THEN
+      IF NOT role_row.rolcanlogin OR role_row.rolsuper OR role_row.rolcreatedb
+         OR role_row.rolcreaterole OR role_row.rolreplication OR role_row.rolbypassrls
+         OR EXISTS(SELECT 1 FROM pg_auth_members WHERE member=role_row.oid OR roleid=role_row.oid)
+         OR database_row.datdba<>role_row.oid
+         OR EXISTS(SELECT 1 FROM aclexplode(COALESCE(database_row.datacl,acldefault('d',database_row.datdba))) WHERE grantee<>role_row.oid)
+      THEN RAISE EXCEPTION 'Existing Jotwink privileges, membership, ownership or ACL require manual inspection';
+      END IF;
+    END IF;
+  END LOOP;
+END $$;
+
+SELECT format('CREATE ROLE jotwink_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'jotwink_app_password')
+WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='jotwink_app') \gexec
+SELECT 'CREATE DATABASE jotwink OWNER jotwink_app'
+WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='jotwink') \gexec
+REVOKE ALL ON DATABASE jotwink FROM PUBLIC;
+SELECT format('CREATE ROLE keycloak_jotwink_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD %L', :'keycloak_jotwink_app_password')
+WHERE NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='keycloak_jotwink_app') \gexec
+SELECT 'CREATE DATABASE keycloak_jotwink OWNER keycloak_jotwink_app'
+WHERE NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='keycloak_jotwink') \gexec
+REVOKE ALL ON DATABASE keycloak_jotwink FROM PUBLIC;
+
+SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('jotwink-databases-bootstrap'));

compose.host.yml

@@ -132,3 +132,52 @@ services:
       options:
         max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
         max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"
+
+  jotwink_backup:
+    image: ${JOTWINK_BACKUP_IMAGE:-postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825}
+    profiles: [jotwink]
+    user: "999:999"
+    network_mode: host
+    command:
+      - /usr/local/bin/run-jotwink-backup-loop.sh
+    environment:
+      PGHOST: 127.0.0.1
+      PGPORT: "5432"
+      PGUSER: ${POSTGRES_USER:-postgres}
+      PGSSLMODE: verify-full
+      PGSSLROOTCERT: /etc/postgresql/ca.crt
+      POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password
+      JOTWINK_BACKUP_ENCRYPTION_CERT: /run/config/jotwink_backup_recipient.pem
+      JOTWINK_BACKUP_ROOT: /backups
+      JOTWINK_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS:-21600}
+      JOTWINK_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS:-300}
+      JOTWINK_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS:-35}
+    read_only: true
+    tmpfs:
+      - /tmp:mode=0700,uid=999,gid=999
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH:-/var/lib/makepad/postgres-backups/jotwink}:/backups"
+      - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro"
+      - "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro"
+      - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/jotwink-recipient.pem}:/run/config/jotwink_backup_recipient.pem:ro"
+      - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-jotwink-backup.sh}:/usr/local/bin/run-jotwink-backup.sh:ro"
+      - "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-jotwink-backup-loop.sh}:/usr/local/bin/run-jotwink-backup-loop.sh:ro"
+    healthcheck:
+      test: ["CMD", "/usr/local/bin/run-jotwink-backup-loop.sh", "healthcheck"]
+      interval: 5m
+      timeout: 10s
+      retries: 3
+      start_period: 10m
+    logging:
+      driver: json-file
+      options:
+        max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
+        max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"

compose.yml

@@ -56,6 +56,10 @@ configs:
     file: ./scripts/run-runtrace-backup.sh
   runtrace_backup_loop_script:
     file: ./scripts/run-runtrace-backup-loop.sh
+  jotwink_backup_script:
+    file: ./scripts/run-jotwink-backup.sh
+  jotwink_backup_loop_script:
+    file: ./scripts/run-jotwink-backup-loop.sh
   brio_encrypted_backup_script:
     file: ./scripts/run-brio-encrypted-backup.sh
   brio_encrypted_backup_loop_script:

config/runtrace-pg_hba.conf

@@ -1,12 +1,21 @@
-# Runtrace customer data and identity traffic must use TLS. Other shared
-# databases retain their current SCRAM transport policy during migration.
+# Runtrace and Jotwink customer/identity traffic must use TLS. Jotwink is also
+# source-restricted to the application WireGuard peer and Keycloak host.
+# Other shared databases retain their current SCRAM policy during migration.
 local   all                 all                                     trust
 hostnossl runtrace          all                 all                 reject
 hostnossl keycloak_runtrace all                 all                 reject
 hostnossl brio_staging      all                 all                 reject
 hostnossl keycloak_brio_staging all             all                 reject
 hostssl runtrace            all                 all                 scram-sha-256
 hostssl keycloak_runtrace   all                 all                 scram-sha-256
+hostnossl jotwink           all                 all                 reject
+hostnossl keycloak_jotwink  all                 all                 reject
+hostssl jotwink             jotwink_app         10.80.0.1/32        scram-sha-256
+hostssl keycloak_jotwink    keycloak_jotwink_app 88.99.209.165/32   scram-sha-256
+hostssl jotwink             postgres            127.0.0.1/32        scram-sha-256
+hostssl keycloak_jotwink    postgres            127.0.0.1/32        scram-sha-256
+hostssl jotwink             all                 all                 reject
+hostssl keycloak_jotwink    all                 all                 reject
 hostssl brio_staging        brio_staging_app    all                 scram-sha-256
 hostssl brio_staging        brio_staging_backup all                 scram-sha-256
 hostssl keycloak_brio_staging keycloak_brio_staging_app all        scram-sha-256

envs/production/.env.db

@@ -37,3 +37,11 @@ MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE=20m
 MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES=5
 MAKEPAD_POSTGRES_LOG_MAX_SIZE=20m
 MAKEPAD_POSTGRES_LOG_MAX_FILES=5
+JOTWINK_BACKUP_IMAGE=postgres:16-bookworm@sha256:bb3e1a57e5407e0a5280b4211980a5e537f4abd234a87014ac979849a78dd825
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH=/var/lib/makepad/postgres-backups/jotwink
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH=/etc/makepad/secrets/postgres-backup-password
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_CONFIG=makepad_postgres_jotwink_backup_recipient_v1
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH=/etc/makepad/postgres-backup/jotwink-recipient.pem
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS=21600
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS=300
+MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS=35

scripts/jotwink-hba.py

@@ -0,0 +1,48 @@
+"""Add or validate the complete Jotwink HBA block without altering other rules."""
+import os
+from pathlib import Path
+import sys
+import tempfile
+
+RULES = [
+ 'hostnossl jotwink all all reject',
+ 'hostnossl keycloak_jotwink all all reject',
+ 'hostssl jotwink jotwink_app 10.80.0.1/32 scram-sha-256',
+ 'hostssl keycloak_jotwink keycloak_jotwink_app 88.99.209.165/32 scram-sha-256',
+ 'hostssl jotwink postgres 127.0.0.1/32 scram-sha-256',
+ 'hostssl keycloak_jotwink postgres 127.0.0.1/32 scram-sha-256',
+ 'hostssl jotwink all all reject',
+ 'hostssl keycloak_jotwink all all reject',
+]
+
+def apply(path):
+    if path.is_symlink() or not path.is_file():
+        raise ValueError('HBA path must be a regular non-symlink file')
+    stat = path.stat()
+    lines = path.read_text().splitlines(keepends=True)
+    parsed = [(i, line.split('#', 1)[0].split()) for i, line in enumerate(lines)]
+    scoped = [(i, ' '.join(parts)) for i, parts in parsed if len(parts)>1 and set(parts[1].split(',')) & {'jotwink','keycloak_jotwink'}]
+    broad = [i for i, parts in parsed if len(parts)>1 and parts[0].startswith('host') and 'all' in parts[1].split(',') and not (len(parts) == 5 and parts[-1] == 'reject' and not set(parts[2].split(',')) & {'all', 'jotwink_app', 'keycloak_jotwink_app', 'postgres'} and not any(x.startswith('+') for x in parts[2].split(',')))]
+    fallback = [i for i, parts in parsed if parts == 'host all all all scram-sha-256'.split()]
+    if len(fallback)!=1 or broad != fallback:
+        raise ValueError('Ambiguous shared fallback rules; manual inspection required')
+    if scoped:
+        if [rule for _, rule in scoped] != RULES or max(i for i,_ in scoped)>=fallback[0]:
+            raise ValueError('Partial, reordered or shadowed Jotwink policy; refusing mutation')
+        return
+    lines.insert(fallback[0], '# Jotwink TLS and exact source restrictions\n'+'\n'.join(RULES)+'\n')
+    fd, temporary = tempfile.mkstemp(prefix=path.name+'.jotwink.', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'w') as out:
+            os.fchmod(out.fileno(), stat.st_mode & 0o7777)
+            if (os.fstat(out.fileno()).st_uid, os.fstat(out.fileno()).st_gid)!=(stat.st_uid,stat.st_gid):
+                os.fchown(out.fileno(),stat.st_uid,stat.st_gid)
+            out.write(''.join(lines));out.flush();os.fsync(out.fileno())
+        os.replace(temporary,path)
+    finally:
+        if os.path.exists(temporary):os.unlink(temporary)
+
+if __name__=='__main__':
+    try: apply(Path(sys.argv[1]))
+    except (ValueError,OSError) as error:
+        print(str(error),file=sys.stderr);sys.exit(1)

scripts/run-jotwink-backup-loop.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -eu
+
+interval_seconds=${JOTWINK_BACKUP_INTERVAL_SECONDS:-21600}
+retry_seconds=${JOTWINK_BACKUP_RETRY_SECONDS:-300}
+backup_root=${JOTWINK_BACKUP_ROOT:-/backups}
+
+validate_timing_value() {
+  case "$2" in
+    ''|*[!0-9]*) echo "$1 must be a positive integer." >&2; exit 1 ;;
+  esac
+}
+validate_timing_value JOTWINK_BACKUP_INTERVAL_SECONDS "${interval_seconds}"
+validate_timing_value JOTWINK_BACKUP_RETRY_SECONDS "${retry_seconds}"
+if [ "${interval_seconds}" -lt 300 ] || [ "${retry_seconds}" -lt 30 ]; then
+  echo "Jotwink backup interval must be at least 300 seconds and retry delay at least 30 seconds." >&2
+  exit 1
+fi
+
+if [ "${1:-}" = "healthcheck" ]; then
+  status_file=${backup_root}/last-success.json
+  [ -s "${status_file}" ] || exit 1
+  now=$(date +%s)
+  if modified=$(stat -c %Y "${status_file}" 2>/dev/null); then :; else modified=$(stat -f %m "${status_file}"); fi
+  max_age=$((interval_seconds * 2 + retry_seconds))
+  [ $((now - modified)) -le "${max_age}" ]
+  exit
+fi
+
+while :; do
+  if /usr/local/bin/run-jotwink-backup.sh; then
+    sleep "${interval_seconds}"
+  else
+    echo "Jotwink PostgreSQL backup failed; retrying in ${retry_seconds} seconds." >&2
+    sleep "${retry_seconds}"
+  fi
+done