From 2086ad5ea96108547d02305c4d120adb673744a6 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Thu, 3 Sep 2026 17:50:20 -0400 Subject: [PATCH] fix(ansible): move pgBackRest spool-path off the 10GB root volume pgBackRest's spool-path was never set explicitly, so it defaulted to /var/spool/pgbackrest on the AMI's 10GB root volume instead of /data (the dedicated EBS volume PGDATA lives on). Sets spool-path = /data/pgbackrest_spool in the ansible-managed pgbackrest.conf, and updates the AppArmor profile to allow writes there. The AMI-bake-time directory-creation task added here does not create this directory on any live instance's real /data -- that volume is always a separately-provisioned EBS volume attached at real instance launch, never part of the AMI image at any Packer stage. So the config value and the AppArmor grant here are correct and needed groundwork, but on their own they do not make archive-async safe to enable anywhere in the fleet. The real fix is a runtime mechanism, tracked at INDATA-1153. --- ansible/files/pgbackrest_config/pgbackrest.conf | 2 ++ ansible/files/postgresql_config/sbpostgres_apparmor | 8 ++++++-- ansible/tasks/setup-pgbackrest.yml | 3 ++- nix/packages/supascan/internal/config/defaults.go | 3 +++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ansible/files/pgbackrest_config/pgbackrest.conf b/ansible/files/pgbackrest_config/pgbackrest.conf index f54336c4e8..279ca2fd82 100644 --- a/ansible/files/pgbackrest_config/pgbackrest.conf +++ b/ansible/files/pgbackrest_config/pgbackrest.conf @@ -13,6 +13,8 @@ log-level-console = info log-level-file = detail log-subprocess = y resume = n +# spool-path shares a filesystem with pg_wal so archive-get can hand off WAL segments via rename instead of copy; only reachable once archive-async is enabled. The matching AppArmor grants live in ansible/files/postgresql_config/sbpostgres_apparmor and the audit exclusion in nix/packages/supascan/internal/config/defaults.go -- keep this path in sync across those, plus the directory-creation loop entry in ansible/tasks/setup-pgbackrest.yml (bake-time only; see INDATA-1153 for why that alone doesn't create this directory on any live instance's real /data). +spool-path = /data/pgbackrest_spool start-fast = y # Note: the [supabase] stanza (pg1-path, pg1-socket-path, pg1-user) has been # removed from this file. supabase-admin-agent owns that stanza and writes it diff --git a/ansible/files/postgresql_config/sbpostgres_apparmor b/ansible/files/postgresql_config/sbpostgres_apparmor index ea59999849..651ccbe8b2 100644 --- a/ansible/files/postgresql_config/sbpostgres_apparmor +++ b/ansible/files/postgresql_config/sbpostgres_apparmor @@ -127,7 +127,9 @@ profile sbpostgres flags=(attach_disconnected) { /var/lib/postgresql/data/standby.signal rw, /var/lib/pgbackrest rw, /etc/pgbackrest/conf.d/** rw, - /var/spool/pgbackrest/** rw, + # bare-dir grant needed alongside ** (same as /data/pgdata/pg_wal/) since ** doesn't cover the directory inode itself; path must match spool-path in ansible/files/pgbackrest_config/pgbackrest.conf, which has the full cross-file sync note + /data/pgbackrest_spool/ rw, + /data/pgbackrest_spool/** rw, /var/log/pgbackrest/** rw, /etc/** r, /usr/local/** r, @@ -190,7 +192,9 @@ profile sbpostgres flags=(attach_disconnected) { /var/lib/postgresql/data/standby.signal rw, /var/lib/pgbackrest rw, /etc/pgbackrest/conf.d/** rw, - /var/spool/pgbackrest/** rw, + # bare-dir grant needed alongside ** (same as /data/pgdata/pg_wal/) since ** doesn't cover the directory inode itself; path must match spool-path in ansible/files/pgbackrest_config/pgbackrest.conf, which has the full cross-file sync note + /data/pgbackrest_spool/ rw, + /data/pgbackrest_spool/** rw, /var/log/pgbackrest/** rw, /etc/** r, /usr/local/** r, diff --git a/ansible/tasks/setup-pgbackrest.yml b/ansible/tasks/setup-pgbackrest.yml index 6d4eebcdb1..2c427d402c 100644 --- a/ansible/tasks/setup-pgbackrest.yml +++ b/ansible/tasks/setup-pgbackrest.yml @@ -47,7 +47,8 @@ # (running as the pgbackrest user) cannot read conf files created by adminapi. - {dir: /etc/pgbackrest/conf.d, mode: '02770'} - {dir: /var/lib/pgbackrest} - - {dir: /var/spool/pgbackrest} + # setgid (02770), same reason as conf.d above (postgres_shell and pgbackrest_shell both write here per sbpostgres_apparmor); must match spool-path in pgbackrest.conf; bake-time only, see INDATA-1153 for why that alone doesn't create this on any live instance's real /data + - {dir: /data/pgbackrest_spool, mode: '02770'} - {dir: /var/log/pgbackrest} loop_control: loop_var: backrest_dir diff --git a/nix/packages/supascan/internal/config/defaults.go b/nix/packages/supascan/internal/config/defaults.go index f65a8f19f4..efe23fb1a7 100644 --- a/nix/packages/supascan/internal/config/defaults.go +++ b/nix/packages/supascan/internal/config/defaults.go @@ -63,6 +63,9 @@ var DefaultExclusions = Config{ // PostgreSQL data directory - contents are dynamic database state "/data/pgdata", + // pgBackRest spool - transient WAL archive-get/archive-push queue, not durable state + "/data/pgbackrest_spool", + // Deployment/provisioning tools - internal implementation details "/opt/saltstack",